diff --git a/.bumpversion.toml b/.bumpversion.toml
index 7906eb3efd5..cb662fe9fd9 100644
--- a/.bumpversion.toml
+++ b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "4.0.0-beta.7"
+current_version = "6.0.0-beta.1"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(-(?P<prerelease>(beta|rc))\\.(?P<prerelease_num>\\d+))?"
 serialize = [
     "{major}.{minor}.{patch}-{prerelease}.{prerelease_num}",
@@ -115,6 +115,11 @@ filename = "Cargo.toml"
 search = 'lance-testing = {{ version = "={current_version}"'
 replace = 'lance-testing = {{ version = "={new_version}"'
 
+[[tool.bumpversion.files]]
+filename = "Cargo.toml"
+search = 'lance-tokenizer = {{ version = "={current_version}"'
+replace = 'lance-tokenizer = {{ version = "={new_version}"'
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'fsst = {{ version = "={current_version}"'
@@ -145,4 +150,3 @@ replace = 'version = "{new_version}"'
 filename = "java/pom.xml"
 search = "<version>{current_version}</version>"
 replace = "<version>{new_version}</version>"
-
diff --git a/.github/actions/setup-release-env/action.yml b/.github/actions/setup-release-env/action.yml
index c06f28271bb..8840ebfcb3b 100644
--- a/.github/actions/setup-release-env/action.yml
+++ b/.github/actions/setup-release-env/action.yml
@@ -4,7 +4,7 @@ runs:
   using: "composite"
   steps:
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4
       with:
         python-version: "3.11"
 
@@ -14,7 +14,7 @@ runs:
         pip install bump-my-version packaging PyGithub PyYAML
 
     - name: Set up Rust
-      uses: actions-rs/toolchain@v1
+      uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1
       with:
         toolchain: stable
         override: true
diff --git a/.github/workflows/approve-rc.yml b/.github/workflows/approve-rc.yml
index 622cd65a174..1ffff04c913 100644
--- a/.github/workflows/approve-rc.yml
+++ b/.github/workflows/approve-rc.yml
@@ -13,6 +13,9 @@ on:
         default: false
         type: boolean
 
+permissions:
+  contents: read
+
 jobs:
   approve-rc:
     runs-on: ubuntu-latest
@@ -27,7 +30,7 @@ jobs:
       run: echo "${{ toJSON(github.event.inputs) }}"
 
     - name: Check out repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       with:
         token: ${{ secrets.LANCE_RELEASE_TOKEN }}
         fetch-depth: 0
@@ -73,7 +76,7 @@ jobs:
 
     - name: Create GitHub Release (if not dry run)
       if: ${{ !inputs.dry_run }}
-      uses: softprops/action-gh-release@v2
+      uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
       with:
         tag_name: ${{ steps.approve.outputs.STABLE_TAG }}
         name: ${{ steps.approve.outputs.STABLE_TAG }}
diff --git a/.github/workflows/benchmark-comment-trigger.yml b/.github/workflows/benchmark-comment-trigger.yml
index 1259f4faa05..30aed29711d 100644
--- a/.github/workflows/benchmark-comment-trigger.yml
+++ b/.github/workflows/benchmark-comment-trigger.yml
@@ -13,6 +13,10 @@ on:
   issue_comment:
     types: [created]
 
+permissions:
+  contents: read
+  pull-requests: read
+
 jobs:
   forward-to-bench:
     # Only process comments on PRs that mention @bench-bot and contain 'benchmark'
@@ -24,7 +28,7 @@ jobs:
     steps:
       - name: Get PR details
         id: pr
-        uses: actions/github-script@v7
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
         with:
           script: |
             const pr = await github.rest.pulls.get({
@@ -36,7 +40,7 @@ jobs:
             core.setOutput('head_sha', pr.data.head.sha);
 
       - name: Forward to lance-bench
-        uses: peter-evans/repository-dispatch@v2
+        uses: peter-evans/repository-dispatch@bf47d102fdb849e755b0b0023ea3e81a44b6f570 # v2
         with:
           token: ${{ secrets.LANCE_BENCH_DISPATCH_TOKEN }}
           repository: lancedb/lance-bench
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index e6b5a199001..7874d6fdee2 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -3,6 +3,9 @@ name: Run benchmarks
 on:
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   dataset:
     timeout-minutes: 30
@@ -11,12 +14,12 @@ jobs:
       matrix:
         dataset: ["sift"]
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       with:
         fetch-depth: 0
         lfs: true
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4
       with:
         python-version: "3.10"
     - uses: ./.github/workflows/build_linux_wheel
@@ -32,7 +35,7 @@ jobs:
       run: |
         ./test_dataset.sh
     - name: Archive results
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
         name: ${{ matrix.dataset }}-results
         path: |
diff --git a/.github/workflows/buf-publish.yml b/.github/workflows/buf-publish.yml
index 45e714ac8ed..b7b1acd8579 100644
--- a/.github/workflows/buf-publish.yml
+++ b/.github/workflows/buf-publish.yml
@@ -11,16 +11,19 @@ on:
         default: ''
         type: string
 
+permissions:
+  contents: read
+
 jobs:
   push-module:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       with:
         ref: ${{ inputs.ref || github.ref }}
         fetch-depth: 0
         lfs: true
-    - uses: bufbuild/buf-setup-action@v1
-    - uses: bufbuild/buf-push-action@v1
+    - uses: bufbuild/buf-setup-action@a47c93e0b1648d5651a065437926377d060baa99 # v1.50.0
+    - uses: bufbuild/buf-push-action@a654ff18effe4641ebea4a4ce242c49800728459 # v1.2.0
       with:
         buf_token: ${{ secrets.BUF_TOKEN }}
diff --git a/.github/workflows/build_linux_wheel/action.yml b/.github/workflows/build_linux_wheel/action.yml
index 19098fcf0d4..fbfcff687ce 100644
--- a/.github/workflows/build_linux_wheel/action.yml
+++ b/.github/workflows/build_linux_wheel/action.yml
@@ -32,7 +32,7 @@ runs:
         rm -f python/target/wheels/pylance-*.whl || true
     - name: Build x86_64 Manylinux2014 wheel
       if: ${{ inputs.arm-build == 'false' && inputs.manylinux == '2_17' }}
-      uses: PyO3/maturin-action@v1
+      uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1
       with:
         command: build
         working-directory: python
@@ -48,7 +48,7 @@ runs:
             && rm /tmp/protoc.zip
     - name: Build x86_64 Manylinux {manylinux} wheel
       if: ${{ inputs.arm-build == 'false' && inputs.manylinux != '2_17' }}
-      uses: PyO3/maturin-action@v1
+      uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1
       with:
         command: build
         working-directory: python
@@ -67,7 +67,7 @@ runs:
             && rm /tmp/protoc.zip
     - name: Build Arm Manylinux Wheel
       if: ${{ inputs.arm-build == 'true' }}
-      uses: PyO3/maturin-action@v1
+      uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1
       with:
         command: build
         working-directory: python
diff --git a/.github/workflows/build_mac_wheel/action.yml b/.github/workflows/build_mac_wheel/action.yml
index 3a3af5917fd..9d45bde42aa 100644
--- a/.github/workflows/build_mac_wheel/action.yml
+++ b/.github/workflows/build_mac_wheel/action.yml
@@ -17,7 +17,7 @@ runs:
       run: |
         brew install protobuf
     - name: Build wheel
-      uses: PyO3/maturin-action@v1
+      uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1
       with:
         command: build
         args: ${{ inputs.args }}
diff --git a/.github/workflows/build_windows_wheel/action.yml b/.github/workflows/build_windows_wheel/action.yml
index 28f13656110..03b601db019 100644
--- a/.github/workflows/build_windows_wheel/action.yml
+++ b/.github/workflows/build_windows_wheel/action.yml
@@ -22,12 +22,12 @@ runs:
         Add-Content $env:GITHUB_PATH "C:\protoc\bin"
       shell: powershell
     - name: Build wheel
-      uses: PyO3/maturin-action@v1
+      uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1
       with:
         command: build
         args: ${{ inputs.args }}
         working-directory: python
-    - uses: actions/upload-artifact@v4
+    - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
         name: windows-wheels
         path: python\target\wheels
diff --git a/.github/workflows/cargo-publish.yml b/.github/workflows/cargo-publish.yml
index ea9963315a7..0ca2877475f 100644
--- a/.github/workflows/cargo-publish.yml
+++ b/.github/workflows/cargo-publish.yml
@@ -24,11 +24,15 @@ env:
   CARGO_INCREMENTAL: "0"
   RUSTFLAGS: "-C debuginfo=0"
 
+permissions:
+  contents: read
+
 jobs:
   build:
     # Needs additional disk space for the full build.
-    runs-on: warp-ubuntu-latest-x64-8x
+    runs-on: ubuntu-24.04-8x
     permissions:
+      contents: read
       id-token: write
     timeout-minutes: 60
     env:
@@ -39,7 +43,7 @@ jobs:
       run:
         working-directory: .
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Check if stable release
         id: check_version
         run: |
@@ -60,19 +64,20 @@ jobs:
             echo "Stable version detected: $TAG"
             echo "skip=false" >> $GITHUB_OUTPUT
           fi
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         if: steps.check_version.outputs.skip != 'true'
         with:
           workspaces: rust
       - name: Verify and checkout specified tag
         if: github.event_name == 'workflow_dispatch' && steps.check_version.outputs.skip != 'true'
         run: |
+          TAG="${{ github.event.inputs.tag }}"
           git fetch --all --tags
-          if git rev-parse ${{ github.event.inputs.tag }} >/dev/null 2>&1; then
-            git checkout ${{ github.event.inputs.tag }}
-            echo "Successfully checked out tag ${{ github.event.inputs.tag }}"
+          if git rev-parse "$TAG" >/dev/null 2>&1; then
+            git checkout "$TAG"
+            echo "Successfully checked out tag $TAG"
           else
-            echo "Error: Tag ${{ github.event.inputs.tag }} does not exist"
+            echo "Error: Tag $TAG does not exist"
             echo "Available tags:"
             git tag -l
             exit 1
@@ -85,7 +90,7 @@ jobs:
       # Wait until https://github.com/rust-lang/crates-io-auth-action/issues/51 fixed
       # - uses: rust-lang/crates-io-auth-action@v1
       #   id: auth
-      - uses: albertlockett/publish-crates@v2.2
+      - uses: albertlockett/publish-crates@85f0989f1298bc3889830b5fc28122c7586efeec # v2.2
         if: steps.check_version.outputs.skip != 'true'
         with:
           # registry-token: ${{ steps.auth.outputs.token }}
@@ -102,7 +107,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - uses: ./.github/actions/create-failure-issue
         with:
           job-results: ${{ toJSON(needs) }}
diff --git a/.github/workflows/ci-benchmarks.yml b/.github/workflows/ci-benchmarks.yml
index ebe55cd146e..009c13fcbbd 100644
--- a/.github/workflows/ci-benchmarks.yml
+++ b/.github/workflows/ci-benchmarks.yml
@@ -6,6 +6,9 @@ on:
     branches:
       - main
 
+permissions:
+  contents: read
+
 jobs:
   bench_regress:
     timeout-minutes: 120
@@ -19,21 +22,21 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 0
           lfs: true
       - name: Authenticate with GCS
-        uses: "google-github-actions/auth@v2"
+        uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2
         with:
           credentials_json: "${{ secrets.GCLOUD_BENCH_STORAGE_USER_KEY }}"
       - name: Install bencher
-        uses: bencherdev/bencher@main
+        uses: bencherdev/bencher@8151077aa7b1bceaac11c4b308265417cae60e2b # v0.5.10
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: 3.11 # Ray does not support 3.12 yet.
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           workspaces: python
       - name: Install dependencies
diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml
index 571a740bb5d..00b54f0f29e 100644
--- a/.github/workflows/claude-code-review.yml
+++ b/.github/workflows/claude-code-review.yml
@@ -1,7 +1,7 @@
 name: Claude Code Review
 
 on:
-  pull_request_target:
+  pull_request:
     types: [opened]
     # Optional: Only run on specific file changes
     # paths:
@@ -12,33 +12,30 @@ on:
 
 jobs:
   claude-review:
-    if: github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR'
-    # Optional: Filter by PR author
-    # if: |
-    #   github.event.pull_request.user.login == 'external-contributor' ||
-    #   github.event.pull_request.user.login == 'new-developer' ||
-    #   github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'
+    if: |
+      github.event.pull_request.author_association == 'MEMBER' ||
+      github.event.pull_request.author_association == 'OWNER'
 
     runs-on: ubuntu-latest
     permissions:
       contents: read
       pull-requests: write
       issues: write
-      id-token: write
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 1
 
       - name: Run Claude Code Review
         id: claude-review
-        uses: anthropics/claude-code-action@v1
+        uses: anthropics/claude-code-action@3ac52d0da9f8ec9ca7b4dc23bb477e36ef9c77a9 # v1.0.79
+        env:
+          CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: '1'
         with:
           anthropic_api_key: ${{ secrets.CLAUDE_TOKEN }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          allowed_non_write_users: ${{ github.event.pull_request.user.login }}
           prompt: |
             REPO: ${{ github.repository }}
             PR NUMBER: ${{ github.event.pull_request.number }}
@@ -65,5 +62,5 @@ jobs:
           # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
           # or https://code.claude.com/docs/en/cli-reference for available options
           claude_args: |
-            --allowed-tools "Bash(gh issue view:*),Bash(gh search:*),Bash(gh issue list:*),Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr list:*)"
+            --allowed-tools "Bash(gh pr comment ${{ github.event.pull_request.number }}:*),Bash(gh pr diff ${{ github.event.pull_request.number }}:*),Bash(gh pr view ${{ github.event.pull_request.number }}:*)"
             --model "claude-opus-4-6"
diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml
index bbb84322a25..3cb01a00fc8 100644
--- a/.github/workflows/claude.yml
+++ b/.github/workflows/claude.yml
@@ -13,26 +13,31 @@ on:
 jobs:
   claude:
     if: |
-      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
-      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
+      (
+        (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude') &&
+          contains(fromJSON('["MEMBER","COLLABORATOR","OWNER","CONTRIBUTOR"]'), github.event.comment.author_association)) ||
+        (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude') &&
+          contains(fromJSON('["MEMBER","COLLABORATOR","OWNER","CONTRIBUTOR"]'), github.event.comment.author_association)) ||
+        (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude') &&
+          contains(fromJSON('["MEMBER","COLLABORATOR","OWNER","CONTRIBUTOR"]'), github.event.review.author_association)) ||
+        (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')) &&
+          contains(fromJSON('["MEMBER","COLLABORATOR","OWNER","CONTRIBUTOR"]'), github.event.issue.author_association))
+      )
     runs-on: ubuntu-latest
     permissions:
       contents: read
       pull-requests: write
       issues: write
-      id-token: write
       actions: read # Required for Claude to read CI results on PRs
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 1
 
       - name: Run Claude Code
         id: claude
-        uses: anthropics/claude-code-action@v1
+        uses: anthropics/claude-code-action@26ec041249acb0a944c0a47b6c0c13f05dbc5b44 # v1
         with:
           anthropic_api_key: ${{ secrets.CLAUDE_TOKEN }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/codex-backport-pr.yml b/.github/workflows/codex-backport-pr.yml
index 164ca428afe..0b43cca2460 100644
--- a/.github/workflows/codex-backport-pr.yml
+++ b/.github/workflows/codex-backport-pr.yml
@@ -35,7 +35,7 @@ permissions:
 
 jobs:
   backport:
-    runs-on: warp-ubuntu-latest-x64-4x
+    runs-on: ubuntu-24.04-4x
     timeout-minutes: 60
     env:
       CC: clang
@@ -48,13 +48,13 @@ jobs:
           echo "guidelines = ${{ inputs.guidelines }}"
 
       - name: Checkout Repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 0
           persist-credentials: true
 
       - name: Set up Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
         with:
           node-version: 20
 
@@ -62,14 +62,14 @@ jobs:
         run: npm install -g @openai/codex
 
       - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
+        uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
         with:
           toolchain: stable
           components: clippy, rustfmt
 
-      - uses: rui314/setup-mold@v1
+      - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
 
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
 
       - name: Install system dependencies
         run: |
@@ -77,7 +77,7 @@ jobs:
           sudo apt-get install -y protobuf-compiler libssl-dev
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: '3.11'
 
@@ -86,7 +86,7 @@ jobs:
           pip install maturin ruff pytest pyarrow pandas polars
 
       - name: Set up Java
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
         with:
           distribution: temurin
           java-version: '11'
diff --git a/.github/workflows/codex-fix-ci.yml b/.github/workflows/codex-fix-ci.yml
index d311db94e4d..b15bd367c7b 100644
--- a/.github/workflows/codex-fix-ci.yml
+++ b/.github/workflows/codex-fix-ci.yml
@@ -35,7 +35,7 @@ permissions:
 
 jobs:
   fix-ci:
-    runs-on: warp-ubuntu-latest-x64-4x
+    runs-on: ubuntu-24.04-4x
     timeout-minutes: 60
     env:
       CC: clang
@@ -48,14 +48,14 @@ jobs:
           echo "guidelines = ${{ inputs.guidelines }}"
 
       - name: Checkout Repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           ref: ${{ inputs.branch }}
           fetch-depth: 0
           persist-credentials: true
 
       - name: Set up Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
         with:
           node-version: 20
 
@@ -63,14 +63,14 @@ jobs:
         run: npm install -g @openai/codex
 
       - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
+        uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
         with:
           toolchain: stable
           components: clippy, rustfmt
 
-      - uses: rui314/setup-mold@v1
+      - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
 
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
 
       - name: Install system dependencies
         run: |
@@ -78,7 +78,7 @@ jobs:
           sudo apt-get install -y protobuf-compiler libssl-dev
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: '3.11'
 
@@ -87,7 +87,7 @@ jobs:
           pip install maturin ruff pytest pyarrow pandas polars
 
       - name: Set up Java
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
         with:
           distribution: temurin
           java-version: '11'
diff --git a/.github/workflows/create-rc.yml b/.github/workflows/create-rc.yml
index 86f3df2c88c..1dee4c260b6 100644
--- a/.github/workflows/create-rc.yml
+++ b/.github/workflows/create-rc.yml
@@ -13,6 +13,9 @@ on:
         default: false
         type: boolean
 
+permissions:
+  contents: read
+
 jobs:
   create-rc:
     runs-on: ubuntu-latest
@@ -24,7 +27,7 @@ jobs:
       run: echo "${{ toJSON(github.event.inputs) }}"
 
     - name: Check out repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       with:
         token: ${{ secrets.LANCE_RELEASE_TOKEN }}
         fetch-depth: 0
@@ -84,7 +87,7 @@ jobs:
 
     - name: Create GitHub Pre-Release (if not dry run)
       if: ${{ !inputs.dry_run }}
-      uses: softprops/action-gh-release@v2
+      uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
       with:
         tag_name: ${{ steps.create.outputs.RC_TAG }}
         name: ${{ steps.create.outputs.RC_TAG }}
diff --git a/.github/workflows/create-release-branch.yml b/.github/workflows/create-release-branch.yml
index a338977270f..1a6fb9b78fb 100644
--- a/.github/workflows/create-release-branch.yml
+++ b/.github/workflows/create-release-branch.yml
@@ -14,6 +14,9 @@ on:
         default: false
         type: boolean
 
+permissions:
+  contents: read
+
 jobs:
   create-release-branch:
     runs-on: ubuntu-latest
@@ -28,7 +31,7 @@ jobs:
       run: echo "${{ toJSON(github.event.inputs) }}"
 
     - name: Check out repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       with:
         ref: ${{ inputs.source_release_branch || 'main' }}
         token: ${{ secrets.LANCE_RELEASE_TOKEN }}
@@ -90,7 +93,7 @@ jobs:
 
     - name: Create GitHub Pre-Release (if not dry run)
       if: ${{ !inputs.dry_run }}
-      uses: softprops/action-gh-release@v2
+      uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
       with:
         tag_name: ${{ steps.create_branch.outputs.RC_TAG }}
         name: ${{ steps.create_branch.outputs.RC_TAG }}
diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml
index 02c0ce3b751..2eda3de66b4 100644
--- a/.github/workflows/docs-check.yml
+++ b/.github/workflows/docs-check.yml
@@ -13,6 +13,9 @@ on:
       - docs/**
       - .github/workflows/docs-check.yml
 
+permissions:
+  contents: read
+
 env:
   RUSTFLAGS: "-C debuginfo=0"
   # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
@@ -25,13 +28,13 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: "Set up Python"
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version-file: "docs/pyproject.toml"
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
           enable-cache: true
       - name: Check links
diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml
index a8f83f6ce11..36628e92ca7 100644
--- a/.github/workflows/docs-deploy.yml
+++ b/.github/workflows/docs-deploy.yml
@@ -24,34 +24,34 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Checkout lance-spark
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           repository: lancedb/lance-spark
           path: lance-spark
       - name: Checkout lance-namespace
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           repository: lancedb/lance-namespace
           path: lance-namespace
       - name: Checkout lance-ray
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           repository: lancedb/lance-ray
           path: lance-ray
       - name: Checkout lance-huggingface
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           repository: lance-format/lance-huggingface
           path: lance-huggingface
       - name: Checkout lance-namespace-impls
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           repository: lance-format/lance-namespace-impls
           path: lance-namespace-impls
       - name: Checkout lance-trino
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           repository: lance-format/lance-trino
           path: lance-trino
@@ -60,11 +60,11 @@ jobs:
           git config user.name github-actions[bot]
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
       - name: "Set up Python"
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version-file: "docs/pyproject.toml"
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
           enable-cache: true
       - name: Copy lance-namespace docs
diff --git a/.github/workflows/file_verification.yml b/.github/workflows/file_verification.yml
index 94f36443ba8..cf06ad83d0b 100644
--- a/.github/workflows/file_verification.yml
+++ b/.github/workflows/file_verification.yml
@@ -8,13 +8,13 @@ on:
 jobs:
   run:
     timeout-minutes: 45
-    runs-on: warp-ubuntu-latest-x64-8x
+    runs-on: ubuntu-24.04-8x
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: "3.11"
 
@@ -32,7 +32,7 @@ jobs:
           echo "AWS credentials validation successful"
 
       - name: Set up AWS CLI
-        uses: aws-actions/configure-aws-credentials@v3
+        uses: aws-actions/configure-aws-credentials@50ac8dd1e1b10d09dac7b8727528b91bed831ac0 # v3
         with:
           aws-access-key-id: ${{ secrets.LANCEDB_S3_READER_ACCESS_KEY }}
           aws-secret-access-key: ${{ secrets.LANCEDB_S3_READER_SECRET }}
diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml
index 30d07658d17..a51cf969a87 100644
--- a/.github/workflows/java-publish.yml
+++ b/.github/workflows/java-publish.yml
@@ -24,18 +24,21 @@ on:
         required: false
         type: string
 
+permissions:
+  contents: read
+
 jobs:
   linux-arm64:
     name: Build on Linux Arm64
-    runs-on: warp-ubuntu-latest-arm64-8x
+    runs-on: ubuntu-24.04-arm64-8x
     timeout-minutes: 60
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           ref: ${{ inputs.ref || github.ref }}
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
       - name: Check glibc version outside docker
         run: ldd --version
       - name: Build and run in Debian 10 Arm64 container
@@ -97,7 +100,7 @@ jobs:
 
             cargo build --release
           "
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: liblance_jni_linux_arm_64.zip
           path: java/lance-jni/target/release/liblance_jni.so
@@ -109,11 +112,11 @@ jobs:
     timeout-minutes: 60
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           ref: ${{ inputs.ref || github.ref }}
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
       - name: Check glibc version outside docker
         run: ldd --version
       - name: Build and run in Debian 10 X86-64 container
@@ -175,7 +178,7 @@ jobs:
 
             cargo build --release
           "
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: liblance_jni_linux_x86_64.zip
           path: java/lance-jni/target/release/liblance_jni.so
@@ -190,12 +193,12 @@ jobs:
       - linux-x86
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           ref: ${{ inputs.ref || github.ref }}
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
       - name: Set up Java 11
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
         with:
           distribution: corretto
           java-version: 11
@@ -205,13 +208,13 @@ jobs:
           server-password: SONATYPE_TOKEN
           gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
           gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }}
-      - uses: Homebrew/actions/setup-homebrew@master
+      - uses: Homebrew/actions/setup-homebrew@50b8c2ab4a835c38897ed2c56c293b07167c0b59 # master 2026-03-07
       - name: Install dependencies
         run: |
           brew install protobuf
           brew install gpg
       - name: Download artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
       - name: Copy native libs
         run: |
           mkdir -p ./java/target/classes/nativelib/linux-x86-64 ./java/target/classes/nativelib/linux-aarch64
@@ -250,7 +253,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - uses: ./.github/actions/create-failure-issue
         with:
           job-results: ${{ toJSON(needs) }}
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index f4b8222bbaa..83403988244 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -20,14 +20,17 @@ env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: "1"
 
+permissions:
+  contents: read
+
 jobs:
   rust-clippy-fmt:
     runs-on: ubuntu-24.04
     name: Rust Clippy and Fmt Check
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           workspaces: |
             lance
@@ -36,9 +39,9 @@ jobs:
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
-      - uses: rui314/setup-mold@v1
+      - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
       - name: Install cargo-llvm-cov
-        uses: taiki-e/install-action@cargo-llvm-cov
+        uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # cargo-llvm-cov
       - name: Run cargo fmt
         working-directory: java/lance-jni
         run: cargo fmt --check
@@ -47,7 +50,7 @@ jobs:
         run: cargo clippy --all-targets -- -D warnings
 
   build-and-test-java:
-    runs-on: warp-ubuntu-latest-x64-4x
+    runs-on: ubuntu-24.04-4x
     timeout-minutes: 60
     strategy:
       matrix:
@@ -55,23 +58,23 @@ jobs:
     name: Build and Test with Java ${{ matrix.java-version }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Install dependencies
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
       # pin the toolchain version to avoid surprises
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1
         with:
           toolchain: stable
-      - uses: rui314/setup-mold@v1
+      - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
       - name: Install cargo-llvm-cov
-        uses: taiki-e/install-action@cargo-llvm-cov
-      - uses: Swatinem/rust-cache@v2
+        uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # cargo-llvm-cov
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           workspaces: java/lance-jni -> ../target/rust-maven-plugin/lance-jni
       - name: Set up Java ${{ matrix.java-version }}
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
         with:
           distribution: temurin
           java-version: ${{ matrix.java-version }}
diff --git a/.github/workflows/license-header-check.yml b/.github/workflows/license-header-check.yml
index 1e321687681..488ca65e585 100644
--- a/.github/workflows/license-header-check.yml
+++ b/.github/workflows/license-header-check.yml
@@ -12,14 +12,21 @@ on:
       - rust/**
       - python/**
       - protos/**
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Install license-header-checker
-        run: curl -s https://raw.githubusercontent.com/lluissm/license-header-checker/master/install.sh | bash
+        run: |
+          set -euo pipefail
+          curl -sfSL https://raw.githubusercontent.com/lluissm/license-header-checker/master/install.sh -o /tmp/install-lhc.sh
+          bash /tmp/install-lhc.sh
+          rm -f /tmp/install-lhc.sh
       - name: Check license headers (rust)
         run: ./bin/license-header-checker -a -v ./rust/license_header.txt rust rs && [[ -z `git status -s` ]]
       - name: Check license headers (python)
diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml
index 0c2c609e438..955dcec3860 100644
--- a/.github/workflows/nightly_run.yml
+++ b/.github/workflows/nightly_run.yml
@@ -5,13 +5,18 @@ on:
     - cron: "0 0 * * *" # Runs every day at midnight UTC
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   run:
     runs-on: ubuntu-24.04
     if: github.repository == 'lancedb/lance'
+    permissions:
+      actions: write
     steps:
       - name: Nightly Run File Verification Workflow
-        uses: benc-uk/workflow-dispatch@v1
+        uses: benc-uk/workflow-dispatch@7a027648b88c2413826b6ddd6c76114894dc5ec4 # v1
         with:
           workflow: file_verification.yml
           ref: main
@@ -19,13 +24,13 @@ jobs:
 
   jumbo-tests:
     # jumbo tests need more resources
-    runs-on: warp-ubuntu-latest-x64-8x
+    runs-on: ubuntu-24.04-8x
     if: github.repository == 'lancedb/lance'
     timeout-minutes: 60
     permissions:
       contents: read
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Install dependencies
         run: |
           sudo apt update
diff --git a/.github/workflows/notebook.yml b/.github/workflows/notebook.yml
index 6efec33e92c..d428d7d8b58 100644
--- a/.github/workflows/notebook.yml
+++ b/.github/workflows/notebook.yml
@@ -15,6 +15,9 @@ on:
       - protos/**
       - notebooks/**
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -24,12 +27,12 @@ jobs:
     timeout-minutes: 30
     runs-on: "ubuntu-22.04"
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       with:
         fetch-depth: 0
         lfs: true
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4
       with:
         python-version: "3.10"
     - uses: ./.github/workflows/build_linux_wheel
diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml
index 96b0c5a338c..51c899f56a3 100644
--- a/.github/workflows/pr-title.yml
+++ b/.github/workflows/pr-title.yml
@@ -17,7 +17,7 @@ jobs:
     name: Label PR
     runs-on: ubuntu-latest
     steps:
-      - uses: srvaroa/labeler@master
+      - uses: srvaroa/labeler@bf262763a8a8e191f5847873aecc0f29df84f957 # v1.14.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
@@ -28,7 +28,7 @@ jobs:
     name: Verify PR title / description conforms to semantic-release
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
         with:
           node-version: "20"
       # These rules are disabled because Github will always ensure there
@@ -44,14 +44,17 @@ jobs:
               "body-leading-blank": [0, "always"]
             }
           }' > .commitlintrc.js
-      - run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
-        env:
-          COMMIT_MSG: >
-            ${{ github.event.pull_request.title }}
-
-            ${{ github.event.pull_request.body }}
+      - name: Write commit message to file
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        with:
+          script: |
+            const fs = require('fs');
+            const title = context.payload.pull_request.title || '';
+            const body = context.payload.pull_request.body || '';
+            fs.writeFileSync(process.env.RUNNER_TEMP + '/commit_msg.txt', title + '\n\n' + body);
+      - run: npx commitlint --extends @commitlint/config-conventional --verbose < "$RUNNER_TEMP/commit_msg.txt"
       - if: failure()
-        uses: actions/github-script@v7
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
         with:
           script: |
             const message = `**ACTION NEEDED**
diff --git a/.github/workflows/publish-beta.yml b/.github/workflows/publish-beta.yml
index 0ef837e124d..a3f92f4a8e0 100644
--- a/.github/workflows/publish-beta.yml
+++ b/.github/workflows/publish-beta.yml
@@ -14,6 +14,9 @@ on:
         default: false
         type: boolean
 
+permissions:
+  contents: read
+
 jobs:
   publish-beta:
     runs-on: ubuntu-latest
@@ -27,7 +30,7 @@ jobs:
       run: echo "${{ toJSON(github.event.inputs) }}"
 
     - name: Check out repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       with:
         token: ${{ secrets.LANCE_RELEASE_TOKEN }}
         fetch-depth: 0
@@ -80,7 +83,7 @@ jobs:
 
     - name: Create GitHub Pre-Release (if not dry run)
       if: ${{ !inputs.dry_run }}
-      uses: softprops/action-gh-release@v2
+      uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
       with:
         tag_name: ${{ steps.publish.outputs.BETA_TAG }}
         name: ${{ steps.publish.outputs.BETA_TAG }}
diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml
index 20195e679f2..b2bfe284fb5 100644
--- a/.github/workflows/pypi-publish.yml
+++ b/.github/workflows/pypi-publish.yml
@@ -26,6 +26,9 @@ on:
       - ".github/workflows/build_windows_wheel/**"
       - ".github/workflows/upload_wheel/**"
 
+permissions:
+  contents: read
+
 jobs:
   linux:
     timeout-minutes: 60
@@ -45,20 +48,20 @@ jobs:
           - platform: aarch64
             manylinux: "2_17"
             extra_args: ""
-            runner: warp-ubuntu-latest-arm64-4x
+            runner: ubuntu-24.04-arm64-4x
           - platform: aarch64
             manylinux: "2_28"
             extra_args: "--features fp16kernels"
-            runner: warp-ubuntu-latest-arm64-4x
+            runner: ubuntu-24.04-arm64-4x
     runs-on: ${{ matrix.config.runner }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           ref: ${{ inputs.ref || github.ref }}
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4
         with:
           python-version: 3.${{ matrix.python-minor-version }}
       - name: Handle tag
@@ -82,7 +85,7 @@ jobs:
           manylinux: ${{ matrix.config.manylinux }}
       - name: Upload wheels as artifacts
         if: github.event_name == 'workflow_dispatch'
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: pylance-debug-manylinux_${{ matrix.config.manylinux }}_${{ matrix.config.platform }}
           path: python/target/wheels/*.whl
@@ -105,13 +108,13 @@ jobs:
     env:
       MACOSX_DEPLOYMENT_TARGET: 10.15
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           ref: ${{ inputs.ref || github.ref }}
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4
         with:
           python-version: 3.13
       - name: Handle tag
@@ -133,7 +136,7 @@ jobs:
           args: "--release ${{ (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && !inputs.debug)) && '--strip' || '' }} --target ${{ matrix.config.target }} --features fp16kernels"
       - name: Upload wheels as artifacts
         if: github.event_name == 'workflow_dispatch'
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: pylance-debug-macosx_arm64
           path: python/target/wheels/*.whl
@@ -146,18 +149,18 @@ jobs:
           repo: ${{ steps.handle_tag.outputs.repo }}
   windows:
     timeout-minutes: 60
-    runs-on: windows-latest
+    runs-on: windows-latest-4x
     strategy:
       matrix:
         python-minor-version: ["9"]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           ref: ${{ inputs.ref || github.ref }}
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4
         with:
           python-version: 3.${{ matrix.python-minor-version }}
       - name: Handle tag
@@ -180,7 +183,7 @@ jobs:
           args: "--release ${{ (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && !inputs.debug)) && '--strip' || '' }}"
       - name: Upload wheels as artifacts
         if: github.event_name == 'workflow_dispatch'
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: pylance-debug-win_amd64
           path: python/target/wheels/*.whl
@@ -200,7 +203,7 @@ jobs:
       issues: write
     if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - uses: ./.github/actions/create-failure-issue
         with:
           job-results: ${{ toJSON(needs) }}
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 6755d38087c..c60d8d4e8ce 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -19,6 +19,9 @@ on:
       - .github/workflows/build_mac_wheel/**
       - .github/workflows/run_tests/**
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -39,7 +42,7 @@ env:
 jobs:
   lint:
     timeout-minutes: 45
-    runs-on: "warp-ubuntu-latest-x64-4x"
+    runs-on: "ubuntu-24.04-4x"
     defaults:
       run:
         shell: bash
@@ -49,21 +52,21 @@ jobs:
       CC: clang-18
       CXX: clang++-18
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: 3.11 # Ray does not support 3.12 yet.
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           workspaces: python
           prefix-key: ${{ env.CACHE_PREFIX }}
       - name: Install linting tools
         run: |
-          pip install ruff==0.11.2 maturin tensorflow tqdm ray[data] pyright datasets polars[pyarrow,pandas]
+          pip install ruff==0.11.2 maturin tensorflow tqdm ray[data] pyright datasets polars[pyarrow,pandas] lance-namespace
           pip install torch --index-url https://download.pytorch.org/whl/cpu
       - name: Lint Python
         run: |
@@ -96,21 +99,21 @@ jobs:
       matrix:
         python-minor-version: ["9", "13"]
     name: "Python Linux 3.${{ matrix.python-minor-version }} x86_64"
-    runs-on: "warp-ubuntu-latest-x64-4x"
+    runs-on: "ubuntu-24.04-4x"
     defaults:
       run:
         shell: bash
         working-directory: python
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: 3.${{ matrix.python-minor-version }}
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           workspaces: python
           prefix-key: ${{ env.CACHE_PREFIX }}
@@ -122,7 +125,7 @@ jobs:
           memtest: true
       - name: Upload wheels as artifacts
         if: ${{ matrix.python-minor-version == '13' }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: linux-wheels
           path: python/target/wheels/pylance-*.whl
@@ -137,16 +140,16 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: 3.13
       - name: Download wheels
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: linux-wheels
           path: python/wheels
@@ -161,22 +164,22 @@ jobs:
 
   linux-arm:
     timeout-minutes: 45
-    runs-on: warp-ubuntu-latest-arm64-4x
+    runs-on: ubuntu-24.04-arm64-4x
     name: Python Linux 3.13 ARM
     defaults:
       run:
         shell: bash
         working-directory: python
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: 3.13
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           workspaces: python
           prefix-key: ${{ env.CACHE_PREFIX }}
@@ -199,15 +202,15 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: "3.13"
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           workspaces: python
           prefix-key: ${{ env.CACHE_PREFIX }}
@@ -219,18 +222,18 @@ jobs:
           skip-torch: "true"
 
   windows:
-    runs-on: windows-latest
+    runs-on: windows-latest-4x
     timeout-minutes: 90
     defaults:
       run:
         shell: powershell
         working-directory: python
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 0
           lfs: true
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           workspaces: python
           prefix-key: ${{ env.CACHE_PREFIX }}
@@ -240,21 +243,21 @@ jobs:
       - uses: ./.github/workflows/run_tests
   aws-integtest:
     timeout-minutes: 45
-    runs-on: "warp-ubuntu-latest-x64-4x"
+    runs-on: "ubuntu-24.04-4x"
     defaults:
       run:
         shell: bash
         working-directory: python
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: "3.11" # TODO: upgrade when ray supports 3.12
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           workspaces: python
           prefix-key: ${{ env.CACHE_PREFIX }}
diff --git a/.github/workflows/recurring-tests.yml b/.github/workflows/recurring-tests.yml
index df99d6513ef..dd8205a9567 100644
--- a/.github/workflows/recurring-tests.yml
+++ b/.github/workflows/recurring-tests.yml
@@ -5,6 +5,9 @@ on:
     - cron: "0 0 * * 0" # Runs at 00:00 UTC every Sunday
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   get-pylance-versions:
     runs-on: ubuntu-latest
@@ -59,7 +62,7 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 0
           lfs: true
@@ -68,7 +71,7 @@ jobs:
           sudo apt update
           sudo apt install -y protobuf-compiler
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: "3.13"
       - name: Install dependencies
@@ -76,7 +79,7 @@ jobs:
         shell: bash
         run: |
           pip install -e ".[tests]"
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           workspaces: python
       - name: Install Pylance
@@ -100,7 +103,7 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           fetch-depth: 0
           lfs: true
@@ -109,10 +112,10 @@ jobs:
           sudo apt update
           sudo apt install -y protobuf-compiler
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: "3.13"
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           workspaces: python
       - name: Install Lance
diff --git a/.github/workflows/run_tests/action.yml b/.github/workflows/run_tests/action.yml
index 64d009f7a2d..ab761532eeb 100644
--- a/.github/workflows/run_tests/action.yml
+++ b/.github/workflows/run_tests/action.yml
@@ -18,7 +18,7 @@ runs:
   steps:
     - name: Setup MSVC for torch.compile
       if: runner.os == 'Windows'
-      uses: ilammy/msvc-dev-cmd@v1
+      uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1
     - name: Install dependencies
       working-directory: python
       shell: bash
diff --git a/.github/workflows/rust-benchmark.yml b/.github/workflows/rust-benchmark.yml
index 7f26fb4e305..bb0960148a9 100644
--- a/.github/workflows/rust-benchmark.yml
+++ b/.github/workflows/rust-benchmark.yml
@@ -40,7 +40,7 @@ jobs:
           sudo apt update
           sudo apt install -y protobuf-compiler
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Run linalg benchmarks
         working-directory: ./rust/lance-linalg
         run: |
@@ -53,7 +53,7 @@ jobs:
           cargo bench --bench sq --bench hnsw --bench inverted --bench pq_dist_table --bench pq_assignment -- --output-format bencher | tee -a ../../output.txt
       - name: Store benchmark result
         if: github.event_name != 'pull_request'
-        uses: benchmark-action/github-action-benchmark@v1
+        uses: benchmark-action/github-action-benchmark@a7bc2366eda11037936ea57d811a43b3418d3073 # v1.21.0
         with:
           name: Lance Rust Benchmarks
           tool: "cargo"
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 275dbd32323..6ba6a1723fc 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -17,6 +17,9 @@ on:
       - Cargo.lock
       - deny.toml
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -32,8 +35,8 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 15
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1
         with:
           components: rustfmt
       - name: Check formatting
@@ -43,7 +46,7 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Install dependencies
         run: |
           sudo apt update
@@ -53,11 +56,12 @@ jobs:
 
   clippy:
     permissions:
+      contents: read
       checks: write
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
       - name: Install dependencies
         run: |
           sudo apt update
@@ -73,14 +77,14 @@ jobs:
     name: Check Rust dependencies (cargo-deny)
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
-      - uses: EmbarkStudios/cargo-deny-action@v2
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: EmbarkStudios/cargo-deny-action@3fd3802e88374d3fe9159b834c7714ec57d6c979 # v2
         with:
           log-level: warn
           command: check
 
   linux-build:
-    runs-on: "warp-ubuntu-latest-x64-4x"
+    runs-on: "ubuntu-24.04-8x"
     timeout-minutes: 60
     env:
       # Need up-to-date compilers for kernels
@@ -89,14 +93,15 @@ jobs:
       # Treat warnings as errors to catch issues early
       RUSTFLAGS: "-D warnings"
     steps:
-      - uses: actions/checkout@v4
-      # pin the toolchain version to avoid surprises
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      # pin the toolchain version to avoid surprises (ethnum 1.5.2
+      # fails on nightly > 2026-04-16 due to a transmute size check)
       - name: Setup rust toolchain
         run: |
-          rustup toolchain install nightly
-          rustup default nightly
-      - uses: rui314/setup-mold@v1
-      - uses: Swatinem/rust-cache@v2
+          rustup toolchain install nightly-2026-04-16
+          rustup default nightly-2026-04-16
+      - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
       - name: Install dependencies
         run: |
           sudo apt update
@@ -104,13 +109,13 @@ jobs:
       - name: Start DynamodDB and S3
         run: docker compose -f docker-compose.yml up -d --wait
       - name: Install cargo-llvm-cov
-        uses: taiki-e/install-action@cargo-llvm-cov
+        uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # cargo-llvm-cov
       - name: Run tests
         run: |
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -`
-          cargo +nightly llvm-cov --profile ci --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES}
+          cargo +nightly-2026-04-16 llvm-cov --profile ci --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES}
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           codecov_yml_path: codecov.yml
@@ -118,16 +123,16 @@ jobs:
           flags: unittests
           fail_ci_if_error: false
   linux-arm:
-    runs-on: warp-ubuntu-latest-arm64-4x
+    runs-on: ubuntu-24.04-arm64-8x
     timeout-minutes: 75
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Setup rust toolchain
         run: |
           rustup toolchain install stable
           rustup default stable
-      - uses: rui314/setup-mold@v1
-      - uses: Swatinem/rust-cache@v2
+      - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
       - name: Install dependencies
         run: |
           sudo apt -y -qq update
@@ -143,19 +148,19 @@ jobs:
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -`
           cargo test --profile ci --locked --features ${ALL_FEATURES}
   query-integration-tests:
-    runs-on: warp-ubuntu-latest-x64-4x
+    runs-on: ubuntu-24.04-4x
     timeout-minutes: 75
     env:
       # We use opt-level 1 which makes some tests 5x faster to run.
       RUSTFLAGS: "-C debuginfo=1 -C opt-level=1"
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Setup rust toolchain
         run: |
           rustup toolchain install stable
           rustup default stable
-      - uses: rui314/setup-mold@v1
-      - uses: Swatinem/rust-cache@v2
+      - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           cache-targets: false
           cache-workspace-crates: true
@@ -170,14 +175,14 @@ jobs:
         run: |
           cargo test --locked -p lance --no-default-features --features fp16kernels,slow_tests --test integration_tests
   build-no-lock:
-    runs-on: warp-ubuntu-latest-x64-8x
+    runs-on: ubuntu-24.04-8x
     timeout-minutes: 30
     env:
       # Need up-to-date compilers for kernels
       CC: clang
       CXX: clang++
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Set up Rust
         run: |
           rustup update stable
@@ -185,7 +190,7 @@ jobs:
       # Remote cargo.lock to force a fresh build
       - name: Remove Cargo.lock
         run: rm -f Cargo.lock
-      - uses: rui314/setup-mold@v1
+      - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
       - name: Install dependencies
         run: |
           sudo apt update
@@ -205,7 +210,7 @@ jobs:
       run:
         working-directory: ./rust
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Select new xcode
         # Default XCode right now is 15.0.1, which contains a bug that causes
         # backtraces to not show properly. See:
@@ -216,7 +221,7 @@ jobs:
       - name: Set up Rust
         run: |
           rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
       - name: Build tests
         run: |
           cargo test --profile ci --locked --features fp16kernels,cli,dynamodb,substrait --no-run
@@ -227,13 +232,13 @@ jobs:
         run: |
           cargo check --profile ci --benches --features fp16kernels,cli,dynamodb,substrait
   windows-build:
-    runs-on: warp-windows-latest-x64-4x
+    runs-on: windows-latest-4x
     defaults:
       run:
         working-directory: rust
     steps:
-      - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
       - name: Install Protoc v21.12
         working-directory: C:\
         run: |
@@ -262,10 +267,10 @@ jobs:
       CC: clang
       CXX: clang++
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           submodules: true
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
       - name: Install dependencies
         run: |
           sudo apt update
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index d57ae0c396c..b1779b7b8ba 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -13,7 +13,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v10
+      - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10
         with:
           close-pr-message: "Thank you for your contribution. This PR has been inactive for a while, so we're closing it to free up bandwidth. Feel free to reopen it if you still find it useful."
           days-before-issue-stale: 360
diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml
index 73c240e6d81..52f52e66f39 100644
--- a/.github/workflows/typos.yml
+++ b/.github/workflows/typos.yml
@@ -5,13 +5,16 @@ on:
       - main
       - release/**
 
+permissions:
+  contents: read
+
 jobs:
   run:
     name: Spell Check with Typos
     runs-on: "ubuntu-24.04"
     steps:
     - name: Checkout Actions Repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
 
     - name: Check spelling of the entire repository
-      uses: crate-ci/typos@v1.26.0
\ No newline at end of file
+      uses: crate-ci/typos@6802cc60d4e7f78b9d5454f6cf3935c042d5e1e3 # v1.26.0
\ No newline at end of file
diff --git a/.typos.toml b/.typos.toml
index cec3c147941..9285dd52a23 100644
--- a/.typos.toml
+++ b/.typos.toml
@@ -13,6 +13,7 @@ rabit = "rabit"
 flate = "flate"
 Ines = "Ines"
 alph = "alph"
+caf = "caf"
 
 [default.expect]
 nprobs = "nprobes"
@@ -22,5 +23,6 @@ nprob = "nprobe"
 extend-exclude = [
     "notebooks/*.ipynb",
     "*_THIRD_PARTY_LICENSES.*",
+    "rust/lance-tokenizer/src/stop_word_filter/stopwords.rs",
 ]
 # If a line ends with # or // and has spellchecker:disable-line, ignore it
diff --git a/AGENTS.md b/AGENTS.md
index 8543d23521a..df823a8ac5e 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -46,6 +46,12 @@ Key technical traits: async-first (tokio), Arrow-native, versioned writes with m
 
 See [python/AGENTS.md](python/AGENTS.md) and [java/AGENTS.md](java/AGENTS.md).
 
+## Language-Specific Environment Contract
+
+- For language-specific tasks, always follow the environment and command rules in the corresponding subdirectory guide before running build, test, lint, format, or tooling commands.
+- Do not substitute a different environment manager or toolchain just because a command appears missing, unavailable, or slow.
+- If a language-specific command fails outside the documented workflow, treat that as an environment usage mistake first. Fix the environment usage, rerun with the prescribed commands, and only then conclude that a dependency or tool is unavailable.
+
 ### Integration Testing
 
 ```bash
@@ -62,6 +68,7 @@ AWS_DEFAULT_REGION=us-east-1 pytest --run-integration python/tests/test_s3_ddb.p
 - Comments should explain non-obvious "why" reasoning, not restate what the code does.
 - Remove debug prints (`println!`, `dbg!`, `print()`) before merging — use `tracing` or logging frameworks.
 - Extract logic repeated in 2+ places into a shared helper; inline single-use logic at its call site.
+- Think carefully before adding a helper: only introduce one when it materially reduces cognitive load or eliminates substantial duplication, and do not add thin wrappers that only rename or forward existing calls.
 - Keep PRs focused — no drive-by refactors, reformatting, or cosmetic changes.
 - Be mindful of memory use: avoid collecting streams of `RecordBatch` into memory; use `RoaringBitmap` instead of `HashSet<u32>`.
 
diff --git a/Cargo.lock b/Cargo.lock
index 13973540900..ce5370cae2f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -23,12 +23,6 @@ version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
 
-[[package]]
-name = "adler32"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
-
 [[package]]
 name = "aes"
 version = "0.8.4"
@@ -37,7 +31,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
 dependencies = [
  "cfg-if",
  "cipher",
- "cpufeatures",
+ "cpufeatures 0.2.17",
 ]
 
 [[package]]
@@ -116,9 +110,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
 
 [[package]]
 name = "anstream"
-version = "0.6.21"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
+checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
 dependencies = [
  "anstyle",
  "anstyle-parse",
@@ -131,15 +125,15 @@ dependencies = [
 
 [[package]]
 name = "anstyle"
-version = "1.0.13"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.7"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
+checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
 dependencies = [
  "utf8parse",
 ]
@@ -179,15 +173,6 @@ dependencies = [
  "num-traits",
 ]
 
-[[package]]
-name = "arc-swap"
-version = "1.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5"
-dependencies = [
- "rustversion",
-]
-
 [[package]]
 name = "arrayref"
 version = "0.3.9"
@@ -328,7 +313,7 @@ dependencies = [
  "arrow-schema",
  "arrow-select",
  "flatbuffers",
- "lz4_flex 0.12.0",
+ "lz4_flex",
  "zstd",
 ]
 
@@ -382,29 +367,13 @@ dependencies = [
  "half",
 ]
 
-[[package]]
-name = "arrow-scalar"
-version = "57.0.0"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-data",
- "arrow-ord",
- "arrow-row",
- "arrow-schema",
- "half",
- "proptest",
- "rstest",
-]
-
 [[package]]
 name = "arrow-schema"
 version = "57.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "serde_core",
  "serde_json",
 ]
@@ -498,7 +467,7 @@ dependencies = [
  "futures-lite 2.6.1",
  "parking",
  "polling",
- "rustix 1.1.4",
+ "rustix",
  "slab",
  "windows-sys 0.61.2",
 ]
@@ -529,7 +498,7 @@ dependencies = [
  "cfg-if",
  "event-listener 5.4.1",
  "futures-lite 2.6.1",
- "rustix 1.1.4",
+ "rustix",
 ]
 
 [[package]]
@@ -545,9 +514,9 @@ dependencies = [
 
 [[package]]
 name = "async-signal"
-version = "0.2.13"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43c070bbf59cd3570b6b2dd54cd772527c7c3620fce8be898406dd3ed6adc64c"
+checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485"
 dependencies = [
  "async-io",
  "async-lock",
@@ -555,7 +524,7 @@ dependencies = [
  "cfg-if",
  "futures-core",
  "futures-io",
- "rustix 1.1.4",
+ "rustix",
  "signal-hook-registry",
  "slab",
  "windows-sys 0.61.2",
@@ -627,7 +596,7 @@ dependencies = [
  "aws-smithy-types",
  "aws-types",
  "bytes",
- "fastrand 2.3.0",
+ "fastrand 2.4.1",
  "hex",
  "http 1.4.0",
  "ring",
@@ -652,9 +621,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-rs"
-version = "1.16.1"
+version = "1.16.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf"
+checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f"
 dependencies = [
  "aws-lc-sys",
  "zeroize",
@@ -662,9 +631,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-sys"
-version = "0.38.0"
+version = "0.40.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e"
+checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7"
 dependencies = [
  "cc",
  "cmake",
@@ -689,7 +658,7 @@ dependencies = [
  "aws-types",
  "bytes",
  "bytes-utils",
- "fastrand 2.3.0",
+ "fastrand 2.4.1",
  "http 0.2.12",
  "http 1.4.0",
  "http-body 0.4.6",
@@ -717,7 +686,7 @@ dependencies = [
  "aws-smithy-types",
  "aws-types",
  "bytes",
- "fastrand 2.3.0",
+ "fastrand 2.4.1",
  "http 0.2.12",
  "http 1.4.0",
  "regex-lite",
@@ -745,13 +714,13 @@ dependencies = [
  "aws-smithy-xml",
  "aws-types",
  "bytes",
- "fastrand 2.3.0",
+ "fastrand 2.4.1",
  "hex",
  "hmac",
  "http 0.2.12",
  "http 1.4.0",
  "http-body 1.0.1",
- "lru 0.16.3",
+ "lru",
  "percent-encoding",
  "regex-lite",
  "sha2",
@@ -776,7 +745,7 @@ dependencies = [
  "aws-smithy-types",
  "aws-types",
  "bytes",
- "fastrand 2.3.0",
+ "fastrand 2.4.1",
  "http 0.2.12",
  "http 1.4.0",
  "regex-lite",
@@ -800,7 +769,7 @@ dependencies = [
  "aws-smithy-types",
  "aws-types",
  "bytes",
- "fastrand 2.3.0",
+ "fastrand 2.4.1",
  "http 0.2.12",
  "http 1.4.0",
  "regex-lite",
@@ -825,7 +794,7 @@ dependencies = [
  "aws-smithy-types",
  "aws-smithy-xml",
  "aws-types",
- "fastrand 2.3.0",
+ "fastrand 2.4.1",
  "http 0.2.12",
  "http 1.4.0",
  "regex-lite",
@@ -844,20 +813,15 @@ dependencies = [
  "aws-smithy-runtime-api",
  "aws-smithy-types",
  "bytes",
- "crypto-bigint 0.5.5",
  "form_urlencoded",
  "hex",
  "hmac",
  "http 0.2.12",
  "http 1.4.0",
- "p256",
  "percent-encoding",
- "ring",
  "sha2",
- "subtle",
  "time",
  "tracing",
- "zeroize",
 ]
 
 [[package]]
@@ -934,23 +898,17 @@ dependencies = [
  "aws-smithy-async",
  "aws-smithy-runtime-api",
  "aws-smithy-types",
- "h2 0.3.27",
- "h2 0.4.13",
- "http 0.2.12",
+ "h2",
  "http 1.4.0",
- "http-body 0.4.6",
- "hyper 0.14.32",
- "hyper 1.8.1",
- "hyper-rustls 0.24.2",
- "hyper-rustls 0.27.7",
+ "hyper",
+ "hyper-rustls",
  "hyper-util",
  "pin-project-lite",
- "rustls 0.21.12",
- "rustls 0.23.37",
+ "rustls",
  "rustls-native-certs",
  "rustls-pki-types",
  "tokio",
- "tokio-rustls 0.26.4",
+ "tokio-rustls",
  "tower",
  "tracing",
 ]
@@ -996,7 +954,7 @@ dependencies = [
  "aws-smithy-runtime-api",
  "aws-smithy-types",
  "bytes",
- "fastrand 2.3.0",
+ "fastrand 2.4.1",
  "http 0.2.12",
  "http 1.4.0",
  "http-body 0.4.6",
@@ -1087,7 +1045,7 @@ dependencies = [
  "http 1.4.0",
  "http-body 1.0.1",
  "http-body-util",
- "hyper 1.8.1",
+ "hyper",
  "hyper-util",
  "itoa",
  "matchit",
@@ -1242,7 +1200,7 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef"
 dependencies = [
- "fastrand 2.3.0",
+ "fastrand 2.4.1",
  "gloo-timers",
  "tokio",
 ]
@@ -1262,12 +1220,6 @@ dependencies = [
  "windows-link",
 ]
 
-[[package]]
-name = "base16ct"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce"
-
 [[package]]
 name = "base64"
 version = "0.13.1"
@@ -1352,9 +1304,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.11.0"
+version = "2.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
+checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
 
 [[package]]
 name = "bitpacking"
@@ -1388,16 +1340,16 @@ dependencies = [
 
 [[package]]
 name = "blake3"
-version = "1.8.3"
+version = "1.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d"
+checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e"
 dependencies = [
  "arrayref",
  "arrayvec",
  "cc",
  "cfg-if",
  "constant_time_eq",
- "cpufeatures",
+ "cpufeatures 0.3.0",
 ]
 
 [[package]]
@@ -1431,31 +1383,6 @@ dependencies = [
  "piper",
 ]
 
-[[package]]
-name = "bon"
-version = "3.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d13a61f2963b88eef9c1be03df65d42f6996dfeac1054870d950fcf66686f83"
-dependencies = [
- "bon-macros",
- "rustversion",
-]
-
-[[package]]
-name = "bon-macros"
-version = "3.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d314cc62af2b6b0c65780555abb4d02a03dd3b799cd42419044f0c38d99738c0"
-dependencies = [
- "darling 0.23.0",
- "ident_case",
- "prettyplease",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "brotli"
 version = "8.0.2"
@@ -1528,9 +1455,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.56"
+version = "1.2.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
+checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1547,12 +1474,6 @@ dependencies = [
  "smallvec",
 ]
 
-[[package]]
-name = "census"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0"
-
 [[package]]
 name = "cfg-if"
 version = "1.0.4"
@@ -1628,9 +1549,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.60"
+version = "4.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a"
+checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1638,9 +1559,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.60"
+version = "4.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876"
+checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1650,9 +1571,9 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.5.55"
+version = "4.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5"
+checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -1662,24 +1583,24 @@ dependencies = [
 
 [[package]]
 name = "clap_lex"
-version = "1.0.0"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831"
+checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
 
 [[package]]
 name = "cmake"
-version = "0.1.57"
+version = "0.1.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d"
+checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
 dependencies = [
  "cc",
 ]
 
 [[package]]
 name = "colorchoice"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
+checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
 
 [[package]]
 name = "comfy-table"
@@ -1794,15 +1715,6 @@ version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
-[[package]]
-name = "core2"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "cpp_demangle"
 version = "0.5.1"
@@ -1821,6 +1733,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "cpufeatures"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "crc"
 version = "3.3.0"
@@ -1963,28 +1884,6 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
 
-[[package]]
-name = "crypto-bigint"
-version = "0.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef"
-dependencies = [
- "generic-array",
- "rand_core 0.6.4",
- "subtle",
- "zeroize",
-]
-
-[[package]]
-name = "crypto-bigint"
-version = "0.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
-dependencies = [
- "rand_core 0.6.4",
- "subtle",
-]
-
 [[package]]
 name = "crypto-common"
 version = "0.1.7"
@@ -2036,16 +1935,6 @@ dependencies = [
  "darling_macro 0.20.11",
 ]
 
-[[package]]
-name = "darling"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
-dependencies = [
- "darling_core 0.23.0",
- "darling_macro 0.23.0",
-]
-
 [[package]]
 name = "darling_core"
 version = "0.14.4"
@@ -2074,19 +1963,6 @@ dependencies = [
  "syn 2.0.117",
 ]
 
-[[package]]
-name = "darling_core"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
-dependencies = [
- "ident_case",
- "proc-macro2",
- "quote",
- "strsim 0.11.1",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "darling_macro"
 version = "0.14.4"
@@ -2109,23 +1985,6 @@ dependencies = [
  "syn 2.0.117",
 ]
 
-[[package]]
-name = "darling_macro"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
-dependencies = [
- "darling_core 0.23.0",
- "quote",
- "syn 2.0.117",
-]
-
-[[package]]
-name = "dary_heap"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04"
-
 [[package]]
 name = "dashmap"
 version = "6.1.0"
@@ -2142,9 +2001,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503f1f4a9060ae6e650d3dff5dc7a21266fea1302d890768d45b4b28586e830f"
+checksum = "7541353e77dc7262b71ca27be07d8393661737e3a73b5d1b1c6f7d814c64fa2a"
 dependencies = [
  "arrow",
  "arrow-schema",
@@ -2182,7 +2041,7 @@ dependencies = [
  "object_store",
  "parking_lot",
  "parquet",
- "rand 0.9.2",
+ "rand 0.9.4",
  "regex",
  "sqlparser",
  "tempfile",
@@ -2193,9 +2052,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14417a3ee4ae3d092b56cd6c1d32e8ff3e2c9ec130ecb2276ec91c89fd599399"
+checksum = "9997731f90fa5398ef831ad0e69600f92c861b79c0d38bd1a29b6f0e3a0ce4c8"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2218,9 +2077,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog-listing"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d0eba824adb45a4b3ac6f0251d40df3f6a9382371cad136f4f14ac9ebc6bc10"
+checksum = "2b30a3dd50dec860c9559275c8d97d9de602e611237a6ecfbda0b3b63b872352"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2241,9 +2100,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0039deefbd00c56adf5168b7ca58568fb058e4ba4c5a03b09f8be371b4e434b6"
+checksum = "d551054acec0398ca604512310b77ce05c46f66e54b54d48200a686e385cca4e"
 dependencies = [
  "ahash",
  "arrow",
@@ -2264,9 +2123,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common-runtime"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ec7e3e60b813048331f8fb9673583173e5d2dd8fef862834ee871fc98b57ca7"
+checksum = "567d40e285f5b79f8737b576605721cd6c1133b5d2b00bdbd5d9838d90d0812f"
 dependencies = [
  "futures",
  "log",
@@ -2275,9 +2134,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "802068957f620302ecf05f84ff4019601aeafd36f5f3f1334984af2e34265129"
+checksum = "27d2668f51b3b30befae2207472569e37807fdedd1d14da58acc6f8ca6257eae"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2297,16 +2156,16 @@ dependencies = [
  "itertools 0.14.0",
  "log",
  "object_store",
- "rand 0.9.2",
+ "rand 0.9.4",
  "tokio",
  "url",
 ]
 
 [[package]]
 name = "datafusion-datasource-arrow"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90fc387d5067c62d494a6647d29c5ad4fcdd5a6e50ab4ea1d2568caa2d66f2cc"
+checksum = "e02e1b3e3a8ec55f1f62de4252b0407c8567363d056078769a197e24fc834a0f"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -2328,9 +2187,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-csv"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efd5e20579bb6c8bd4e6c620253972fb723822030c280dd6aa047f660d09eeba"
+checksum = "b559d7bf87d4f900f847baba8509634f838d9718695389e903604cdcccdb01f3"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2351,9 +2210,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-json"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0788b0d48fcef31880a02013ea3cc18e5a4e0eacc3b0abdd2cd0597b99dc96e"
+checksum = "250e2d7591ba8b638f063854650faa40bca4e8bd4059b2ece8836f6388d02db4"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2373,9 +2232,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-parquet"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66639b70f1f363f5f0950733170100e588f1acfacac90c1894e231194aa35957"
+checksum = "0b043149f2c3557ca94abc58de40f68a8d412ff53365c06126ed234f8596399d"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2403,15 +2262,15 @@ dependencies = [
 
 [[package]]
 name = "datafusion-doc"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e44b41f3e8267c6cf3eec982d63f34db9f1dd5f30abfd2e1f124f0871708952e"
+checksum = "b9496cb0db222dbb9a3735760ceca7fc56f35e1d5502c38d0caa77a81e9c1f6a"
 
 [[package]]
 name = "datafusion-execution"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e456f60e5d38db45335e84617006d90af14a8c8c5b8e959add708b2daaa0e2c"
+checksum = "dc45d23c516ed8d3637751e44e09e21b45b3f58b473c802dddd1f1ad4fe435ff"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2423,16 +2282,16 @@ dependencies = [
  "log",
  "object_store",
  "parking_lot",
- "rand 0.9.2",
+ "rand 0.9.4",
  "tempfile",
  "url",
 ]
 
 [[package]]
 name = "datafusion-expr"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6507c719804265a58043134580c1c20767e7c23ba450724393f03ec982769ad9"
+checksum = "63dd30526d2db4fda6440806a41e4676334a94bc0596cc9cc2a0efed20ef2c44"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2452,9 +2311,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-expr-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a413caa9c5885072b539337aed68488f0291653e8edd7d676c92df2480f6cab0"
+checksum = "1b486b5f6255d40976b88bb83813b0d035a8333e0ec39864824e78068cf42fa6"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2465,9 +2324,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "189256495dc9cbbb8e20dbcf161f60422e628d201a78df8207e44bd4baefadb6"
+checksum = "07356c94118d881130dd0ffbff127540407d969c8978736e324edcd6c41cd48f"
 dependencies = [
  "arrow",
  "arrow-buffer",
@@ -2487,7 +2346,7 @@ dependencies = [
  "log",
  "md-5",
  "num-traits",
- "rand 0.9.2",
+ "rand 0.9.4",
  "regex",
  "sha2",
  "unicode-segmentation",
@@ -2496,9 +2355,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-aggregate"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12e73dfee4cd67c4a507ffff4c5a711d39983adf544adbc09c09bf06f789f413"
+checksum = "b644f9cf696df9233ce6958b9807666d78563b56f923267474dd6c07795f1f8f"
 dependencies = [
  "ahash",
  "arrow",
@@ -2517,9 +2376,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-aggregate-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87727bd9e65f4f9ac6d608c9810b7da9eaa3b18b26a4a4b76520592d49020acf"
+checksum = "c1de2deaaabe8923ce9ea9f29c47bbb4ee14f67ea2fe1ab5398d9bbebcf86e56"
 dependencies = [
  "ahash",
  "arrow",
@@ -2530,9 +2389,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-nested"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e5ef761359224b7c2b5a1bfad6296ac63225f8583d08ad18af9ba1a89ac3887"
+checksum = "552f8d92e4331ee91d23c02d12bb6acf32cbfd5215117e01c0fb63cd4b15af1a"
 dependencies = [
  "arrow",
  "arrow-ord",
@@ -2553,9 +2412,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-table"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b17dac25dfda2d2a90ff0ad1c054a11fb1523766226bec6e9bd8c410daee2ae"
+checksum = "970fd0cdd3df8802b9a9975ff600998289ba9d46682a4f7285cba4820c9ada78"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2569,9 +2428,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-window"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c594a29ddb22cbdbce500e4d99b5b2392c5cecb4c1086298b41d1ffec14dbb77"
+checksum = "40b4c21a7c8a986a1866c0a87ab756d0bbf7b5f41f306009fa2d9af79c52ed31"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2587,9 +2446,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-window-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aa1b15ed81c7543f62264a30dd49dec4b1b0b698053b968f53be32dfba4f729"
+checksum = "b1210ad73b8b3211aeaf4a42bef9bd7a2b7fce3ec119a478831f18c6ff7f7b93"
 dependencies = [
  "datafusion-common",
  "datafusion-physical-expr-common",
@@ -2597,9 +2456,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-macros"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c00c31c4795597aa25b74cab5174ac07a53051f27ce1e011ecaffa9eaeecef81"
+checksum = "aaa566a963013a38681ad82a727a654bc7feb19632426aea8c3412d415d200c5"
 dependencies = [
  "datafusion-doc",
  "quote",
@@ -2608,9 +2467,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-optimizer"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80ccf60767c09302b2e0fc3afebb3761a6d508d07316fab8c5e93312728a21bb"
+checksum = "ff9aa82b240252a88dee118372f9b9757c545ab9e53c0736bebab2e7da0ef1f2"
 dependencies = [
  "arrow",
  "chrono",
@@ -2627,9 +2486,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c64b7f277556944e4edd3558da01d9e9ff9f5416f1c0aa7fee088e57bd141a7e"
+checksum = "7d48022b8af9988c1d852644f9e8b5584c490659769a550c5e8d39457a1da0a5"
 dependencies = [
  "ahash",
  "arrow",
@@ -2650,9 +2509,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-adapter"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7abaee372ea2d19c016ee9ef8629c4415257d291cdd152bc7f0b75f28af1b63"
+checksum = "ae7a8abc0b4fe624000972a9b145b30b7f1b680bffaa950ea53f78d9b21c27c3"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2665,9 +2524,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42237efe621f92adc22d111b531fdbc2cc38ca9b5e02327535628fb103ae2157"
+checksum = "147253ca3e6b9d59c162de64c02800973018660e13340dd1886dd038d17ac429"
 dependencies = [
  "ahash",
  "arrow",
@@ -2682,9 +2541,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-optimizer"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd093498bd1319c6e5c76e9dfa905e78486f01b34579ce97f2e3a49f84c37fac"
+checksum = "689156bb2282107b6239db8d7ef44b4dab10a9b33d3491a0c74acac5e4fedd72"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2700,9 +2559,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-plan"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cbe61b12daf81a9f20ba03bd3541165d51f86e004ef37426b11881330eed261"
+checksum = "68253dc0ee5330aa558b2549c9b0da5af9fc17d753ae73022939014ad616fc28"
 dependencies = [
  "ahash",
  "arrow",
@@ -2731,9 +2590,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-pruning"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0124331116db7f79df92ebfd2c3b11a8f90240f253555c9bb084f10b6fecf1dd"
+checksum = "0fcad240a54d0b1d3e8f668398900260a53122d522b2102ab57218590decacd6"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2748,9 +2607,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-session"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1673e3c58ba618a6ea0568672f00664087b8982c581e9afd5aa6c3c79c9b431f"
+checksum = "f58e83a68bb67007a8fcbf005c44cefe441270c7ee7f6dee10c0e0109b556f6d"
 dependencies = [
  "async-trait",
  "datafusion-common",
@@ -2762,9 +2621,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-sql"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5272d256dab5347bb39d2040589f45d8c6b715b27edcb5fffe88cc8b9c3909cb"
+checksum = "be53e9eb55db0fbb8980bb6d87f2435b0524acf4c718ed54a57cabbb299b2ab3"
 dependencies = [
  "arrow",
  "bigdecimal",
@@ -2779,9 +2638,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-substrait"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "199790fd96e852997b30da4ff11109378c944841757d93875ea85fc69587ec91"
+checksum = "7dadf18107366b3470e47bdde8eeb9294b36444e451830d8e5fc8dc7b4ad9c58"
 dependencies = [
  "async-recursion",
  "async-trait",
@@ -2844,16 +2703,6 @@ dependencies = [
  "syn 1.0.109",
 ]
 
-[[package]]
-name = "der"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de"
-dependencies = [
- "const-oid",
- "zeroize",
-]
-
 [[package]]
 name = "der"
 version = "0.7.10"
@@ -3023,12 +2872,6 @@ version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"
 
-[[package]]
-name = "downcast-rs"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc"
-
 [[package]]
 name = "dunce"
 version = "1.0.5"
@@ -3051,44 +2894,12 @@ dependencies = [
  "num-traits",
 ]
 
-[[package]]
-name = "ecdsa"
-version = "0.14.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c"
-dependencies = [
- "der 0.6.1",
- "elliptic-curve",
- "rfc6979",
- "signature 1.6.4",
-]
-
 [[package]]
 name = "either"
 version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
-[[package]]
-name = "elliptic-curve"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3"
-dependencies = [
- "base16ct",
- "crypto-bigint 0.4.9",
- "der 0.6.1",
- "digest",
- "ff",
- "generic-array",
- "group",
- "pkcs8 0.9.0",
- "rand_core 0.6.4",
- "sec1",
- "subtle",
- "zeroize",
-]
-
 [[package]]
 name = "encode_unicode"
 version = "1.0.0"
@@ -3179,9 +2990,9 @@ dependencies = [
 
 [[package]]
 name = "env_filter"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f"
+checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef"
 dependencies = [
  "log",
  "regex",
@@ -3189,9 +3000,9 @@ dependencies = [
 
 [[package]]
 name = "env_logger"
-version = "0.11.9"
+version = "0.11.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d"
+checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a"
 dependencies = [
  "anstream",
  "anstyle",
@@ -3284,12 +3095,6 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
 
-[[package]]
-name = "fastdivide"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471"
-
 [[package]]
 name = "fastrand"
 version = "1.9.0"
@@ -3301,19 +3106,9 @@ dependencies = [
 
 [[package]]
 name = "fastrand"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
-
-[[package]]
-name = "ff"
-version = "0.12.1"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160"
-dependencies = [
- "rand_core 0.6.4",
- "subtle",
-]
+checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
 
 [[package]]
 name = "filetime"
@@ -3356,7 +3151,7 @@ version = "25.12.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "rustc_version",
 ]
 
@@ -3421,18 +3216,11 @@ dependencies = [
 
 [[package]]
 name = "fragile"
-version = "2.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619"
-
-[[package]]
-name = "fs4"
-version = "0.8.4"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8"
+checksum = "8878864ba14bb86e818a412bfd6f18f9eabd4ec0f008a28e8f7eb61db532fcf9"
 dependencies = [
- "rustix 0.38.44",
- "windows-sys 0.52.0",
+ "futures-core",
 ]
 
 [[package]]
@@ -3443,11 +3231,11 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-array",
  "lance-datagen",
- "rand 0.9.2",
+ "rand 0.9.4",
  "rand_xoshiro",
  "test-log",
  "tokio",
@@ -3537,7 +3325,7 @@ version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad"
 dependencies = [
- "fastrand 2.3.0",
+ "fastrand 2.4.1",
  "futures-core",
  "futures-io",
  "parking",
@@ -3644,9 +3432,9 @@ dependencies = [
 
 [[package]]
 name = "geo-types"
-version = "0.7.18"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24f8647af4005fa11da47cd56252c6ef030be8fa97bdbf355e7dfb6348f0a82c"
+checksum = "94776032c45f950d30a13af6113c2ad5625316c9abfbccee4dd5a6695f8fe0f5"
 dependencies = [
  "approx",
  "num-traits",
@@ -3823,7 +3611,7 @@ dependencies = [
  "derive_builder 0.20.2",
  "http 1.4.0",
  "reqwest",
- "rustls 0.23.37",
+ "rustls",
  "rustls-pemfile",
  "serde",
  "serde_json",
@@ -3832,36 +3620,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "group"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7"
-dependencies = [
- "ff",
- "rand_core 0.6.4",
- "subtle",
-]
-
-[[package]]
-name = "h2"
-version = "0.3.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d"
-dependencies = [
- "bytes",
- "fnv",
- "futures-core",
- "futures-sink",
- "futures-util",
- "http 0.2.12",
- "indexmap",
- "slab",
- "tokio",
- "tokio-util",
- "tracing",
-]
-
 [[package]]
 name = "h2"
 version = "0.4.13"
@@ -3914,8 +3672,6 @@ version = "0.15.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
 dependencies = [
- "allocator-api2",
- "equivalent",
  "foldhash 0.1.5",
 ]
 
@@ -3930,6 +3686,12 @@ dependencies = [
  "foldhash 0.2.0",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"
+
 [[package]]
 name = "heapless"
 version = "0.8.0"
@@ -3972,7 +3734,7 @@ dependencies = [
  "log",
  "native-tls",
  "num_cpus",
- "rand 0.9.2",
+ "rand 0.9.4",
  "reqwest",
  "serde",
  "serde_json",
@@ -4000,12 +3762,6 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
-[[package]]
-name = "htmlescape"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
-
 [[package]]
 name = "http"
 version = "0.2.12"
@@ -4101,46 +3857,21 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
 
 [[package]]
 name = "hyper"
-version = "0.14.32"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7"
-dependencies = [
- "bytes",
- "futures-channel",
- "futures-core",
- "futures-util",
- "h2 0.3.27",
- "http 0.2.12",
- "http-body 0.4.6",
- "httparse",
- "httpdate",
- "itoa",
- "pin-project-lite",
- "socket2 0.5.10",
- "tokio",
- "tower-service",
- "tracing",
- "want",
-]
-
-[[package]]
-name = "hyper"
-version = "1.8.1"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"
+checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"
 dependencies = [
  "atomic-waker",
  "bytes",
  "futures-channel",
  "futures-core",
- "h2 0.4.13",
+ "h2",
  "http 1.4.0",
  "http-body 1.0.1",
  "httparse",
  "httpdate",
  "itoa",
  "pin-project-lite",
- "pin-utils",
  "smallvec",
  "tokio",
  "want",
@@ -4148,33 +3879,17 @@ dependencies = [
 
 [[package]]
 name = "hyper-rustls"
-version = "0.24.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
-dependencies = [
- "futures-util",
- "http 0.2.12",
- "hyper 0.14.32",
- "log",
- "rustls 0.21.12",
- "tokio",
- "tokio-rustls 0.24.1",
-]
-
-[[package]]
-name = "hyper-rustls"
-version = "0.27.7"
+version = "0.27.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58"
+checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
 dependencies = [
  "http 1.4.0",
- "hyper 1.8.1",
+ "hyper",
  "hyper-util",
- "rustls 0.23.37",
+ "rustls",
  "rustls-native-certs",
- "rustls-pki-types",
  "tokio",
- "tokio-rustls 0.26.4",
+ "tokio-rustls",
  "tower-service",
  "webpki-roots 1.0.6",
 ]
@@ -4187,7 +3902,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
 dependencies = [
  "bytes",
  "http-body-util",
- "hyper 1.8.1",
+ "hyper",
  "hyper-util",
  "native-tls",
  "tokio",
@@ -4207,12 +3922,12 @@ dependencies = [
  "futures-util",
  "http 1.4.0",
  "http-body 1.0.1",
- "hyper 1.8.1",
+ "hyper",
  "ipnet",
  "libc",
  "percent-encoding",
  "pin-project-lite",
- "socket2 0.6.2",
+ "socket2",
  "system-configuration",
  "tokio",
  "tower-service",
@@ -4298,12 +4013,13 @@ dependencies = [
 
 [[package]]
 name = "icu_collections"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43"
+checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"
 dependencies = [
  "displaydoc",
  "potential_utf",
+ "utf8_iter",
  "yoke",
  "zerofrom",
  "zerovec",
@@ -4311,9 +4027,9 @@ dependencies = [
 
 [[package]]
 name = "icu_locale_core"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6"
+checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
 dependencies = [
  "displaydoc",
  "litemap",
@@ -4324,9 +4040,9 @@ dependencies = [
 
 [[package]]
 name = "icu_normalizer"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599"
+checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
 dependencies = [
  "icu_collections",
  "icu_normalizer_data",
@@ -4338,15 +4054,15 @@ dependencies = [
 
 [[package]]
 name = "icu_normalizer_data"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"
+checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
 
 [[package]]
 name = "icu_properties"
-version = "2.1.2"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec"
+checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
 dependencies = [
  "icu_collections",
  "icu_locale_core",
@@ -4358,15 +4074,15 @@ dependencies = [
 
 [[package]]
 name = "icu_properties_data"
-version = "2.1.2"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af"
+checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
 
 [[package]]
 name = "icu_provider"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614"
+checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
 dependencies = [
  "displaydoc",
  "icu_locale_core",
@@ -4410,51 +4126,14 @@ dependencies = [
  "icu_properties",
 ]
 
-[[package]]
-name = "include-flate"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e01b7cb6ca682a621e7cda1c358c9724b53a7b4409be9be1dd443b7f3a26f998"
-dependencies = [
- "include-flate-codegen",
- "include-flate-compress",
- "libflate",
- "zstd",
-]
-
-[[package]]
-name = "include-flate-codegen"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f49bf5274aebe468d6e6eba14a977eaf1efa481dc173f361020de70c1c48050"
-dependencies = [
- "include-flate-compress",
- "libflate",
- "proc-macro-error",
- "proc-macro2",
- "quote",
- "syn 2.0.117",
- "zstd",
-]
-
-[[package]]
-name = "include-flate-compress"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eae6a40e716bcd5931f5dbb79cd921512a4f647e2e9413fded3171fca3824dbc"
-dependencies = [
- "libflate",
- "zstd",
-]
-
 [[package]]
 name = "indexmap"
-version = "2.13.0"
+version = "2.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
+checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
 dependencies = [
  "equivalent",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.0",
  "serde",
  "serde_core",
 ]
@@ -4521,6 +4200,17 @@ version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
+[[package]]
+name = "io-uring"
+version = "0.7.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdd7bddefd0a8833b88a4b68f90dae22c7450d11b354198baee3874fd811b344"
+dependencies = [
+ "bitflags 2.11.1",
+ "cfg-if",
+ "libc",
+]
+
 [[package]]
 name = "ipnet"
 version = "2.12.0"
@@ -4529,9 +4219,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
 
 [[package]]
 name = "iri-string"
-version = "0.7.10"
+version = "0.7.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a"
+checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20"
 dependencies = [
  "memchr",
  "serde",
@@ -4601,27 +4291,26 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.17"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "jieba-macros"
-version = "0.8.1"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "348294e44ee7e3c42685da656490f8febc7359632544019621588902216da95c"
+checksum = "a29cfc5dcd898604c6f80363411fa6b6b08e27d1d253d6225b9cb6702ea02fc0"
 dependencies = [
  "phf_codegen",
 ]
 
 [[package]]
 name = "jieba-rs"
-version = "0.8.1"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "766bd7012aa5ba49411ebdf4e93bddd59b182d2918e085d58dec5bb9b54b7105"
+checksum = "3245d6e9d1d5facbd6a23848d6b67e3439738ccbb4fa5a3d65da315ba1a910a2"
 dependencies = [
  "cedarwood",
- "include-flate",
  "jieba-macros",
  "phf 0.13.1",
  "regex",
@@ -4681,19 +4370,21 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.91"
+version = "0.3.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c"
+checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca"
 dependencies = [
+ "cfg-if",
+ "futures-util",
  "once_cell",
  "wasm-bindgen",
 ]
 
 [[package]]
 name = "jsonb"
-version = "0.5.5"
+version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a901f06163d352fbe41c3c2ff5e08b75330a003cc941e988fb501022f5421e6"
+checksum = "eb98fb29636087c40ad0d1274d9a30c0c1e83e03ae93f6e7e89247b37fcc6953"
 dependencies = [
  "byteorder",
  "ethnum",
@@ -4702,11 +4393,11 @@ dependencies = [
  "jiff",
  "nom 8.0.0",
  "num-traits",
- "ordered-float 5.1.0",
- "rand 0.9.2",
- "ryu",
+ "ordered-float 5.3.0",
+ "rand 0.9.4",
  "serde",
  "serde_json",
+ "zmij",
 ]
 
 [[package]]
@@ -4735,7 +4426,7 @@ dependencies = [
 
 [[package]]
 name = "lance"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "all_asserts",
  "approx",
@@ -4743,6 +4434,7 @@ dependencies = [
  "arrow-arith",
  "arrow-array",
  "arrow-buffer",
+ "arrow-cast",
  "arrow-ipc",
  "arrow-ord",
  "arrow-row",
@@ -4793,6 +4485,7 @@ dependencies = [
  "lance-table",
  "lance-test-macros",
  "lance-testing",
+ "lance-tokenizer",
  "lapack",
  "libc",
  "log",
@@ -4806,15 +4499,17 @@ dependencies = [
  "pprof",
  "pretty_assertions",
  "prost",
+ "prost-build",
  "prost-types",
- "rand 0.9.2",
+ "protobuf-src",
+ "rand 0.9.4",
+ "rand_distr",
  "roaring",
  "rstest",
  "semver",
  "serde",
  "serde_json",
  "snafu",
- "tantivy",
  "tempfile",
  "test-log",
  "tokio",
@@ -4830,12 +4525,13 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-cast",
  "arrow-data",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
@@ -4845,12 +4541,41 @@ dependencies = [
  "half",
  "jsonb",
  "num-traits",
- "rand 0.9.2",
+ "rand 0.9.4",
+]
+
+[[package]]
+name = "lance-arrow-scalar"
+version = "57.0.0"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
+ "arrow-data",
+ "arrow-ord",
+ "arrow-row",
+ "arrow-schema",
+ "half",
+ "proptest",
+ "rstest",
+]
+
+[[package]]
+name = "lance-arrow-stats"
+version = "57.0.0"
+dependencies = [
+ "arrow-array",
+ "arrow-schema",
+ "arrow-select",
+ "half",
+ "lance-arrow-scalar",
+ "proptest",
+ "rstest",
 ]
 
 [[package]]
 name = "lance-bitpacking"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrayref",
  "paste",
@@ -4859,7 +4584,7 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4884,7 +4609,7 @@ dependencies = [
  "pin-project",
  "proptest",
  "prost",
- "rand 0.9.2",
+ "rand 0.9.4",
  "roaring",
  "rstest",
  "serde_json",
@@ -4899,11 +4624,12 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-array",
  "arrow-buffer",
+ "arrow-cast",
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
@@ -4933,7 +4659,7 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4945,15 +4671,15 @@ dependencies = [
  "half",
  "hex",
  "pprof",
- "rand 0.9.2",
- "rand_distr 0.5.1",
+ "rand 0.9.4",
+ "rand_distr",
  "rand_xoshiro",
  "random_word",
 ]
 
 [[package]]
 name = "lance-encoding"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4986,9 +4712,10 @@ dependencies = [
  "prost-build",
  "prost-types",
  "protobuf-src",
- "rand 0.9.2",
+ "rand 0.9.4",
  "rand_xoshiro",
  "rstest",
+ "serial_test",
  "snafu",
  "strum 0.26.3",
  "test-log",
@@ -5000,7 +4727,7 @@ dependencies = [
 
 [[package]]
 name = "lance-examples"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "all_asserts",
  "arrow",
@@ -5018,7 +4745,7 @@ dependencies = [
  "lance-linalg",
  "object_store",
  "parquet",
- "rand 0.9.2",
+ "rand 0.9.4",
  "tempfile",
  "tokenizers",
  "tokio",
@@ -5026,7 +4753,7 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5059,7 +4786,7 @@ dependencies = [
  "prost-build",
  "prost-types",
  "protobuf-src",
- "rand 0.9.2",
+ "rand 0.9.4",
  "rstest",
  "snafu",
  "test-log",
@@ -5069,7 +4796,7 @@ dependencies = [
 
 [[package]]
 name = "lance-geo"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "datafusion",
  "geo-traits",
@@ -5083,7 +4810,7 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "approx",
  "arrow",
@@ -5130,9 +4857,9 @@ dependencies = [
  "lance-linalg",
  "lance-table",
  "lance-testing",
+ "lance-tokenizer",
+ "libc",
  "libm",
- "lindera",
- "lindera-tantivy",
  "log",
  "ndarray",
  "num-traits",
@@ -5142,8 +4869,8 @@ dependencies = [
  "prost-build",
  "prost-types",
  "protobuf-src",
- "rand 0.9.2",
- "rand_distr 0.5.1",
+ "rand 0.9.4",
+ "rand_distr",
  "rangemap",
  "rayon",
  "roaring",
@@ -5152,7 +4879,6 @@ dependencies = [
  "serde_json",
  "smallvec",
  "snafu",
- "tantivy",
  "tempfile",
  "test-log",
  "tokio",
@@ -5163,7 +4889,7 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -5184,12 +4910,15 @@ dependencies = [
  "deepsize",
  "futures",
  "http 1.4.0",
+ "io-uring",
  "lance-arrow",
  "lance-core",
  "lance-namespace",
+ "libc",
  "log",
  "mock_instant",
  "mockall",
+ "moka",
  "object_store",
  "object_store_opendal",
  "opendal",
@@ -5197,7 +4926,7 @@ dependencies = [
  "pin-project",
  "pprof",
  "prost",
- "rand 0.9.2",
+ "rand 0.9.4",
  "rstest",
  "serde",
  "snafu",
@@ -5211,7 +4940,7 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "approx",
  "arrow-array",
@@ -5227,18 +4956,19 @@ dependencies = [
  "num-traits",
  "pprof",
  "proptest",
- "rand 0.9.2",
+ "rand 0.9.4",
 ]
 
 [[package]]
 name = "lance-namespace"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "async-trait",
  "bytes",
  "lance-core",
  "lance-namespace-reqwest-client",
+ "serde",
  "snafu",
  "tempfile",
  "tokio",
@@ -5246,7 +4976,7 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-datafusion"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5264,7 +4994,7 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -5287,11 +5017,12 @@ dependencies = [
  "lance-core",
  "lance-index",
  "lance-io",
+ "lance-linalg",
  "lance-namespace",
  "lance-table",
  "log",
  "object_store",
- "rand 0.9.2",
+ "rand 0.9.4",
  "reqwest",
  "rstest",
  "serde",
@@ -5309,9 +5040,9 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-reqwest-client"
-version = "0.5.2"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ad4c947349acd6e37e984eba0254588bd894e6128434338b9e6904e56fb4633"
+checksum = "ee2e48de899e2931afb67fcddd0a08e439bf5d8b6ea2a2ed9cb8f4df669bd5cc"
 dependencies = [
  "reqwest",
  "serde",
@@ -5322,7 +5053,7 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5352,7 +5083,7 @@ dependencies = [
  "prost-build",
  "prost-types",
  "protobuf-src",
- "rand 0.9.2",
+ "rand 0.9.4",
  "rangemap",
  "roaring",
  "rstest",
@@ -5368,7 +5099,7 @@ dependencies = [
 
 [[package]]
 name = "lance-test-macros"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5377,18 +5108,29 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-array",
  "arrow-schema",
  "lance-arrow",
  "num-traits",
- "rand 0.9.2",
+ "rand 0.9.4",
+]
+
+[[package]]
+name = "lance-tokenizer"
+version = "6.0.0-beta.1"
+dependencies = [
+ "jieba-rs",
+ "lindera",
+ "rust-stemmers",
+ "serde",
+ "unicode-normalization",
 ]
 
 [[package]]
 name = "lance-tools"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "clap",
  "lance-core",
@@ -5435,12 +5177,6 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
 
-[[package]]
-name = "levenshtein_automata"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
-
 [[package]]
 name = "lexical-core"
 version = "1.0.6"
@@ -5500,33 +5236,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.182"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
-
-[[package]]
-name = "libflate"
-version = "2.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3248b8d211bd23a104a42d81b4fa8bb8ac4a3b75e7a43d85d2c9ccb6179cd74"
-dependencies = [
- "adler32",
- "core2",
- "crc32fast",
- "dary_heap",
- "libflate_lz77",
-]
-
-[[package]]
-name = "libflate_lz77"
-version = "2.2.0"
+version = "0.2.185"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a599cb10a9cd92b1300debcef28da8f70b935ec937f44fcd1b70a7c986a11c5c"
-dependencies = [
- "core2",
- "hashbrown 0.16.1",
- "rle-decode-fast",
-]
+checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"
 
 [[package]]
 name = "libm"
@@ -5536,14 +5248,14 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 
 [[package]]
 name = "libredox"
-version = "0.1.14"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a"
+checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "libc",
  "plain",
- "redox_syscall 0.7.3",
+ "redox_syscall 0.7.4",
 ]
 
 [[package]]
@@ -5609,7 +5321,7 @@ dependencies = [
  "md5",
  "memmap2",
  "once_cell",
- "rand 0.9.2",
+ "rand 0.9.4",
  "reqwest",
  "serde",
  "tar",
@@ -5657,17 +5369,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "lindera-tantivy"
-version = "0.44.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d0d17afa51b4f4cd2f3e50d4c0845800f3a774ed7f706612fbd91eba0c84b41"
-dependencies = [
- "lindera",
- "tantivy",
- "tantivy-tokenizer-api",
-]
-
 [[package]]
 name = "lindera-unidic"
 version = "0.44.1"
@@ -5681,12 +5382,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "linux-raw-sys"
-version = "0.4.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
-
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -5695,9 +5390,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
 
 [[package]]
 name = "litemap"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
+checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
 
 [[package]]
 name = "lock_api"
@@ -5729,18 +5424,9 @@ dependencies = [
 
 [[package]]
 name = "lru"
-version = "0.12.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
-dependencies = [
- "hashbrown 0.15.5",
-]
-
-[[package]]
-name = "lru"
-version = "0.16.3"
+version = "0.16.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593"
+checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39"
 dependencies = [
  "hashbrown 0.16.1",
 ]
@@ -5772,15 +5458,9 @@ dependencies = [
 
 [[package]]
 name = "lz4_flex"
-version = "0.11.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a"
-
-[[package]]
-name = "lz4_flex"
-version = "0.12.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e"
+checksum = "98c23545df7ecf1b16c303910a69b079e8e251d60f7dd2cc9b4177f2afaf1746"
 dependencies = [
  "twox-hash",
 ]
@@ -5856,15 +5536,6 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0"
 
-[[package]]
-name = "measure_time"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e"
-dependencies = [
- "log",
-]
-
 [[package]]
 name = "memchr"
 version = "2.8.0"
@@ -5914,9 +5585,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "1.1.1"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc"
+checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
 dependencies = [
  "libc",
  "wasi 0.11.1+wasi-snapshot-preview1",
@@ -5957,9 +5628,9 @@ dependencies = [
 
 [[package]]
 name = "moka"
-version = "0.12.14"
+version = "0.12.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85f8024e1c8e71c778968af91d43700ce1d11b219d127d79fb2934153b82b42b"
+checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046"
 dependencies = [
  "async-lock",
  "crossbeam-channel",
@@ -6003,12 +5674,6 @@ version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
 
-[[package]]
-name = "murmurhash32"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
-
 [[package]]
 name = "native-tls"
 version = "0.2.18"
@@ -6117,9 +5782,9 @@ dependencies = [
 
 [[package]]
 name = "num-conv"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
+checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
 
 [[package]]
 name = "num-format"
@@ -6173,9 +5838,9 @@ dependencies = [
 
 [[package]]
 name = "num_enum"
-version = "0.7.5"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c"
+checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26"
 dependencies = [
  "num_enum_derive",
  "rustversion",
@@ -6183,9 +5848,9 @@ dependencies = [
 
 [[package]]
 name = "num_enum_derive"
-version = "0.7.5"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7"
+checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8"
 dependencies = [
  "proc-macro-crate",
  "proc-macro2",
@@ -6252,13 +5917,13 @@ dependencies = [
  "http-body-util",
  "httparse",
  "humantime",
- "hyper 1.8.1",
+ "hyper",
  "itertools 0.14.0",
  "md-5",
  "parking_lot",
  "percent-encoding",
  "quick-xml 0.38.4",
- "rand 0.9.2",
+ "rand 0.9.4",
  "reqwest",
  "ring",
  "rustls-pemfile",
@@ -6292,9 +5957,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.21.3"
+version = "1.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "once_cell_polyfill"
@@ -6302,19 +5967,13 @@ version = "1.70.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
 
-[[package]]
-name = "oneshot"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
-
 [[package]]
 name = "onig"
 version = "6.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "libc",
  "once_cell",
  "onig_sys",
@@ -6368,11 +6027,11 @@ dependencies = [
 
 [[package]]
 name = "openssl"
-version = "0.10.75"
+version = "0.10.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328"
+checksum = "bfe4646e360ec77dff7dde40ed3d6c5fee52d156ef4a62f53973d38294dad87f"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "cfg-if",
  "foreign-types",
  "libc",
@@ -6400,9 +6059,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.111"
+version = "0.9.113"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321"
+checksum = "ad2f2c0eba47118757e4c6d2bff2838f3e0523380021356e7875e858372ce644"
 dependencies = [
  "cc",
  "libc",
@@ -6427,9 +6086,9 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "5.1.0"
+version = "5.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d"
+checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e"
 dependencies = [
  "num-traits",
 ]
@@ -6450,26 +6109,6 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
 
-[[package]]
-name = "ownedbytes"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e"
-dependencies = [
- "stable_deref_trait",
-]
-
-[[package]]
-name = "p256"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594"
-dependencies = [
- "ecdsa",
- "elliptic-curve",
- "sha2",
-]
-
 [[package]]
 name = "parking"
 version = "2.2.1"
@@ -6521,7 +6160,7 @@ dependencies = [
  "futures",
  "half",
  "hashbrown 0.16.1",
- "lz4_flex 0.12.0",
+ "lz4_flex",
  "num-bigint",
  "num-integer",
  "num-traits",
@@ -6679,7 +6318,7 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
 dependencies = [
- "fastrand 2.3.0",
+ "fastrand 2.4.1",
  "phf_shared 0.13.1",
 ]
 
@@ -6740,7 +6379,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1"
 dependencies = [
  "atomic-waker",
- "fastrand 2.3.0",
+ "fastrand 2.4.1",
  "futures-io",
 ]
 
@@ -6750,9 +6389,9 @@ version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
 dependencies = [
- "der 0.7.10",
- "pkcs8 0.10.2",
- "spki 0.7.3",
+ "der",
+ "pkcs8",
+ "spki",
 ]
 
 [[package]]
@@ -6763,21 +6402,11 @@ checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6"
 dependencies = [
  "aes",
  "cbc",
- "der 0.7.10",
+ "der",
  "pbkdf2",
  "scrypt",
  "sha2",
- "spki 0.7.3",
-]
-
-[[package]]
-name = "pkcs8"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba"
-dependencies = [
- "der 0.6.1",
- "spki 0.6.0",
+ "spki",
 ]
 
 [[package]]
@@ -6786,17 +6415,17 @@ version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
 dependencies = [
- "der 0.7.10",
+ "der",
  "pkcs5",
  "rand_core 0.6.4",
- "spki 0.7.3",
+ "spki",
 ]
 
 [[package]]
 name = "pkg-config"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
 
 [[package]]
 name = "plain"
@@ -6842,7 +6471,7 @@ dependencies = [
  "concurrent-queue",
  "hermit-abi",
  "pin-project-lite",
- "rustix 1.1.4",
+ "rustix",
  "windows-sys 0.61.2",
 ]
 
@@ -6854,18 +6483,18 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
 
 [[package]]
 name = "portable-atomic-util"
-version = "0.2.5"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5"
+checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3"
 dependencies = [
  "portable-atomic",
 ]
 
 [[package]]
 name = "potential_utf"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77"
+checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"
 dependencies = [
  "zerovec",
 ]
@@ -6956,37 +6585,13 @@ dependencies = [
 
 [[package]]
 name = "proc-macro-crate"
-version = "3.4.0"
+version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983"
+checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
 dependencies = [
  "toml_edit",
 ]
 
-[[package]]
-name = "proc-macro-error"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
-dependencies = [
- "proc-macro-error-attr",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
- "version_check",
-]
-
-[[package]]
-name = "proc-macro-error-attr"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
-dependencies = [
- "proc-macro2",
- "quote",
- "version_check",
-]
-
 [[package]]
 name = "proc-macro2"
 version = "1.0.106"
@@ -6998,15 +6603,15 @@ dependencies = [
 
 [[package]]
 name = "proptest"
-version = "1.10.0"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37566cb3fdacef14c0737f9546df7cfeadbfbc9fef10991038bf5015d0c80532"
+checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
 dependencies = [
  "bit-set",
  "bit-vec",
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "num-traits",
- "rand 0.9.2",
+ "rand 0.9.4",
  "rand_chacha 0.9.0",
  "rand_xorshift",
  "regex-syntax",
@@ -7132,8 +6737,8 @@ dependencies = [
  "quinn-proto",
  "quinn-udp",
  "rustc-hash",
- "rustls 0.23.37",
- "socket2 0.6.2",
+ "rustls",
+ "socket2",
  "thiserror 2.0.18",
  "tokio",
  "tracing",
@@ -7142,17 +6747,17 @@ dependencies = [
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.13"
+version = "0.11.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31"
+checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
 dependencies = [
  "bytes",
  "getrandom 0.3.4",
  "lru-slab",
- "rand 0.9.2",
+ "rand 0.9.4",
  "ring",
  "rustc-hash",
- "rustls 0.23.37",
+ "rustls",
  "rustls-pki-types",
  "slab",
  "thiserror 2.0.18",
@@ -7170,7 +6775,7 @@ dependencies = [
  "cfg_aliases",
  "libc",
  "once_cell",
- "socket2 0.6.2",
+ "socket2",
  "tracing",
  "windows-sys 0.60.2",
 ]
@@ -7228,9 +6833,9 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.9.2"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
+checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
 dependencies = [
  "rand_chacha 0.9.0",
  "rand_core 0.9.5",
@@ -7293,16 +6898,6 @@ dependencies = [
  "getrandom 0.3.4",
 ]
 
-[[package]]
-name = "rand_distr"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
-dependencies = [
- "num-traits",
- "rand 0.8.5",
-]
-
 [[package]]
 name = "rand_distr"
 version = "0.5.1"
@@ -7310,7 +6905,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
 dependencies = [
  "num-traits",
- "rand 0.9.2",
+ "rand 0.9.4",
 ]
 
 [[package]]
@@ -7349,7 +6944,7 @@ dependencies = [
  "ahash",
  "brotli",
  "paste",
- "rand 0.9.2",
+ "rand 0.9.4",
  "unicase",
 ]
 
@@ -7367,9 +6962,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
 
 [[package]]
 name = "rayon"
-version = "1.11.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
 dependencies = [
  "either",
  "rayon-core",
@@ -7402,16 +6997,16 @@ version = "0.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
 ]
 
 [[package]]
 name = "redox_syscall"
-version = "0.7.3"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16"
+checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
 ]
 
 [[package]]
@@ -7530,12 +7125,12 @@ dependencies = [
  "encoding_rs",
  "futures-core",
  "futures-util",
- "h2 0.4.13",
+ "h2",
  "http 1.4.0",
  "http-body 1.0.1",
  "http-body-util",
- "hyper 1.8.1",
- "hyper-rustls 0.27.7",
+ "hyper",
+ "hyper-rustls",
  "hyper-tls",
  "hyper-util",
  "js-sys",
@@ -7546,7 +7141,7 @@ dependencies = [
  "percent-encoding",
  "pin-project-lite",
  "quinn",
- "rustls 0.23.37",
+ "rustls",
  "rustls-native-certs",
  "rustls-pki-types",
  "serde",
@@ -7555,7 +7150,7 @@ dependencies = [
  "sync_wrapper",
  "tokio",
  "tokio-native-tls",
- "tokio-rustls 0.26.4",
+ "tokio-rustls",
  "tokio-util",
  "tower",
  "tower-http 0.6.8",
@@ -7568,17 +7163,6 @@ dependencies = [
  "webpki-roots 1.0.6",
 ]
 
-[[package]]
-name = "rfc6979"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb"
-dependencies = [
- "crypto-bigint 0.4.9",
- "hmac",
- "zeroize",
-]
-
 [[package]]
 name = "rgb"
 version = "0.8.53"
@@ -7602,12 +7186,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "rle-decode-fast"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
-
 [[package]]
 name = "roaring"
 version = "0.11.3"
@@ -7636,11 +7214,11 @@ dependencies = [
  "num-integer",
  "num-traits",
  "pkcs1",
- "pkcs8 0.10.2",
+ "pkcs8",
  "rand_core 0.6.4",
  "sha2",
- "signature 2.2.0",
- "spki 0.7.3",
+ "signature",
+ "spki",
  "subtle",
  "zeroize",
 ]
@@ -7714,9 +7292,9 @@ checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
 
 [[package]]
 name = "rustc-hash"
-version = "2.1.1"
+version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
 
 [[package]]
 name = "rustc_version"
@@ -7727,56 +7305,31 @@ dependencies = [
  "semver",
 ]
 
-[[package]]
-name = "rustix"
-version = "0.38.44"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
-dependencies = [
- "bitflags 2.11.0",
- "errno",
- "libc",
- "linux-raw-sys 0.4.15",
- "windows-sys 0.59.0",
-]
-
 [[package]]
 name = "rustix"
 version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "errno",
  "libc",
- "linux-raw-sys 0.12.1",
+ "linux-raw-sys",
  "windows-sys 0.61.2",
 ]
 
 [[package]]
 name = "rustls"
-version = "0.21.12"
+version = "0.23.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e"
-dependencies = [
- "log",
- "ring",
- "rustls-webpki 0.101.7",
- "sct",
-]
-
-[[package]]
-name = "rustls"
-version = "0.23.37"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
+checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21"
 dependencies = [
  "aws-lc-rs",
  "log",
  "once_cell",
  "ring",
  "rustls-pki-types",
- "rustls-webpki 0.103.9",
+ "rustls-webpki",
  "subtle",
  "zeroize",
 ]
@@ -7814,19 +7367,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.101.7"
+version = "0.103.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
-dependencies = [
- "ring",
- "untrusted",
-]
-
-[[package]]
-name = "rustls-webpki"
-version = "0.103.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53"
+checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06"
 dependencies = [
  "aws-lc-rs",
  "ring",
@@ -7876,11 +7419,20 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "scc"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc"
+dependencies = [
+ "sdd",
+]
+
 [[package]]
 name = "schannel"
-version = "0.1.28"
+version = "0.1.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1"
+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
 dependencies = [
  "windows-sys 0.61.2",
 ]
@@ -7933,28 +7485,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "sct"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
-dependencies = [
- "ring",
- "untrusted",
-]
-
-[[package]]
-name = "sec1"
-version = "0.3.0"
+name = "sdd"
+version = "3.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928"
-dependencies = [
- "base16ct",
- "der 0.6.1",
- "generic-array",
- "pkcs8 0.9.0",
- "subtle",
- "zeroize",
-]
+checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca"
 
 [[package]]
 name = "security-framework"
@@ -7962,7 +7496,7 @@ version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "core-foundation 0.10.1",
  "core-foundation-sys",
  "libc",
@@ -7981,9 +7515,9 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.27"
+version = "1.0.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
 dependencies = [
  "serde",
  "serde_core",
@@ -8119,6 +7653,32 @@ dependencies = [
  "unsafe-libyaml",
 ]
 
+[[package]]
+name = "serial_test"
+version = "3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f"
+dependencies = [
+ "futures-executor",
+ "futures-util",
+ "log",
+ "once_cell",
+ "parking_lot",
+ "scc",
+ "serial_test_derive",
+]
+
+[[package]]
+name = "serial_test_derive"
+version = "3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
 [[package]]
 name = "sha1"
 version = "0.10.6"
@@ -8126,7 +7686,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "digest",
 ]
 
@@ -8137,7 +7697,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "digest",
 ]
 
@@ -8166,16 +7726,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "signature"
-version = "1.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c"
-dependencies = [
- "digest",
- "rand_core 0.6.4",
-]
-
 [[package]]
 name = "signature"
 version = "2.2.0"
@@ -8188,9 +7738,9 @@ dependencies = [
 
 [[package]]
 name = "simd-adler32"
-version = "0.3.8"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
+checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
 
 [[package]]
 name = "simdutf8"
@@ -8216,15 +7766,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
 
-[[package]]
-name = "sketches-ddsketch"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "slab"
 version = "0.4.12"
@@ -8266,22 +7807,12 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
 
 [[package]]
 name = "socket2"
-version = "0.5.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
-dependencies = [
- "libc",
- "windows-sys 0.52.0",
-]
-
-[[package]]
-name = "socket2"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0"
+checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
 dependencies = [
  "libc",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -8297,11 +7828,11 @@ dependencies = [
 
 [[package]]
 name = "spade"
-version = "2.15.0"
+version = "2.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb313e1c8afee5b5647e00ee0fe6855e3d529eb863a0fdae1d60006c4d1e9990"
+checksum = "9699399fd9349b00b184f5635b074f9ec93afffef30c853f8c875b32c0f8c7fa"
 dependencies = [
- "hashbrown 0.15.5",
+ "hashbrown 0.16.1",
  "num-traits",
  "robust",
  "smallvec",
@@ -8322,16 +7853,6 @@ dependencies = [
  "lock_api",
 ]
 
-[[package]]
-name = "spki"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b"
-dependencies = [
- "base64ct",
- "der 0.6.1",
-]
-
 [[package]]
 name = "spki"
 version = "0.7.3"
@@ -8339,7 +7860,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
 dependencies = [
  "base64ct",
- "der 0.7.10",
+ "der",
 ]
 
 [[package]]
@@ -8487,9 +8008,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "symbolic-common"
-version = "12.17.2"
+version = "12.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "751a2823d606b5d0a7616499e4130a516ebd01a44f39811be2b9600936509c23"
+checksum = "6aba7211a1803a826f108af9f4d86d25abe880712f2a6449479279e861b293f8"
 dependencies = [
  "debugid",
  "memmap2",
@@ -8499,9 +8020,9 @@ dependencies = [
 
 [[package]]
 name = "symbolic-demangle"
-version = "12.17.2"
+version = "12.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79b237cfbe320601dd24b4ac817a5b68bb28f5508e33f08d42be0682cadc8ac9"
+checksum = "595bddd9d363c2ef6fc9fb33b98416ff209c5aae8bdca89a7dbbf2ef5e0ecc45"
 dependencies = [
  "cpp_demangle",
  "rustc-demangle",
@@ -8556,7 +8077,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "core-foundation 0.9.4",
  "system-configuration-sys",
 ]
@@ -8577,152 +8098,6 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
 
-[[package]]
-name = "tantivy"
-version = "0.24.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64a966cb0e76e311f09cf18507c9af192f15d34886ee43d7ba7c7e3803660c43"
-dependencies = [
- "aho-corasick",
- "arc-swap",
- "base64 0.22.1",
- "bitpacking",
- "bon",
- "byteorder",
- "census",
- "crc32fast",
- "crossbeam-channel",
- "downcast-rs",
- "fastdivide",
- "fnv",
- "fs4",
- "htmlescape",
- "hyperloglogplus",
- "itertools 0.14.0",
- "levenshtein_automata",
- "log",
- "lru 0.12.5",
- "lz4_flex 0.11.5",
- "measure_time",
- "memmap2",
- "once_cell",
- "oneshot",
- "rayon",
- "regex",
- "rust-stemmers",
- "rustc-hash",
- "serde",
- "serde_json",
- "sketches-ddsketch",
- "smallvec",
- "tantivy-bitpacker",
- "tantivy-columnar",
- "tantivy-common",
- "tantivy-fst",
- "tantivy-query-grammar",
- "tantivy-stacker",
- "tantivy-tokenizer-api",
- "tempfile",
- "thiserror 2.0.18",
- "time",
- "uuid",
- "winapi",
-]
-
-[[package]]
-name = "tantivy-bitpacker"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1adc286a39e089ae9938935cd488d7d34f14502544a36607effd2239ff0e2494"
-dependencies = [
- "bitpacking",
-]
-
-[[package]]
-name = "tantivy-columnar"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6300428e0c104c4f7db6f95b466a6f5c1b9aece094ec57cdd365337908dc7344"
-dependencies = [
- "downcast-rs",
- "fastdivide",
- "itertools 0.14.0",
- "serde",
- "tantivy-bitpacker",
- "tantivy-common",
- "tantivy-sstable",
- "tantivy-stacker",
-]
-
-[[package]]
-name = "tantivy-common"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e91b6ea6090ce03dc72c27d0619e77185d26cc3b20775966c346c6d4f7e99d7f"
-dependencies = [
- "async-trait",
- "byteorder",
- "ownedbytes",
- "serde",
- "time",
-]
-
-[[package]]
-name = "tantivy-fst"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
-dependencies = [
- "byteorder",
- "regex-syntax",
- "utf8-ranges",
-]
-
-[[package]]
-name = "tantivy-query-grammar"
-version = "0.24.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e810cdeeebca57fc3f7bfec5f85fdbea9031b2ac9b990eb5ff49b371d52bbe6a"
-dependencies = [
- "nom 7.1.3",
- "serde",
- "serde_json",
-]
-
-[[package]]
-name = "tantivy-sstable"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "709f22c08a4c90e1b36711c1c6cad5ae21b20b093e535b69b18783dd2cb99416"
-dependencies = [
- "futures-util",
- "itertools 0.14.0",
- "tantivy-bitpacker",
- "tantivy-common",
- "tantivy-fst",
- "zstd",
-]
-
-[[package]]
-name = "tantivy-stacker"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bcdebb267671311d1e8891fd9d1301803fdb8ad21ba22e0a30d0cab49ba59c1"
-dependencies = [
- "murmurhash32",
- "rand_distr 0.4.3",
- "tantivy-common",
-]
-
-[[package]]
-name = "tantivy-tokenizer-api"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfa942fcee81e213e09715bbce8734ae2180070b97b33839a795ba1de201547d"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "tap"
 version = "1.0.1"
@@ -8731,9 +8106,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "tar"
-version = "0.4.44"
+version = "0.4.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"
+checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973"
 dependencies = [
  "filetime",
  "libc",
@@ -8742,14 +8117,14 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.26.0"
+version = "3.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0"
+checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
- "fastrand 2.3.0",
+ "fastrand 2.4.1",
  "getrandom 0.4.2",
  "once_cell",
- "rustix 1.1.4",
+ "rustix",
  "windows-sys 0.61.2",
 ]
 
@@ -8895,9 +8270,9 @@ dependencies = [
 
 [[package]]
 name = "tinystr"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869"
+checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
 dependencies = [
  "displaydoc",
  "zerovec",
@@ -8915,9 +8290,9 @@ dependencies = [
 
 [[package]]
 name = "tinyvec"
-version = "1.10.0"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
+checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
 dependencies = [
  "tinyvec_macros",
 ]
@@ -8963,9 +8338,9 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.50.0"
+version = "1.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d"
+checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776"
 dependencies = [
  "bytes",
  "libc",
@@ -8973,16 +8348,16 @@ dependencies = [
  "parking_lot",
  "pin-project-lite",
  "signal-hook-registry",
- "socket2 0.6.2",
+ "socket2",
  "tokio-macros",
  "windows-sys 0.61.2",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "2.6.1"
+version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c"
+checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -8999,23 +8374,13 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "tokio-rustls"
-version = "0.24.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
-dependencies = [
- "rustls 0.21.12",
- "tokio",
-]
-
 [[package]]
 name = "tokio-rustls"
 version = "0.26.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
 dependencies = [
- "rustls 0.23.37",
+ "rustls",
  "tokio",
 ]
 
@@ -9045,18 +8410,18 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "0.7.5+spec-1.1.0"
+version = "1.1.1+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
 dependencies = [
  "serde_core",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.23.10+spec-1.0.0"
+version = "0.25.11+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269"
+checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b"
 dependencies = [
  "indexmap",
  "toml_datetime",
@@ -9066,9 +8431,9 @@ dependencies = [
 
 [[package]]
 name = "toml_parser"
-version = "1.0.9+spec-1.1.0"
+version = "1.1.2+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4"
+checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
 dependencies = [
  "winnow",
 ]
@@ -9095,7 +8460,7 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "bytes",
  "http 1.4.0",
  "http-body 1.0.1",
@@ -9113,7 +8478,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
 dependencies = [
  "async-compression",
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "bytes",
  "futures-core",
  "futures-util",
@@ -9208,9 +8573,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-subscriber"
-version = "0.3.22"
+version = "0.3.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e"
+checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
 dependencies = [
  "matchers",
  "nu-ansi-term",
@@ -9246,7 +8611,7 @@ version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
 dependencies = [
- "rand 0.9.2",
+ "rand 0.9.4",
 ]
 
 [[package]]
@@ -9355,9 +8720,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.12.0"
+version = "1.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
+checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c"
 
 [[package]]
 name = "unicode-width"
@@ -9406,7 +8771,7 @@ dependencies = [
  "log",
  "native-tls",
  "once_cell",
- "rustls 0.23.37",
+ "rustls",
  "rustls-pki-types",
  "serde",
  "serde_json",
@@ -9454,9 +8819,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.21.0"
+version = "1.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb"
+checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9"
 dependencies = [
  "getrandom 0.4.2",
  "js-sys",
@@ -9560,9 +8925,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.114"
+version = "0.2.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e"
+checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -9573,23 +8938,19 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.64"
+version = "0.4.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8"
+checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8"
 dependencies = [
- "cfg-if",
- "futures-util",
  "js-sys",
- "once_cell",
  "wasm-bindgen",
- "web-sys",
 ]
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.114"
+version = "0.2.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6"
+checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -9597,9 +8958,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.114"
+version = "0.2.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3"
+checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904"
 dependencies = [
  "bumpalo",
  "proc-macro2",
@@ -9610,9 +8971,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.114"
+version = "0.2.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16"
+checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129"
 dependencies = [
  "unicode-ident",
 ]
@@ -9658,7 +9019,7 @@ version = "0.244.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "hashbrown 0.15.5",
  "indexmap",
  "semver",
@@ -9666,9 +9027,9 @@ dependencies = [
 
 [[package]]
 name = "web-sys"
-version = "0.3.91"
+version = "0.3.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9"
+checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -10036,9 +9397,9 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
 
 [[package]]
 name = "winnow"
-version = "0.7.14"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829"
+checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5"
 dependencies = [
  "memchr",
 ]
@@ -10055,7 +9416,7 @@ dependencies = [
  "futures",
  "http 1.4.0",
  "http-body-util",
- "hyper 1.8.1",
+ "hyper",
  "hyper-util",
  "log",
  "once_cell",
@@ -10124,7 +9485,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
 dependencies = [
  "anyhow",
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "indexmap",
  "log",
  "serde",
@@ -10181,9 +9542,9 @@ dependencies = [
 
 [[package]]
 name = "writeable"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
+checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
 
 [[package]]
 name = "wyz"
@@ -10201,7 +9562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
 dependencies = [
  "libc",
- "rustix 1.1.4",
+ "rustix",
 ]
 
 [[package]]
@@ -10230,9 +9591,9 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
 
 [[package]]
 name = "yoke"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954"
+checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
 dependencies = [
  "stable_deref_trait",
  "yoke-derive",
@@ -10241,9 +9602,9 @@ dependencies = [
 
 [[package]]
 name = "yoke-derive"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
+checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -10253,18 +9614,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.40"
+version = "0.8.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5"
+checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.40"
+version = "0.8.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953"
+checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -10273,18 +9634,18 @@ dependencies = [
 
 [[package]]
 name = "zerofrom"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"
+checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df"
 dependencies = [
  "zerofrom-derive",
 ]
 
 [[package]]
 name = "zerofrom-derive"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
+checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -10300,9 +9661,9 @@ checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
 
 [[package]]
 name = "zerotrie"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851"
+checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
 dependencies = [
  "displaydoc",
  "yoke",
@@ -10311,9 +9672,9 @@ dependencies = [
 
 [[package]]
 name = "zerovec"
-version = "0.11.5"
+version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002"
+checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
 dependencies = [
  "yoke",
  "zerofrom",
@@ -10322,9 +9683,9 @@ dependencies = [
 
 [[package]]
 name = "zerovec-derive"
-version = "0.11.2"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
+checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
 dependencies = [
  "proc-macro2",
  "quote",
diff --git a/Cargo.toml b/Cargo.toml
index 307d1d2138b..010f10e0846 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -14,6 +14,7 @@ members = [
     "rust/lance-namespace",
     "rust/lance-namespace-impls",
     "rust/lance-namespace-datafusion",
+    "rust/lance-tokenizer",
     "rust/lance-table",
     "rust/lance-test-macros",
     "rust/lance-testing",
@@ -21,6 +22,7 @@ members = [
     "rust/compression/fsst",
     "rust/compression/bitpacking",
     "rust/arrow-scalar",
+    "rust/arrow-stats",
 ]
 exclude = ["python", "java/lance-jni"]
 # Python package needs to be built by maturin.
@@ -28,7 +30,7 @@ resolver = "3"
 
 
 [workspace.package]
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 edition = "2024"
 authors = ["Lance Devs <dev@lance.org>"]
 license = "Apache-2.0"
@@ -52,28 +54,30 @@ rust-version = "1.91.0"
 
 [workspace.dependencies]
 libc = "0.2.176"
-lance = { version = "=4.0.0-beta.7", path = "./rust/lance", default-features = false }
-lance-arrow = { version = "=4.0.0-beta.7", path = "./rust/lance-arrow" }
-lance-core = { version = "=4.0.0-beta.7", path = "./rust/lance-core" }
-lance-datafusion = { version = "=4.0.0-beta.7", path = "./rust/lance-datafusion" }
-lance-datagen = { version = "=4.0.0-beta.7", path = "./rust/lance-datagen" }
-lance-encoding = { version = "=4.0.0-beta.7", path = "./rust/lance-encoding" }
-lance-file = { version = "=4.0.0-beta.7", path = "./rust/lance-file" }
-lance-geo = { version = "=4.0.0-beta.7", path = "./rust/lance-geo" }
-lance-index = { version = "=4.0.0-beta.7", path = "./rust/lance-index" }
-lance-io = { version = "=4.0.0-beta.7", path = "./rust/lance-io", default-features = false }
-lance-linalg = { version = "=4.0.0-beta.7", path = "./rust/lance-linalg" }
-lance-namespace = { version = "=4.0.0-beta.7", path = "./rust/lance-namespace" }
-lance-namespace-impls = { version = "=4.0.0-beta.7", path = "./rust/lance-namespace-impls" }
-lance-namespace-datafusion = { version = "=4.0.0-beta.7", path = "./rust/lance-namespace-datafusion" }
-lance-namespace-reqwest-client = "0.5.2"
-lance-table = { version = "=4.0.0-beta.7", path = "./rust/lance-table" }
-lance-test-macros = { version = "=4.0.0-beta.7", path = "./rust/lance-test-macros" }
-lance-testing = { version = "=4.0.0-beta.7", path = "./rust/lance-testing" }
+lance = { version = "=6.0.0-beta.1", path = "./rust/lance", default-features = false }
+lance-arrow = { version = "=6.0.0-beta.1", path = "./rust/lance-arrow" }
+lance-core = { version = "=6.0.0-beta.1", path = "./rust/lance-core" }
+lance-datafusion = { version = "=6.0.0-beta.1", path = "./rust/lance-datafusion" }
+lance-datagen = { version = "=6.0.0-beta.1", path = "./rust/lance-datagen" }
+lance-encoding = { version = "=6.0.0-beta.1", path = "./rust/lance-encoding" }
+lance-file = { version = "=6.0.0-beta.1", path = "./rust/lance-file" }
+lance-geo = { version = "=6.0.0-beta.1", path = "./rust/lance-geo" }
+lance-index = { version = "=6.0.0-beta.1", path = "./rust/lance-index" }
+lance-io = { version = "=6.0.0-beta.1", path = "./rust/lance-io", default-features = false }
+lance-linalg = { version = "=6.0.0-beta.1", path = "./rust/lance-linalg" }
+lance-namespace = { version = "=6.0.0-beta.1", path = "./rust/lance-namespace" }
+lance-namespace-impls = { version = "=6.0.0-beta.1", path = "./rust/lance-namespace-impls" }
+lance-namespace-datafusion = { version = "=6.0.0-beta.1", path = "./rust/lance-namespace-datafusion" }
+lance-namespace-reqwest-client = "0.6.1"
+lance-tokenizer = { version = "=6.0.0-beta.1", path = "./rust/lance-tokenizer" }
+lance-table = { version = "=6.0.0-beta.1", path = "./rust/lance-table" }
+lance-test-macros = { version = "=6.0.0-beta.1", path = "./rust/lance-test-macros" }
+lance-testing = { version = "=6.0.0-beta.1", path = "./rust/lance-testing" }
 approx = "0.5.1"
 # Note that this one does not include pyarrow
 arrow = { version = "57.0.0", optional = false, features = ["prettyprint"] }
-arrow-scalar = { version = "=57.0.0", path = "./rust/arrow-scalar" }
+lance-arrow-scalar = { version = "=57.0.0", path = "./rust/arrow-scalar" }
+lance-arrow-stats = { version = "=57.0.0", path = "./rust/arrow-stats" }
 arrow-arith = "57.0.0"
 arrow-array = "57.0.0"
 arrow-buffer = "57.0.0"
@@ -89,13 +93,13 @@ async-trait = "0.1"
 axum = "0.7"
 aws-config = "1.2.0"
 aws-credential-types = "1.2.0"
-aws-sdk-dynamodb = "1.38.0"
-aws-sdk-s3 = "1.38.0"
+aws-sdk-dynamodb = { version = "1.38.0", default-features = false }
+aws-sdk-s3 = { version = "1.38.0", default-features = false }
 half = { "version" = "2.1", default-features = false, features = [
     "num-traits",
     "std",
 ] }
-lance-bitpacking = { version = "=4.0.0-beta.7", path = "./rust/compression/bitpacking" }
+lance-bitpacking = { version = "=6.0.0-beta.1", path = "./rust/compression/bitpacking" }
 bitvec = "1"
 bytes = "1.11.1"
 byteorder = "1.5"
@@ -136,7 +140,7 @@ deepsize = "0.2.0"
 dirs = "6.0.0"
 either = "1.0"
 fst = { version = "0.4.7", features = ["levenshtein"] }
-fsst = { version = "=4.0.0-beta.7", path = "./rust/compression/fsst" }
+fsst = { version = "=6.0.0-beta.1", path = "./rust/compression/fsst" }
 futures = "0.3"
 geoarrow-array = "0.7"
 geoarrow-schema = "0.7"
@@ -146,8 +150,9 @@ geo-types = "0.7.16"
 http = "1.1.0"
 humantime = "2.2.0"
 hyperloglogplus = { version = "0.4.1", features = ["const-loop"] }
+io-uring = "0.7"
 itertools = "0.13"
-jieba-rs = { version = "0.8.1", default-features = false }
+jieba-rs = { version = "0.9.0", default-features = false }
 jsonb = { version = "0.5.3", default-features = false, features = ["databend"] }
 libm = "0.2.15"
 log = "0.4"
@@ -177,12 +182,11 @@ rustc_version = "0.4"
 serde = { version = "^1" }
 serde_json = { version = "1" }
 semver = "1.0"
+serial_test = "3"
 slatedb = "0.3"
 snafu = "0.9"
 strum = "0.26"
-tantivy = { version = "0.24.1", features = ["stopwords"] }
 lindera = { version = "0.44.0" }
-lindera-tantivy = { version = "0.44.0" }
 tempfile = "3"
 test-log = { version = "0.2.15" }
 tokio = { version = "1.23", features = [
diff --git a/README.md b/README.md
index 6eeea984113..2f0b2bca18f 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,8 @@
 
 <a href="https://lance.org">Documentation</a> •
 <a href="https://lance.org/community">Community</a> •
-<a href="https://discord.gg/lance">Discord</a>
+<a href="https://discord.gg/lance">Discord</a> •
+<a href="https://groups.google.com/a/lance.org/g/dev">Mailing List</a>
 
 [CI]: https://github.com/lance-format/lance/actions/workflows/rust.yml
 [CI Badge]: https://github.com/lance-format/lance/actions/workflows/rust.yml/badge.svg
@@ -69,9 +70,6 @@ To install a preview release:
 pip install --pre --extra-index-url https://pypi.fury.io/lance-format/pylance
 ```
 
-> [!NOTE]
-> For versions prior to 1.0.0-beta.4, you can find them at https://pypi.fury.io/lancedb/pylance
-
 > [!TIP]
 > Preview releases are released more often than full releases and contain the
 > latest features and bug fixes. They receive the same level of testing as full releases.
diff --git a/ci/create_rc_discussion.sh b/ci/create_rc_discussion.sh
index 875308f85bd..c19ce9ce22f 100755
--- a/ci/create_rc_discussion.sh
+++ b/ci/create_rc_discussion.sh
@@ -21,7 +21,7 @@ DISCUSSION_TITLE="[VOTE] Release Candidate ${RC_TAG}"
 # Determine vote duration based on release type
 case "$RELEASE_TYPE" in
     major)
-        VOTE_DURATION_DAYS=7
+        VOTE_DURATION_DAYS=3
         ;;
     minor)
         VOTE_DURATION_DAYS=3
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index ec95d95e47f..f592df6966c 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -7,7 +7,7 @@ To build the docs, first install requirements:
 
 ```bash
 cd docs
-uc sync --dev
+uv sync --dev
 ```
 
 Then build and start the docs server:
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index 2a84cf12213..73116cf0c78 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -1,5 +1,5 @@
 site_name: Lance
-site_description: Open Lakehouse Format for Multimodal AI
+site_description: "Documentation for Lance, an open lakehouse format for Multimodal AI"
 site_url: https://lance.org/
 docs_dir: src
 
@@ -77,9 +77,5 @@ extra:
 copyright: © 2025 Lance Format. All rights reserved.
 
 extra_css:
-  - assets/stylesheets/termynal.css
   - assets/stylesheets/home.css
 
-extra_javascript:
-  - assets/javascript/termynal.js
-
diff --git a/docs/src/assets/javascript/termynal.js b/docs/src/assets/javascript/termynal.js
deleted file mode 100644
index 77ec6cb01c8..00000000000
--- a/docs/src/assets/javascript/termynal.js
+++ /dev/null
@@ -1,197 +0,0 @@
-/**
- * termynal.js
- * A lightweight, modern and extensible animated terminal window, using
- * async/await.
- *
- * @author Ines Montani <ines@ines.io>
- * @version 0.0.1
- * @license MIT
- */
-
-'use strict';
-
-/** Generate a terminal widget. */
-class Termynal {
-    /**
-     * Construct the widget's settings.
-     * @param {(string|Node)=} container - Query selector or container element.
-     * @param {Object=} options - Custom settings.
-     * @param {string} options.prefix - Prefix to use for data attributes.
-     * @param {number} options.startDelay - Delay before animation, in ms.
-     * @param {number} options.typeDelay - Delay between each typed character, in ms.
-     * @param {number} options.lineDelay - Delay between each line, in ms.
-     * @param {number} options.progressLength - Number of characters displayed as progress bar.
-     * @param {string} options.progressChar – Character to use for progress bar, defaults to █.
-	 * @param {number} options.progressPercent - Max percent of progress.
-     * @param {string} options.cursor – Character to use for cursor, defaults to ▋.
-     * @param {Object[]} lineData - Dynamically loaded line data objects.
-     * @param {boolean} options.noInit - Don't initialise the animation.
-     */
-    constructor(container = '#termynal', options = {}) {
-        this.container = (typeof container === 'string') ? document.querySelector(container) : container;
-        this.pfx = `data-${options.prefix || 'ty'}`;
-        this.startDelay = options.startDelay
-            || parseFloat(this.container.getAttribute(`${this.pfx}-startDelay`)) || 600;
-        this.typeDelay = options.typeDelay
-            || parseFloat(this.container.getAttribute(`${this.pfx}-typeDelay`)) || 90;
-        this.lineDelay = options.lineDelay
-            || parseFloat(this.container.getAttribute(`${this.pfx}-lineDelay`)) || 1500;
-        this.progressLength = options.progressLength
-            || parseFloat(this.container.getAttribute(`${this.pfx}-progressLength`)) || 40;
-        this.progressChar = options.progressChar
-            || this.container.getAttribute(`${this.pfx}-progressChar`) || '█';
-		this.progressPercent = options.progressPercent
-            || parseFloat(this.container.getAttribute(`${this.pfx}-progressPercent`)) || 100;
-        this.cursor = options.cursor
-            || this.container.getAttribute(`${this.pfx}-cursor`) || '▋';
-        this.lineData = this.lineDataToElements(options.lineData || []);
-        if (!options.noInit) this.init()
-    }
-
-    /**
-     * Initialise the widget, get lines, clear container and start animation.
-     */
-    init() {
-        // Appends dynamically loaded lines to existing line elements.
-        this.lines = [...this.container.querySelectorAll(`[${this.pfx}]`)].concat(this.lineData);
-
-        /** 
-         * Calculates width and height of Termynal container.
-         * If container is empty and lines are dynamically loaded, defaults to browser `auto` or CSS.
-         */ 
-        const containerStyle = getComputedStyle(this.container);
-        this.container.style.width = containerStyle.width !== '0px' ? 
-            containerStyle.width : undefined;
-        this.container.style.minHeight = containerStyle.height !== '0px' ? 
-            containerStyle.height : undefined;
-
-        this.container.setAttribute('data-termynal', '');
-        this.container.innerHTML = '';
-        this.start();
-    }
-
-    /**
-     * Start the animation and rener the lines depending on their data attributes.
-     */
-    async start() {
-        await this._wait(this.startDelay);
-
-        for (let line of this.lines) {
-            const type = line.getAttribute(this.pfx);
-            const delay = line.getAttribute(`${this.pfx}-delay`) || this.lineDelay;
-
-            if (type == 'input') {
-                line.setAttribute(`${this.pfx}-cursor`, this.cursor);
-                await this.type(line);
-                await this._wait(delay);
-            }
-
-            else if (type == 'progress') {
-                await this.progress(line);
-                await this._wait(delay);
-            }
-
-            else {
-                this.container.appendChild(line);
-                await this._wait(delay);
-            }
-
-            line.removeAttribute(`${this.pfx}-cursor`);
-        }
-    }
-
-    /**
-     * Animate a typed line.
-     * @param {Node} line - The line element to render.
-     */
-    async type(line) {
-        const chars = [...line.textContent];
-        const delay = line.getAttribute(`${this.pfx}-typeDelay`) || this.typeDelay;
-        line.textContent = '';
-        this.container.appendChild(line);
-
-        for (let char of chars) {
-            await this._wait(delay);
-            line.textContent += char;
-        }
-    }
-
-    /**
-     * Animate a progress bar.
-     * @param {Node} line - The line element to render.
-     */
-    async progress(line) {
-        const progressLength = line.getAttribute(`${this.pfx}-progressLength`)
-            || this.progressLength;
-        const progressChar = line.getAttribute(`${this.pfx}-progressChar`)
-            || this.progressChar;
-        const chars = progressChar.repeat(progressLength);
-		const progressPercent = line.getAttribute(`${this.pfx}-progressPercent`)
-			|| this.progressPercent;
-        line.textContent = '';
-        this.container.appendChild(line);
-
-        for (let i = 1; i < chars.length + 1; i++) {
-            await this._wait(this.typeDelay);
-            const percent = Math.round(i / chars.length * 100);
-            line.textContent = `${chars.slice(0, i)} ${percent}%`;
-			if (percent>progressPercent) {
-				break;
-			}
-        }
-    }
-
-    /**
-     * Helper function for animation delays, called with `await`.
-     * @param {number} time - Timeout, in ms.
-     */
-    _wait(time) {
-        return new Promise(resolve => setTimeout(resolve, time));
-    }
-
-    /**
-     * Converts line data objects into line elements.
-     * 
-     * @param {Object[]} lineData - Dynamically loaded lines.
-     * @param {Object} line - Line data object.
-     * @returns {Element[]} - Array of line elements.
-     */
-    lineDataToElements(lineData) {
-        return lineData.map(line => {
-            let div = document.createElement('div');
-            div.innerHTML = `<span ${this._attributes(line)}>${line.value || ''}</span>`;
-
-            return div.firstElementChild;
-        });
-    }
-
-    /**
-     * Helper function for generating attributes string.
-     * 
-     * @param {Object} line - Line data object.
-     * @returns {string} - String of attributes.
-     */
-    _attributes(line) {
-        let attrs = '';
-        for (let prop in line) {
-            attrs += this.pfx;
-
-            if (prop === 'type') {
-                attrs += `="${line[prop]}" `
-            } else if (prop !== 'value') {
-                attrs += `-${prop}="${line[prop]}" `
-            }
-        }
-
-        return attrs;
-    }
-}
-
-/**
-* HTML API: If current script has container(s) specified, initialise Termynal.
-*/
-if (document.currentScript.hasAttribute('data-termynal-container')) {
-    const containers = document.currentScript.getAttribute('data-termynal-container');
-    containers.split('|')
-        .forEach(container => new Termynal(container))
-}
diff --git a/docs/src/assets/stylesheets/home.css b/docs/src/assets/stylesheets/home.css
index 086b6b43be6..57795ecd957 100644
--- a/docs/src/assets/stylesheets/home.css
+++ b/docs/src/assets/stylesheets/home.css
@@ -241,89 +241,3 @@
   flex-direction: row-reverse;
 }
 
-/* Terminal container adjustments */
-[data-termynal] {
-  margin: 0 auto;
-  width: 500px;
-  max-width: 100%;
-  font-size: 14px;
-  box-shadow: 0 8px 24px rgba(0, 0, 0, 0.15);
-}
-
-/* Responsive Design */
-@media (max-width: 768px) {
-  .container {
-    padding-right: 10px;
-    padding-left: 10px;
-  }
-
-  .mdx-hero__content h1 {
-    font-size: 2.2rem;
-  }
-
-  .mdx-hero__content p {
-    font-size: 1.1rem;
-  }
-
-  .lance-feature-section {
-    padding: 50px 0;
-  }
-
-  .lance-feature-content {
-    flex-direction: column;
-    gap: 40px;
-  }
-
-  .lance-feature-section.reverse .lance-feature-content {
-    flex-direction: column;
-  }
-
-  .lance-feature-text h2 {
-    font-size: 1.8rem;
-  }
-
-  .lance-feature-text p {
-    font-size: 1rem;
-  }
-
-  .lance-feature-demo {
-    min-width: 100%;
-  }
-
-  [data-termynal] {
-    width: 100% !important;
-    max-width: 100% !important;
-    font-size: 12px;
-    padding: 50px 25px 25px;
-  }
-}
-
-@media (max-width: 960px) {
-  .intro-message .list-inline li {
-    display: block;
-    padding: 0;
-    margin-bottom: 12px;
-  }
-
-  .intro-message .md-button {
-    display: block;
-    margin: 0 auto;
-    width: 90%;
-    max-width: 350px;
-  }
-}
-
-@media (max-width: 480px) {
-  .intro-message h1 {
-    font-size: 1.8rem;
-  }
-
-  .intro-message .md-button {
-    max-width: 300px;
-  }
-
-  [data-termynal] {
-    font-size: 10px;
-    padding: 40px 15px 15px;
-  }
-}
diff --git a/docs/src/assets/stylesheets/termynal.css b/docs/src/assets/stylesheets/termynal.css
deleted file mode 100644
index 456f469bea3..00000000000
--- a/docs/src/assets/stylesheets/termynal.css
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * termynal.js
- *
- * @author Ines Montani <ines@ines.io>
- * @version 0.0.1
- * @license MIT
- */
-
-:root {
-    --color-bg: #252a33;
-    --color-text: #eee;
-    --color-text-subtle: #a2a2a2;
-    --color-keyword: #c678dd;
-    --color-string: #98c379;
-    --color-function: #61afef;
-    --color-number: #d19a66;
-    --color-comment: #5c6370;
-}
-
-[data-termynal] {
-    width: 750px;
-    max-width: 100%;
-    background: var(--color-bg);
-    color: var(--color-text);
-    font-size: 18px;
-    font-family: 'Fira Mono', Consolas, Menlo, Monaco, 'Courier New', Courier, monospace;
-    border-radius: 4px;
-    padding: 75px 45px 35px;
-    position: relative;
-    -webkit-box-sizing: border-box;
-            box-sizing: border-box;
-}
-
-[data-termynal]:before {
-    content: '';
-    position: absolute;
-    top: 15px;
-    left: 15px;
-    display: inline-block;
-    width: 15px;
-    height: 15px;
-    border-radius: 50%;
-    /* A little hack to display the window buttons in one pseudo element. */
-    background: #d9515d;
-    -webkit-box-shadow: 25px 0 0 #f4c025, 50px 0 0 #3ec930;
-            box-shadow: 25px 0 0 #f4c025, 50px 0 0 #3ec930;
-}
-
-[data-termynal]:after {
-    content: 'Python';
-    position: absolute;
-    color: var(--color-text-subtle);
-    top: 5px;
-    left: 0;
-    width: 100%;
-    text-align: center;
-}
-
-[data-ty] {
-    display: block;
-    line-height: 2;
-    white-space: pre;
-}
-
-[data-ty]:before {
-    /* Set up defaults and ensure empty lines are displayed. */
-    content: '';
-    display: inline-block;
-    vertical-align: middle;
-}
-
-[data-ty="input"]:before,
-[data-ty-prompt]:before {
-    margin-right: 0.75em;
-    color: var(--color-text-subtle);
-}
-
-[data-ty="input"]:before {
-    content: '$';
-}
-
-[data-ty][data-ty-prompt]:before {
-    content: attr(data-ty-prompt);
-}
-
-[data-ty-cursor]:after {
-    content: attr(data-ty-cursor);
-    font-family: monospace;
-    margin-left: 0.5em;
-    -webkit-animation: blink 1s infinite;
-            animation: blink 1s infinite;
-}
-
-
-/* Cursor animation */
-
-@-webkit-keyframes blink {
-    50% {
-        opacity: 0;
-    }
-}
-
-@keyframes blink {
-    50% {
-        opacity: 0;
-    }
-}
diff --git a/docs/src/community/communication.md b/docs/src/community/communication.md
index 1777788f345..21b7e9eba61 100644
--- a/docs/src/community/communication.md
+++ b/docs/src/community/communication.md
@@ -24,8 +24,12 @@ There are two mailing lists used by Lance:
 - [dev@lance.org](https://groups.google.com/a/lance.org/g/dev): Archive for discussions from GitHub Discussions (public)
 - [private@lance.org](https://groups.google.com/a/lance.org/g/private): Private discussions, security and harassment issues reporting, voting (private, PMC only)
 
-## Community Events
+## Community Sync
 
-Check out [Lance Community Events](https://calendar.google.com/calendar/u/0?cid=Y29tbXVuaXR5QGxhbmNlLm9yZw) 
-for upcoming virtual or in-person events related to Lance.
-The community can self-organize additional meetups as well.
+We hold Lance community syncs every other Thursday at 9am PT. Everyone is welcome to attend! 
+
+- Subscribe to the [Lance Format mailing list](https://groups.google.com/a/lance.org/g/dev) to receive the meeting invite
+- Feel free to suggest discussion topics on the [meeting notes](https://docs.google.com/document/d/1cP058pJLVjj39DGaFV5yHI6qyWBwp1B2YKyIwXzYlAI/edit?tab=t.0)
+- Past recordings can be found [here](https://www.youtube.com/playlist?list=PLQysAafL04jUgOj7j6mCE5fcZEBB9Amu9)
+
+If you have issues seeing the calendar invite, please reach out via the mailing list or the [Lance discord](https://discord.gg/lance).
\ No newline at end of file
diff --git a/docs/src/community/voting.md b/docs/src/community/voting.md
index d124c0db1b3..8c5ac341e67 100644
--- a/docs/src/community/voting.md
+++ b/docs/src/community/voting.md
@@ -44,7 +44,7 @@ A **-1** binding vote is considered a veto for all decision types. Vetoes:
 | Changes in maintainers and PMC rosters                                        | 3 (excluding the people proposed for change) | PMC                            | Private Mailing List                  | 1 week         |
 | Incubating subproject graduation to subproject                                | 3                                            | PMC                            | GitHub Discussions                    | 3 days         |
 | Subproject management                                                         | 1                                            | PMC                            | GitHub Discussions                    | N/A            |
-| Release a new stable major version of the core project                            | 3                                            | PMC                            | GitHub Discussions                    | 1 week         |
+| Release a new stable major version of the core project                            | 3                                            | PMC                            | GitHub Discussions                    | 3 days         |
 | Release a new stable minor version of the core project                            | 3                                            | PMC                            | GitHub Discussions                    | 3 days         |
 | Release a new stable patch version of the core project                            | 3                                            | PMC                            | GitHub Discussions                    | N/A            |
 | Lance Format Specification modifications                                      | 3 (excluding proposer)                       | PMC                            | GitHub Discussions (with a GitHub PR) | 1 week         |
diff --git a/docs/src/format/file/encoding.md b/docs/src/format/file/encoding.md
index 97b6c007b7d..0ff1f479b1c 100644
--- a/docs/src/format/file/encoding.md
+++ b/docs/src/format/file/encoding.md
@@ -670,3 +670,22 @@ in row-major format rather than the default columnar format. This reduces the nu
 random access but prevents reading individual fields independently.
 
 This is always opt-in and should only be used when all struct fields are typically accessed together.
+
+#### Mini-Block Size Tuning
+
+Each mini-block contains at most 4096 values by default. Because an entire mini-block must be fetched to
+read any value within it, workloads that read only a small contiguous slice of each mini-block may experience
+read amplification.
+
+The default is appropriate for the vast majority of deployments. Local disks and typical cloud object storage
+(where the client and bucket are in the same region) have more than enough bandwidth that the overhead from
+the default mini-block size is negligible. You should only consider changing this setting if you have
+confirmed — through profiling — that mini-block read amplification is saturating your available bandwidth
+(for example, accessing a remote object store over a constrained network link).
+
+The maximum number of values per mini-block can be lowered via an environment variable:
+
+- `LANCE_MINIBLOCK_MAX_VALUES` (default `4096`): upper bound on the number of values in a single mini-block chunk.
+
+Reducing this value produces smaller mini-blocks, which reduces the amount of data fetched per read at the
+cost of more mini-blocks and slightly more metadata overhead.
diff --git a/docs/src/format/table/index.md b/docs/src/format/table/index.md
index 45484e7b8b8..d430061451f 100644
--- a/docs/src/format/table/index.md
+++ b/docs/src/format/table/index.md
@@ -121,6 +121,17 @@ or independently of column indices due to variable encoding widths (for Lance fi
 
 </details>
 
+!!! note "Field-to-column mapping differs between data storage versions"
+
+    In **2.0**, all fields (including non-leaf fields like struct and list containers) are assigned
+    sequential column indices in `column_indices`.
+
+    In **2.1+**, non-leaf fields (unpacked structs, list containers) are assigned `-1` in
+    `column_indices` because their validity information is folded into repetition/definition
+    levels. Only leaf fields and packed structs have column indices.
+
+    See the [5.0.0 migration guide](../../guide/migration.md#500) for a detailed example.
+
 ## Deletion Files
 
 Deletion files (a.k.a. deletion vectors) track deleted rows without rewriting data files.
diff --git a/docs/src/format/table/index/system/frag_reuse.md b/docs/src/format/table/index/system/frag_reuse.md
index f0ba6acab2f..a508b52b920 100644
--- a/docs/src/format/table/index/system/frag_reuse.md
+++ b/docs/src/format/table/index/system/frag_reuse.md
@@ -31,5 +31,27 @@ The index accumulates a new **reuse version** every time a compaction is execute
 As long as all the scalar and vector indices are created after the specific reuse version,
 the indices are all caught up and the specific reuse version can be trimmed.
 
-It is expected that the user schedules an additional process to trim the index periodically
-to keep the list of reuse versions in control.
\ No newline at end of file
+## Impacts
+
+### Conflict Resolution
+
+The presence of the Fragment Reuse Index changes how Lance detects conflicts between concurrent
+operations. Operations that would normally conflict with compaction (such as index building) can
+proceed without conflict when the FRI is in use. For full details on how conflict detection is
+affected, see [conflict resolution](../../transaction.md#conflict-resolution).
+
+### Index Load Cost
+
+When the FRI is present, indices must be remapped at load time. Each time an index is loaded into
+the cache, the FRI is applied to translate old row addresses to current ones. This adds a small
+cost to index loading but does not affect query performance once the index is cached.
+
+### FRI Growth and Cleanup
+
+The FRI grows with each compaction. Every compaction that defers index remapping adds a new reuse
+version to the index. Over time, this can accumulate and increase the cost of index loading since
+more address translations must be applied.
+
+Once all scalar and vector indices have been rebuilt past a given reuse version, that version is no
+longer needed and can be trimmed. Users should schedule a periodic process to trim stale reuse
+versions and keep the FRI size under control.
\ No newline at end of file
diff --git a/docs/src/format/table/index/system/mem_wal.md b/docs/src/format/table/index/system/mem_wal.md
index f9169bcfb76..a93debfe55c 100644
--- a/docs/src/format/table/index/system/mem_wal.md
+++ b/docs/src/format/table/index/system/mem_wal.md
@@ -1,7 +1,7 @@
 # MemWAL Index
 
 The MemWAL Index is a system index that serves as the centralized structure for all MemWAL metadata.
-It stores configuration (region specs, indexes to maintain), merge progress, and region state snapshots.
+It stores configuration (shard specs, indexes to maintain), merge progress, and shard state snapshots.
 
 A table has at most one MemWAL index.
 
diff --git a/docs/src/format/table/mem_wal.md b/docs/src/format/table/mem_wal.md
index 5e6907038fb..6210ecd7da4 100644
--- a/docs/src/format/table/mem_wal.md
+++ b/docs/src/format/table/mem_wal.md
@@ -10,63 +10,63 @@ scan, point lookup, vector search and full-text search.
 A Lance table is called a **base table** under the context of the MemWAL spec.
 It must have an [unenforced primary key](index.md#unenforced-primary-key) defined in the table schema.
 
-On top of the base table, the MemWAL spec defines a set of regions.
-Writers write to regions, and data in each region is merged into the base table asynchronously.
-An index is kept in the base table for readers to quickly discover the state of all regions at a point of time.
+On top of the base table, the MemWAL spec defines a set of shards.
+Writers write to shards, and data in each shard is merged into the base table asynchronously.
+An index is kept in the base table for readers to quickly discover the state of all shards at a point of time.
 
-### MemWAL Region
+### MemWAL Shard
 
-A **MemWAL Region** is the main unit to horizontally scale out writes.
+A **MemWAL Shard** is the main unit to horizontally scale out writes.
 
-Each region has exactly one active writer at any time.
-Writers claim a region and then write data to that region.
-Data in each region is expected to be merged into the base table asynchronously.
+Each shard has exactly one active writer at any time.
+Writers claim a shard and then write data to that shard.
+Data in each shard is expected to be merged into the base table asynchronously.
 
-Rows of the same primary key must be written to one and only one region.
-If two regions contain rows with the same primary key, the following scenario can cause data corruption:
+Rows of the same primary key must be written to one and only one shard.
+If two shards contain rows with the same primary key, the following scenario can cause data corruption:
 
-1. Region A receives a write with primary key `pk=1` at time T1
-2. Region B receives a write with primary key `pk=1` at time T2 (T2 > T1)
-3. The row in region B is merged into the base table first
-4. The row in region A is merged into the base table second
-5. The row from Region A (older) now overwrites the row from Region B (newer)
+1. Shard A receives a write with primary key `pk=1` at time T1
+2. Shard B receives a write with primary key `pk=1` at time T2 (T2 > T1)
+3. The row in shard B is merged into the base table first
+4. The row in shard A is merged into the base table second
+5. The row from Shard A (older) now overwrites the row from Shard B (newer)
 
 This violates the expected "last write wins" semantics.
-By ensuring each primary key is assigned to exactly one region via the region spec,
-merge order between regions becomes irrelevant for correctness.
+By ensuring each primary key is assigned to exactly one shard via the shard spec,
+merge order between shards becomes irrelevant for correctness.
 
-See [MemWAL Region Architecture](#region-architecture) for the complete region architecture.
+See [MemWAL Shard Architecture](#shard-architecture) for the complete shard architecture.
 
 ### MemWAL Index
 
 A **MemWAL Index** is the centralized structure for all MemWAL metadata on top of a base table.
 A table has at most one MemWAL index. It stores:
 
-- **Configuration**: Region specs defining how rows map to regions, and which indexes to maintain
-- **Merge progress**: Last generation merged to base table for each region
+- **Configuration**: Shard specs defining how rows map to shards, and which indexes to maintain
+- **Merge progress**: Last generation merged to base table for each shard
 - **Index catchup progress**: Which merged generation each base table index has been rebuilt to cover
-- **Region snapshots**: Snapshot of all region states for read optimization
+- **Shard snapshots**: Snapshot of all shard states for read optimization
 
 The index is the source of truth for **configuration**, **merge progress** and **index catchup progress**
 Writers and mergers read the MemWAL index to get these configurations before writing.
 
-Each [region's manifest](#region-manifest) is authoritative for its own state.
-Readers use **region snapshots** is a read-only optimization to see a point-in-time view of all regions without the need to open each region manifest.
+Each [shard's manifest](#shard-manifest) is authoritative for its own state.
+Readers use **shard snapshots** is a read-only optimization to see a point-in-time view of all shards without the need to open each shard manifest.
 
 See [MemWAL Index Details](#memwal-index-details) for the complete structure.
 
-## Region Architecture
+## Shard Architecture
 
-![Region Architecture](../../images/mem_wal_regional.png)
+![Shard Architecture](../../images/mem_wal_regional.png)
 
-Within a region, writes are stored in an **in-memory table (MemTable)**.
-It is also written to the region's **Write-Ahead Log (WAL)** for durability guarantee.
+Within a shard, writes are stored in an **in-memory table (MemTable)**.
+It is also written to the shard's **Write-Ahead Log (WAL)** for durability guarantee.
 The MemTable is periodically **flushed** to storage based on memory pressure and other conditions.
 **Flushed MemTables** in storage are then asynchronously **merged** into the base table.
 
 ### MemTable
 
-A MemTable holds rows inserted into the region before flushing to storage.
+A MemTable holds rows inserted into the shard before flushing to storage.
 It serves 2 purposes:
 
 1. build up data and related indexes to be flushed to storage as a flushed MemTable
@@ -93,7 +93,7 @@ When MemTable of generation `N` is discarded, the next MemTable gets assigned ge
 
 ### WAL
 
-WAL serves as the durable storage of all MemTables in a region.
+WAL serves as the durable storage of all MemTables in a shard.
 It consists of data in MemTables ordered by generation.
 Every time we write to the WAL, we call it a **WAL Flush**.
 
@@ -121,7 +121,7 @@ If WAL entry `N` is not flushed fully, WAL entry `N+1` must not exist in storage
 **Replaying** a WAL means to read data in the WAL from a lower to a higher position.
 This is commonly used to recover the latest MemTable after it is lost,
 by reading from the start position of the latest MemTable generation till the highest position in the WAL,
-assuming proper fencing to guard against multiple writers to the same region.
+assuming proper fencing to guard against multiple writers to the same shard.
 
 See [Writer Fencing](#writer-fencing) for the full fencing mechanism.
 
@@ -132,7 +132,7 @@ The writer epoch is stored in the stream's Arrow schema metadata with key `write
 
 #### WAL Storage Layout
 
-Each WAL entry is stored within the WAL directory of the region located at `_mem_wal/{region_id}/wal`.
+Each WAL entry is stored within the WAL directory of the shard located at `_mem_wal/{shard_id}/wal`.
 
 WAL files use bit-reversed 64-bit binary naming to distribute files evenly across the directory keyspace.
 This optimizes S3 throughput by spreading sequential writes across S3's internal partitions, minimizing throttling.
@@ -164,11 +164,11 @@ lower numbers represent older data and must be merged to the base table first to
 Within a single flushed MemTable, if there are multiple rows of the same primary key,
 the row that is last inserted wins.
 
-### Region Manifest
+### Shard Manifest
 
-Each region has a manifest file. This is the source of truth for the state of a region.
+Each shard has a manifest file. This is the source of truth for the state of a shard.
 
-#### Region Manifest Contents
+#### Shard Manifest Contents
 
 The manifest contains:
 
@@ -177,21 +177,21 @@ The manifest contains:
 - **Generation trackers**: `current_generation` (next generation to flush), `flushed_generations` list of generation number and directory path pairs (e.g., generation 1 at `a1b2c3d4_gen_1`)
 
 Note: `wal_entry_position_last_seen` is a hint that may be stale since it's not updated on WAL write.
-It is updated opportunistically by any reader that can update the region manifest.
+It is updated opportunistically by any reader that can update the shard manifest.
 The manifest itself is atomically written, but recovery must try to get newer WAL files to find the actual state beyond this hint.
 
-The manifest is serialized as a protobuf binary file using the `RegionManifest` message.
+The manifest is serialized as a protobuf binary file using the `ShardManifest` message.
 
 <details>
-<summary>RegionManifest protobuf message</summary>
+<summary>ShardManifest protobuf message</summary>
 
 ```protobuf
-%%% mem_wal.message.RegionManifest %%%
+%%% mem_wal.message.ShardManifest %%%
 ```
 
 </details>
 
-#### Region Manifest Versioning
+#### Shard Manifest Versioning
 
 Manifests are versioned starting from 1 and immutable.
 Each update creates a new manifest file at the next version number.
@@ -212,13 +212,13 @@ To read the latest manifest version:
 4. The latest version is the last found version
 
 !!!note
-This works because the write rate to region manifests is significantly lower than read rates. Region manifests are only updated when region metadata changes (MemTable flush), not on every write. This ensures HEAD requests will eventually terminate and find the latest version.
+This works because the write rate to shard manifests is significantly lower than read rates. Shard manifests are only updated when shard metadata changes (MemTable flush), not on every write. This ensures HEAD requests will eventually terminate and find the latest version.
 
-#### Region Manifest Storage Layout
+#### Shard Manifest Storage Layout
 
-All region manifest versions are stored in `_mem_wal/{region_id}/manifest` directory.
+All shard manifest versions are stored in `_mem_wal/{shard_id}/manifest` directory.
 
-Each region manifest version file uses bit-reversed 64-bit binary naming, the same scheme as WAL files.
+Each shard manifest version file uses bit-reversed 64-bit binary naming, the same scheme as WAL files.
 For example, version 5 becomes `1010000000000000000000000000000000000000000000000000000000000000.binpb`.
 
 ## MemWAL Index Details
@@ -228,24 +228,24 @@ The MemWAL Index uses the [standard index storage](index/index.md#index-storage)
 The index stores its data in two parts:
 
 1. **Index details** (`index_details` in `IndexMetadata`): Contains configuration, merge progress, and snapshot metadata
-2. **Region snapshots**: Stored as a Lance file or inline, depending on region count
+2. **Shard snapshots**: Stored as a Lance file or inline, depending on shard count
 
 ### Index Details
 
 The `index_details` field in `IndexMetadata` contains a `MemWalIndexDetails` protobuf message with the following key fields:
 
-- **Configuration fields** (`region_specs`, `maintained_indexes`) are the source of truth for MemWAL configuration.
+- **Configuration fields** (`shard_specs`, `maintained_indexes`) are the source of truth for MemWAL configuration.
   Writers read these fields to determine how to partition data and which indexes to maintain.
-- **Merge progress** (`merged_generations`) tracks the last generation merged to the base table for each region.
+- **Merge progress** (`merged_generations`) tracks the last generation merged to the base table for each shard.
   This field is updated atomically with merge-insert data commits, enabling conflict resolution when multiple mergers operate concurrently.
-  Each entry contains the region UUID and generation number.
+  Each entry contains the shard UUID and generation number.
 - **Index catchup progress** (`index_catchup`) tracks which merged generation each base table index has been rebuilt to cover.
   When data is merged from a flushed MemTable to the base table, the base table's indexes may be rebuilt asynchronously.
   During this window, queries should use the flushed MemTable's pre-built indexes instead of scanning unindexed data in the base table.
   See [Indexed Read Plan](#indexed-read-plan) for details.
-- **Region snapshot fields** (`snapshot_ts_millis`, `num_regions`, `inline_snapshots`) provide a snapshot of region states.
-  The actual region manifests remain authoritative for region state.
-  When `num_regions` is 0, the `inline_snapshots` field may be `None` or an empty Lance file with 0 rows but proper schema.
+- **Shard snapshot fields** (`snapshot_ts_millis`, `num_shards`, `inline_snapshots`) provide a snapshot of shard states.
+  The actual shard manifests remain authoritative for shard state.
+  When `num_shards` is 0, the `inline_snapshots` field may be `None` or an empty Lance file with 0 rows but proper schema.
 
 <details>
 <summary>MemWalIndexDetails protobuf message</summary>
@@ -256,55 +256,55 @@ The `index_details` field in `IndexMetadata` contains a `MemWalIndexDetails` pro
 
 </details>
 
-### Region Identifier
+### Shard Identifier
 
-Each region has a unique identifier across all regions following UUID v4 standard.
-When a new region is created, it is assigned a new identifier.
+Each shard has a unique identifier across all shards following UUID v4 standard.
+When a new shard is created, it is assigned a new identifier.
 
-### Region Spec
+### Shard Spec
 
-A **Region Spec** defines how all rows in a table are logically divided into different regions,
-enabling automatic region assignment and query-time region pruning.
+A **Shard Spec** defines how all rows in a table are logically divided into different shards,
+enabling automatic shard assignment and query-time shard pruning.
 
-Each region spec has:
+Each shard spec has:
 
 - **Spec ID**: A positive integer that uniquely identifies this spec within the MemWAL index. IDs are never reused.
-- **Region fields**: An array of field definitions that determine how to compute region values.
+- **Shard fields**: An array of field definitions that determine how to compute shard values.
 
-Each region is bound to a specific region spec ID, recorded in its [manifest](#region-manifest).
-Regions without a spec ID (`spec_id = 0`) are manually-created regions not governed by any spec.
+Each shard is bound to a specific shard spec ID, recorded in its [manifest](#shard-manifest).
+Shards without a spec ID (`spec_id = 0`) are manually-created shards not governed by any spec.
 
-A region spec's field array consists of **region field** definitions.
-Each region field has the following properties:
+A shard spec's field array consists of **shard field** definitions.
+Each shard field has the following properties:
 
 | Property      | Description                                                               |
 | ------------- | ------------------------------------------------------------------------- |
-| `field_id`    | Unique string identifier for this region field                            |
+| `field_id`    | Unique string identifier for this shard field                            |
 | `source_ids`  | Array of field IDs referencing source columns in the schema               |
-| `transform`   | A well-known region expression, specify this or `expression`              |
+| `transform`   | A well-known shard expression, specify this or `expression`              |
 | `expression`  | A DataFusion SQL expression for custom logic, specify this or `transform` |
-| `result_type` | The output type of the region value                                       |
+| `result_type` | The output type of the shard value                                       |
 
-#### Region Expression
+#### Shard Expression
 
-A **Region Expression** is a [DataFusion SQL expression](https://datafusion.apache.org/user-guide/sql/index.html) that derives a region value from source column(s).
+A **Shard Expression** is a [DataFusion SQL expression](https://datafusion.apache.org/user-guide/sql/index.html) that derives a shard value from source column(s).
 Source columns are referenced as `col0`, `col1`, etc., corresponding to the order of field IDs in `source_ids`.
 
-Region expressions must satisfy the following requirements:
+Shard expressions must satisfy the following requirements:
 
 1. **Deterministic**: The same input value must always produce the same output value.
 2. **Stateless**: The expression must not depend on external state (e.g., current time, random values, session variables).
-3. **Type-promotion resistant**: The expression must produce the same result for equivalent values regardless of their numeric type (e.g., `int32(5)` and `int64(5)` must yield the same region value).
+3. **Type-promotion resistant**: The expression must produce the same result for equivalent values regardless of their numeric type (e.g., `int32(5)` and `int64(5)` must yield the same shard value).
 4. **Column removal resistant**: If a source field ID is not found in the schema, the column should be interpreted as NULL.
 5. **NULL-safe**: The expression should properly handle NULL inputs and have defined behavior (e.g., return NULL if input is NULL for single-column expressions).
 6. **Consistent with result type**: The expression's return type must be consistent with `result_type` in non-NULL cases.
 
-#### Region Transform
+#### Shard Transform
 
-A **Region Transform** is a well-known region expression with a predefined name.
+A **Shard Transform** is a well-known shard expression with a predefined name.
 When a transform is specified, the expression is derived automatically.
 
-| Transform      | Parameters    | Region Expression                                         | Result Type    |
+| Transform      | Parameters    | Shard Expression                                         | Result Type    |
 | -------------- | ------------- | --------------------------------------------------------- | -------------- |
 | `identity`     | (none)        | `col0`                                                    | same as source |
 | `year`         | (none)        | `date_part('year', col0)`                                 | `int32`        |
@@ -322,48 +322,48 @@ The `bucket` and `multi_bucket` transforms use Murmur3 hash functions:
 
 The hash result is wrapped with `abs()` and modulo `N` to produce a non-negative bucket number in the range `[0, N)`.
 
-### Region Snapshot Storage
+### Shard Snapshot Storage
 
-Region snapshots are stored using one of two strategies based on the number of regions:
+Shard snapshots are stored using one of two strategies based on the number of shards:
 
-| Region Count       | Storage Strategy    | Location                                  |
+| Shard Count       | Storage Strategy    | Location                                  |
 | ------------------ | ------------------- | ----------------------------------------- |
 | <= 100 (threshold) | Inline              | `inline_snapshots` field in index details |
 | > 100              | External Lance file | `_indices/{UUID}/index.lance`             |
 
-The threshold (100 regions) is implementation-defined and may vary.
+The threshold (100 shards) is implementation-defined and may vary.
 
-**Inline storage**: For small region counts, snapshots are serialized as a Lance file and stored in the `inline_snapshots` field.
+**Inline storage**: For small shard counts, snapshots are serialized as a Lance file and stored in the `inline_snapshots` field.
 This keeps the index metadata compact while avoiding an additional file read for common cases.
 
-**External Lance file**: For large region counts, snapshots are stored as a Lance file at `_indices/{UUID}/index.lance`.
-This file uses standard Lance format with the region snapshot schema, enabling efficient columnar access and compression.
+**External Lance file**: For large shard counts, snapshots are stored as a Lance file at `_indices/{UUID}/index.lance`.
+This file uses standard Lance format with the shard snapshot schema, enabling efficient columnar access and compression.
 
-### Region Snapshot Arrow Schema
+### Shard Snapshot Arrow Schema
 
-Region snapshots are stored as a Lance file with one row per region.
-The schema has one column per `RegionManifest` field plus region spec columns:
+Shard snapshots are stored as a Lance file with one row per shard.
+The schema has one column per `ShardManifest` field plus shard spec columns:
 
 | Column                            | Type                                             | Description                                              |
 | --------------------------------- | ------------------------------------------------ | -------------------------------------------------------- |
-| `region_id`                       | `fixed_size_binary(16)`                          | Region UUID bytes                                        |
-| `version`                         | `uint64`                                         | Region manifest version                                  |
-| `region_spec_id`                  | `uint32`                                         | Region spec ID (0 if manual)                             |
+| `shard_id`                       | `fixed_size_binary(16)`                          | Shard UUID bytes                                        |
+| `version`                         | `uint64`                                         | Shard manifest version                                  |
+| `shard_spec_id`                  | `uint32`                                         | Shard spec ID (0 if manual)                             |
 | `writer_epoch`                    | `uint64`                                         | Writer fencing token                                     |
 | `replay_after_wal_entry_position` | `uint64`                                         | Last WAL entry position (0-based) flushed to MemTable    |
 | `wal_entry_position_last_seen`    | `uint64`                                         | Last WAL entry position (0-based) seen (hint)            |
 | `current_generation`              | `uint64`                                         | Next generation to flush                                 |
 | `flushed_generations`             | `list<struct<generation: uint64, path: string>>` | Flushed MemTable paths                                   |
-| `region_field_{field_id}`         | varies                                           | Region field value (one column per field in region spec) |
+| `region_field_{field_id}`         | varies                                           | Shard field value (one column per field in shard spec) |
 
-For example, with a region spec containing a field `user_bucket` of type `int32`:
+For example, with a shard spec containing a field `user_bucket` of type `int32`:
 
 | Column                     | Type    | Description                  |
 | -------------------------- | ------- | ---------------------------- |
 | ...                        | ...     | (base columns above)         |
-| `region_field_user_bucket` | `int32` | Bucket value for this region |
+| `region_field_user_bucket` | `int32` | Bucket value for this shard |
 
-This schema directly corresponds to the fields in the `RegionManifest` protobuf message plus the computed region field values.
+This schema directly corresponds to the fields in the `ShardManifest` protobuf message plus the computed shard field values.
 
 ## Storage Layout
 
@@ -373,12 +373,12 @@ Here is a recap of the storage layout with all the files and concepts defined so
 {table_path}/
 ├── _indices/
 │   └── {index_uuid}/                    # MemWAL Index (uses standard index storage)
-│       └── index.lance                  # Serialized region snapshots (Lance file)
+│       └── index.lance                  # Serialized shard snapshots (Lance file)
 │
 └── _mem_wal/
-    └── {region_uuid}/                   # Region directory (UUID v4)
+    └── {region_uuid}/                   # Shard directory (UUID v4)
         ├── manifest/
-        │   ├── {bit_reversed_version}.binpb     # Serialized region manifest (bit-reversed naming)
+        │   ├── {bit_reversed_version}.binpb     # Serialized shard manifest (bit-reversed naming)
         │   └── version_hint.json                # Version hint file
         ├── wal/
         │   ├── {bit_reversed_entry_id}.lance    # WAL data files (bit-reversed naming)
@@ -398,7 +398,7 @@ This specification describes the storage layout for the LSM tree architecture. I
 
 The specification defines:
 
-- **Storage layout**: The directory structure, file formats, and naming conventions for WAL entries, flushed MemTables, region manifests, and the MemWAL index
+- **Storage layout**: The directory structure, file formats, and naming conventions for WAL entries, flushed MemTables, shard manifests, and the MemWAL index
 - **Durability guarantees**: How data is persisted through WAL entries and flushed MemTables
 - **Consistency model**: How readers and writers coordinate through manifests and epoch-based fencing
 
@@ -413,19 +413,19 @@ As long as the storage layout is correct and the documented invariants are maint
 
 ## Writer Expectations
 
-A writer operates on a single region and is responsible for:
+A writer operates on a single shard and is responsible for:
 
-1. Claiming the region using epoch-based fencing
+1. Claiming the shard using epoch-based fencing
 2. Writing data to WAL entries and flushed MemTables following the [storage layout](#storage-layout)
-3. Maintaining the region manifest to track WAL and generation progress
+3. Maintaining the shard manifest to track WAL and generation progress
 
 ### Writer Fencing
 
-Writers use epoch-based fencing to ensure single-writer semantics per region.
+Writers use epoch-based fencing to ensure single-writer semantics per shard.
 
-To claim a region:
+To claim a shard:
 
-1. Load the latest region manifest
+1. Load the latest shard manifest
 2. Increment `writer_epoch` by one
 3. Atomically write a new manifest version
 4. If the write fails (another writer claimed the epoch), reload and retry with a higher epoch
@@ -443,20 +443,20 @@ Background jobs handle merging flushed MemTables to the base table and garbage c
 
 ### MemTable Merger
 
-Flushed MemTables must be merged to the base table in **ascending generation order** within each region. This ordering is essential for correct upsert semantics: newer generations must overwrite older ones.
+Flushed MemTables must be merged to the base table in **ascending generation order** within each shard. This ordering is essential for correct upsert semantics: newer generations must overwrite older ones.
 
 The merge uses Lance's merge-insert operation with atomic transaction semantics:
 
-- `merged_generations[region_id]` is updated atomically with the data commit
+- `merged_generations[shard_id]` is updated atomically with the data commit
 - On commit conflict, check the conflicting commit's `merged_generations` to determine if the generation was already merged
 
 For a concrete example, see [Appendix 2: Concurrent Merger Example](#appendix-2-concurrent-merger-example).
 
 ### Garbage Collector
 
-The garbage collector removes obsolete data from region directories. Flushed MemTables and their referenced WAL files may be deleted after:
+The garbage collector removes obsolete data from shard directories. Flushed MemTables and their referenced WAL files may be deleted after:
 
-1. The generation has been merged to the base table (`generation <= merged_generations[region_id]`)
+1. The generation has been merged to the base table (`generation <= merged_generations[shard_id]`)
 2. All maintained indexes have caught up (`generation <= min(index_catchup[I].caught_up_generation)`)
 3. No retained base table version references the generation for time travel
 
@@ -486,9 +486,9 @@ Without proper merging, queries would return duplicate or stale rows.
 Reader consistency depends on two factors:
 
 1. access to in-memory MemTables
-2. the source of region metadata (either through MemWAL index or region manifests)
+2. the source of shard metadata (either through MemWAL index or shard manifests)
 
-Strong consistency requires access to in-memory MemTables for all regions involved in the query and reading region manifests directly.
+Strong consistency requires access to in-memory MemTables for all shards involved in the query and reading shard manifests directly.
 Otherwise, the query is eventually consistent due to missing unflushed data or stale MemWAL Index snapshots.
 
 !!!note
@@ -510,30 +510,30 @@ Datasets come from:
 3. optionally in-memory MemTables (if accessible).
 
 Each dataset is tagged with a generation number: 0 for the base table, and positive integers for MemTable generations.
-Within a region, the generation number determines data freshness, with higher numbers representing newer data.
-Rows from different regions do not need deduplication since each primary key maps to exactly one region.
+Within a shard, the generation number determines data freshness, with higher numbers representing newer data.
+Rows from different shards do not need deduplication since each primary key maps to exactly one shard.
 
 The planner also collects bloom filters from each generation for staleness detection during search queries.
 
-#### Region Pruning
+#### Shard Pruning
 
-Before executing queries, if region spec is available,
-the planner evaluates filter predicates against region specs to determine which regions may contain matching data.
-This pruning step reduces the number of regions to scan.
+Before executing queries, if shard spec is available,
+the planner evaluates filter predicates against shard specs to determine which shards may contain matching data.
+This pruning step reduces the number of shards to scan.
 
 For each filter predicate:
 
-1. Extract predicates on columns used in region specs
-2. Evaluate which region values can satisfy the predicate
-3. Prune regions whose values cannot match
+1. Extract predicates on columns used in shard specs
+2. Evaluate which shard values can satisfy the predicate
+3. Prune shards whose values cannot match
 
-For example, with a region spec using `bucket(user_id, 10)` and a filter `user_id = 123`:
+For example, with a shard spec using `bucket(user_id, 10)` and a filter `user_id = 123`:
 
 1. Compute `bucket(123, 10) = 3`
-2. Only scan regions with bucket value 3
-3. Skip all other regions
+2. Only scan shards with bucket value 3
+3. Skip all other shards
 
-Region pruning applies to both scan queries and prefilters in search queries.
+Shard pruning applies to both scan queries and prefilters in search queries.
 
 #### Indexed Read Plan
 
@@ -553,12 +553,12 @@ This ensures all reads remain indexed regardless of how far behind the async ind
 
 ### Appendix 1: Writer Fencing Example
 
-This example demonstrates how epoch-based fencing prevents data corruption when two writers compete for the same region.
+This example demonstrates how epoch-based fencing prevents data corruption when two writers compete for the same shard.
 
 #### Initial State
 
 ```
-Region manifest (version 1):
+Shard manifest (version 1):
   writer_epoch: 5
   replay_after_wal_entry_position: 10
   wal_entry_position_last_seen: 12
@@ -611,9 +611,9 @@ This example demonstrates how MemWAL Index and conflict resolution handle concur
 
 ```
 MemWAL Index:
-  merged_generations: {region: 5}
+  merged_generations: {shard: 5}
 
-Region manifest (version 1):
+Shard manifest (version 1):
   current_generation: 8
   flushed_generations: [(6, "abc123_gen_6"), (7, "def456_gen_7")]
 ```
@@ -625,10 +625,10 @@ Two mergers both try to merge generation 6 concurrently.
 | Step | Merger A                  | Merger B                       | MemWAL Index     |
 | ---- | ------------------------- | ------------------------------ | ---------------- |
 | 1    | Reads index: merged_gen=5 |                                | merged_gen=5     |
-| 2    | Reads region manifest     |                                |                  |
+| 2    | Reads shard manifest     |                                |                  |
 | 3    | Starts merging gen 6      |                                |                  |
 | 4    |                           | Reads index: merged_gen=5      | merged_gen=5     |
-| 5    |                           | Reads region manifest          |                  |
+| 5    |                           | Reads shard manifest          |                  |
 | 6    |                           | Starts merging gen 6           |                  |
 | 7    | Commits (merged_gen=6)    |                                | **merged_gen=6** |
 | 8    |                           | Tries to commit                |                  |
@@ -648,7 +648,7 @@ Merger A crashes after committing to the table.
 | 2    | Merges gen 6, commits     |                                  | **merged_gen=6** |
 | 3    | **CRASH**                 |                                  | merged_gen=6     |
 | 4    |                           | Reads index: merged_gen=6        | merged_gen=6     |
-| 5    |                           | Reads region manifest            |                  |
+| 5    |                           | Reads shard manifest            |                  |
 | 6    |                           | **Skips gen 6** (already merged) |                  |
 | 7    |                           | Merges gen 7, commits            | **merged_gen=7** |
 
diff --git a/docs/src/format/table/transaction.md b/docs/src/format/table/transaction.md
index 56b867a4683..d1a5191bf54 100644
--- a/docs/src/format/table/transaction.md
+++ b/docs/src/format/table/transaction.md
@@ -52,6 +52,12 @@ The authoritative specification for transaction types is defined in [`protos/tra
 Each transaction contains a `read_version` field indicating the table version from which the transaction was built,
 a `uuid` field uniquely identifying the transaction, and an `operation` field specifying one of the following transaction types:
 
+In the following section, we will describe each transaction type and its compatibility with other transaction types. This
+compatibility is not always bi-directional. We are describing it from the perspective of the operation being committed. For example, we say that an Append is not compatible with an Overwrite which means that if we are trying to commit an Append, and an
+Overwrite has already been committed (since we started the Append), then the Append will fail. On the other hand, when describing the
+Overwrite operation, we say that it does not conflict with Append. This is because, if we are trying to commit an Overwrite, and an
+Append operation has occurred in the meantime, we still allow the Overwrite to proceed.
+
 ### Append
 
 Adds new fragments to the table without modifying existing data.
@@ -66,6 +72,16 @@ Fragment IDs are not assigned at transaction creation time; they are assigned du
 
 </details>
 
+#### Append Compatibility
+
+The append operation is one of the most common operations and is designed to be compatible with most other operations, even
+itself. This is to ensure that multiple writers can append without worry about conflicts. These are the operations
+that conflict with append:
+
+- Overwrite
+- Restore
+- UpdateMemWalState
+
 ### Delete
 
 Marks rows as deleted using deletion vectors.
@@ -81,6 +97,30 @@ The `predicate` field stores the deletion condition, enabling conflict detection
 
 </details>
 
+#### Delete Compatibility
+
+Delete modifies an existing fragment, so there may be conflicts with other operations on overlapping fragments.
+Generally these conflicts are rebaseable or retryable.
+
+These are the operations that conflict with delete:
+
+- Overwrite
+- Restore
+- UpdateMemWalState
+
+These operations conflict with delete but can be retried:
+
+- Merge (only if there are overlapping fragments)
+- Rewrite (only if there are overlapping fragments)
+- DataReplacement (only if there are overlapping fragments)
+
+These operations conflict with delete but can potentially be rebased. The deletion
+masks from the two operations will be merged. However, if both operations modified
+the same rows, then the conflict becomes a retryable conflict.
+
+- Delete
+- Update
+
 ### Overwrite
 
 Creates or completely overwrites the table with new data, schema, and configuration.
@@ -94,6 +134,18 @@ Creates or completely overwrites the table with new data, schema, and configurat
 
 </details>
 
+#### Overwrite Compatibility
+
+An overwrite operation completely overwrites the table. Generally, we do not care what has happened since
+the read version.
+
+However, the overwrite does not necessarily rewrite the table config. As a result, we consider the following
+to be retryable conflicts:
+
+- UpdateConfig (only if the two operations modify the same config key)
+- Overwrite (always)
+- UpdateMemWalState (always)
+
 ### CreateIndex
 
 Adds, replaces, or removes secondary indices (vector indices, scalar indices, full-text search indices).
@@ -107,6 +159,40 @@ Adds, replaces, or removes secondary indices (vector indices, scalar indices, fu
 
 </details>
 
+#### CreateIndex Compatibility
+
+Indexes record which fragments are covered by the index and we don't require all fragments be covered. As a result, it
+is typically ok for an index to be created concurrently with the addition of new fragments. These new fragments will simply
+be unindexed.
+
+Updates and deletes are also compatible with index creation. This is because it is ok for an index to refer to deleted rows.
+Those results will be filtered out after the index search. If an update occurs then the old value will be filtered out and the
+new value will be considered part of the unindexed set.
+
+If two CreateIndex operations are committed concurrently then it is allowed. If the indexes have different names this is no
+problem. If the indexes have the same name then the second operation will win and replace the first.
+
+These operations conflict with index creation:
+
+- Overwrite
+- Restore
+- UpdateMemWalState
+
+Data replacement operations will conflict with index creation if the column being replaced is being indexed. Rewrite operations
+will conflict with index creation if the rewritten fragments are covered by the index. This is because an index refers to row
+addresses and the rewrite operation changes the row addresses. However, if a fragment reuse index is being used, or if the stable
+row ids feature is enable, then the rewrite operation is compatible with index creation. As a result, these are the operations
+that are retryable conflicts with index creation:
+
+- Rewrite (only if overlapping fragments, no stable row ids, and no fragment reuse index)
+- DataReplacement (only if overlapping fragments and the column being replaced is being indexed)
+
+Some indices are special singleton indices. For example, the fragment reuse index and the mem wal index. If a conflict occurs
+between two operations that are modifying the same singleton index, then we must rebase the operation and merge the indexes.
+As a result, these are the operations that are rebaseable conflicts with index creation:
+
+- CreateIndex (only if both operations are modifying the same singleton index)
+
 ### Rewrite
 
 Reorganizes data without semantic modification.
@@ -123,6 +209,38 @@ New fragment IDs must be reserved via `ReserveFragments` before executing a `Rew
 
 </details>
 
+#### Rewrite Compatibility
+
+Rewrite operations do not change data but they can materialize deletions and they do replace fragments. As a result,
+they can potentially conflict with other operations that modify the fragments being rewritten.
+
+These are the operations that conflict with rewrite:
+
+- Overwrite
+- Restore
+
+Rewrite is not compatible with CreateIndex by default because the operation will change the row addresses that the CreateIndex
+refers to. However, a fragment reuse index or the stable row ids feature can allow these operations to be compatible.
+
+Several operations modify existing fragments. As a result, they can potentially conflict with Rewrite if they modify
+the same fragments. However, Merge is [overly general](#overly-general-operation) and so no conflict detection is possible.
+As a result, here are the operations that are retryable conflicts with Rewrite:
+
+- Merge (always)
+- DataReplacement (only if overlapping fragments)
+- Delete (only if overlapping fragments)
+- Update (only if overlapping fragments)
+- Rewrite (if overlapping fragments or both carry a fragment reuse index)
+- CreateIndex (overlapping fragments and no fragment reuse index or stable row ids)
+
+There is one case where a Rewrite will rebase. This is when the Rewrite operation has a fragment reuse index and there is
+a CreateIndex operation that is writing the fragment reuse index. In this case the Rewrite will rebase and update its
+fragment reuse index to include the conflicting fragment reuse index.
+
+As a result, these are the operations that are rebaseable conflicts with Rewrite:
+
+- CreateIndex (if the CreateIndex is writing the fragment reuse index and the Rewrite is carrying a fragment reuse index)
+
 ### Merge
 
 Adds new columns to the table, modifying the schema.
@@ -137,6 +255,31 @@ All fragments must be updated to include the new columns.
 
 </details>
 
+#### Overly General Operation
+
+The Merge operation is a very generic operation. The set of fragments provided in the operation will be the final set of
+fragments in the resulting dataset. As a result, it has a high potential for conflicts with other operations. If possible,
+more restrictive operations such as Rewrite, DataReplacement, or Append should be preferred over Merge.
+
+#### Merge Compatibility
+
+As mentioned above, Merge is a very generic operation, as a result it has a high potential for conflicts with other operations.
+The following operations conflict with Merge:
+
+- Overwrite
+- Restore
+- UpdateMemWalState
+- Project
+
+These operations are retryable conflicts with Merge:
+
+- Update (always)
+- Append (always)
+- Delete (always)
+- Merge (always)
+- Rewrite (always)
+- DataReplacement (always)
+
 ### Project
 
 Removes columns from the table, modifying the schema.
@@ -151,6 +294,23 @@ This is a metadata-only operation; data files are not modified.
 
 </details>
 
+#### Project Compatibility
+
+Since project only modifies the schema, it is compatible with most other operations. However, it is not compatible with Merge
+because the Merge operation modifies the schema (can potentially add columns) and the logic to rebase those changes does not
+currently exist (project is cheap and easy enough to retry).
+
+These are the operations that conflict with Project:
+
+- Overwrite
+- Restore
+- UpdateMemWalState
+
+The following operations are retryable conflicts with Project:
+
+- Project (always)
+- Merge (always)
+
 ### Restore
 
 Reverts the table to a previous version.
@@ -164,6 +324,13 @@ Reverts the table to a previous version.
 
 </details>
 
+#### Restore Compatibility
+
+The Restore operation reverts the table to a previous version. It's generally assumed this trumps any
+other operation. Here are the operations that conflict with Restore:
+
+- UpdateMemWalState
+
 ### ReserveFragments
 
 Pre-allocates fragment IDs for use in future `Rewrite` operations.
@@ -178,6 +345,14 @@ This allows rewrite operations to reference fragment IDs before the rewrite tran
 
 </details>
 
+#### ReserveFragments Compatibility
+
+The ReserveFragments operation is fairly trivial. The only thing it changes is the max fragment id. So this
+only conflicts with operations that modify the max fragment id. Here are the operations that conflict with ReserveFragments:
+
+- Overwrite
+- Restore
+
 ### Clone
 
 Creates a shallow or deep copy of the table.
@@ -193,6 +368,11 @@ Deep clones are full copies using object storage native copy operations (e.g., S
 
 </details>
 
+#### Clone Compatibility
+
+The Clone operation can only be the first operation in a dataset. If there is an existing dataset, then the Clone operation will fail.
+As a result, there is no such thing as a conflict with Clone.
+
 ### Update
 
 Modifies row values without adding or removing rows.
@@ -207,6 +387,27 @@ Supports two execution modes: REWRITE_ROWS deletes rows in current fragments and
 
 </details>
 
+#### Update Compatibility
+
+Here are the operations that conflict with Update:
+
+- Overwrite
+- Restore
+
+An update operation is both a delete and an append operation. Like a Delete operation, it will modify fragments to change
+the deletion mask. As a result, there will be a retryable conflict with other operations that modify the same fragments.
+Here are the operations that are retryable conflicts with Update:
+
+- Rewrite (only if overlapping fragments)
+- DataReplacement (only if overlapping fragments)
+- Merge (always)
+
+Similar to Delete, the Update operation can rebase other modifications to the deletion mask. Here are the operations that
+are rebaseable conflicts with Update:
+
+- Delete
+- Update
+
 ### UpdateConfig
 
 Modifies table configuration, table metadata, schema metadata, or field metadata without changing data.
@@ -220,6 +421,14 @@ Modifies table configuration, table metadata, schema metadata, or field metadata
 
 </details>
 
+#### UpdateConfig Compatibility
+
+An UpdateConfig operation only modifies table config and tends to be compatible with other operations. Here
+are the operations that conflict with UpdateConfig:
+
+- Overwrite
+- UpdateConfig (only if the two operations modify the same config)
+
 ### DataReplacement
 
 Replaces data in specific column regions with new data files.
@@ -233,6 +442,23 @@ Replaces data in specific column regions with new data files.
 
 </details>
 
+#### DataReplacement Compatibility
+
+A DataReplacement operation only replaces a single column's worth of data. As a result, it can be safer and simpler than Merge
+or Update operations. Here are the operations that conflict with DataReplacement:
+
+- Overwrite
+- Restore
+- UpdateMemWalState
+
+The following operations are retryable conflicts with DataReplacement:
+
+- DataReplacement (only if same field and overlapping fragments)
+- CreateIndex (only if the field being replaced is being indexed)
+- Rewrite (only if overlapping fragments)
+- Update (only if overlapping fragments)
+- Merge (always)
+
 ### UpdateMemWalState
 
 Updates the state of MemWal indices (write-ahead log based indices).
@@ -259,6 +485,12 @@ Adds new base paths to the table, enabling reference to data files in additional
 
 </details>
 
+#### UpdateBases Compatibility
+
+An UpdateBases operation only modifies the base paths. As a result, it only conflicts with other
+UpdateBases operations and even then only conflicts if the two operations have base paths with the
+same id, name, or path.
+
 ## Conflict Resolution
 
 ### Terminology
@@ -383,7 +615,6 @@ In this scenario:
 - The commit fails with a non-retryable error
 - If the caller retries the deletion operation against version 4, it would either delete nothing (if those rows don't exist in v1) or delete different rows (if similar row IDs exist in v1), producing semantically different results than originally intended
 
-
 ## External Manifest Store
 
 If the backing object store does not support atomic operations (rename-if-not-exists or put-if-not-exists), an external manifest store can be used to enable concurrent writers.
@@ -399,21 +630,21 @@ The commit process follows a four-step protocol:
 ![External Store Commit Process](../../images/external_store_commit.gif)
 
 1. **Stage manifest**: `PUT_OBJECT_STORE {dataset}/_versions/{version}.manifest-{uuid}`
-    - Write the new manifest to object storage under a unique path determined by a new UUID
-    - This staged manifest is not yet visible to readers
+   - Write the new manifest to object storage under a unique path determined by a new UUID
+   - This staged manifest is not yet visible to readers
 
 2. **Commit to external store**: `PUT_EXTERNAL_STORE base_uri, version, {dataset}/_versions/{version}.manifest-{uuid}`
-    - Atomically commit the path of the staged manifest to the external store using put-if-not-exists
-    - The commit is effectively complete after this step
-    - If this operation fails due to conflict, another writer has committed this version
+   - Atomically commit the path of the staged manifest to the external store using put-if-not-exists
+   - The commit is effectively complete after this step
+   - If this operation fails due to conflict, another writer has committed this version
 
 3. **Finalize in object store**: `COPY_OBJECT_STORE {dataset}/_versions/{version}.manifest-{uuid} → {dataset}/_versions/{version}.manifest`
-    - Copy the staged manifest to the final path
-    - This makes the manifest discoverable by readers unaware of the external store
+   - Copy the staged manifest to the final path
+   - This makes the manifest discoverable by readers unaware of the external store
 
 4. **Update external store pointer**: `PUT_EXTERNAL_STORE base_uri, version, {dataset}/_versions/{version}.manifest`
-    - Update the external store to point to the finalized manifest path
-    - Completes the synchronization between external store and object storage
+   - Update the external store to point to the finalized manifest path
+   - Completes the synchronization between external store and object storage
 
 **Fault Tolerance:**
 
@@ -428,20 +659,20 @@ The reader follows a validation and synchronization protocol:
 ![External Store Reader Process](../../images/external_store_reader.gif)
 
 1. **Query external store**: `GET_EXTERNAL_STORE base_uri, version` → `path`
-    - Retrieve the manifest path for the requested version
-    - If the path does not end with a UUID, return it directly (synchronization complete)
-    - If the path ends with a UUID, synchronization is required
+   - Retrieve the manifest path for the requested version
+   - If the path does not end with a UUID, return it directly (synchronization complete)
+   - If the path ends with a UUID, synchronization is required
 
 2. **Synchronize to object store**: `COPY_OBJECT_STORE {dataset}/_versions/{version}.manifest-{uuid} → {dataset}/_versions/{version}.manifest`
-    - Attempt to finalize the staged manifest
-    - This operation is idempotent
+   - Attempt to finalize the staged manifest
+   - This operation is idempotent
 
 3. **Update external store**: `PUT_EXTERNAL_STORE base_uri, version, {dataset}/_versions/{version}.manifest`
-    - Update the external store to reflect the finalized path
-    - Future readers will see the synchronized state
+   - Update the external store to reflect the finalized path
+   - Future readers will see the synchronized state
 
 4. **Return finalized path**: Return `{dataset}/_versions/{version}.manifest`
-    - Always return the finalized path
-    - If synchronization fails, return an error to prevent reading inconsistent state
+   - Always return the finalized path
+   - If synchronization fails, return an error to prevent reading inconsistent state
 
 This protocol ensures that datasets using external manifest stores remain portable: copying the dataset directory preserves all data without requiring the external store.
diff --git a/docs/src/guide/.pages b/docs/src/guide/.pages
index 0c9a93c6920..46ddd475799 100644
--- a/docs/src/guide/.pages
+++ b/docs/src/guide/.pages
@@ -7,7 +7,8 @@ nav:
   - Tags and Branches: tags_and_branches.md
   - Object Store Configuration: object_store.md
   - Distributed Write: distributed_write.md
+  - Distributed Indexing: distributed_indexing.md
   - Migration Guide: migration.md
   - Performance Guide: performance.md
   - Tokenizer: tokenizer.md
-  - Extension Arrays: arrays.md
\ No newline at end of file
+  - Extension Arrays: arrays.md
diff --git a/docs/src/guide/distributed_indexing.md b/docs/src/guide/distributed_indexing.md
new file mode 100644
index 00000000000..83da11dbe01
--- /dev/null
+++ b/docs/src/guide/distributed_indexing.md
@@ -0,0 +1,158 @@
+# Distributed Indexing
+
+!!! warning
+    Lance exposes public APIs that can be integrated into an external
+    distributed index build workflow, but Lance itself does not provide a full
+    distributed scheduler or end-to-end orchestration layer.
+
+    This page describes the current model, terminology, and execution flow so
+    that callers can integrate these APIs correctly.
+
+## Overview
+
+Distributed index build in Lance follows the same high-level pattern as distributed
+write:
+
+1. multiple workers build index data in parallel
+2. the caller invokes Lance segment build APIs for one distributed build
+3. Lance plans and builds index artifacts from the worker outputs supplied by the caller
+4. the built artifacts are committed into the dataset manifest
+
+For vector indices, the worker outputs are segments stored directly
+under `indices/<segment_uuid>/`. Lance can turn these outputs into one or more
+physical segments and then commit them as one logical index.
+
+![Distributed Vector Segment Build](../images/distributed_vector_segment_build.svg)
+
+## Terminology
+
+This guide uses the following terms consistently:
+
+- **Segment**: one worker output written by `execute_uncommitted()` under
+  `indices/<segment_uuid>/`
+- **Physical segment**: one index segment that is ready to be committed into
+  the manifest
+- **Logical index**: the user-visible index identified by name; a logical index
+  may contain one or more physical segments
+
+For example, a distributed vector build may create a layout like:
+
+```text
+indices/<segment_uuid_0>/
+├── index.idx
+└── auxiliary.idx
+
+indices/<segment_uuid_1>/
+├── index.idx
+└── auxiliary.idx
+
+indices/<segment_uuid_2>/
+├── index.idx
+└── auxiliary.idx
+```
+
+After segment build, Lance produces one or more segment directories:
+
+```text
+indices/<physical_segment_uuid_0>/
+├── index.idx
+└── auxiliary.idx
+
+indices/<physical_segment_uuid_1>/
+├── index.idx
+└── auxiliary.idx
+```
+
+These physical segments are then committed together as one logical index. In the
+common no-merge case, the input segments are already the physical
+segments and `build_all()` returns them unchanged.
+
+## Roles
+
+There are two parties involved in distributed indexing:
+
+- **Workers** build segments
+- **The caller** launches workers, chooses how those segments should be turned
+  into physical segments, provides any additional inputs requested by the
+  segment build APIs, and
+  commits the final result
+
+Lance does not provide a distributed scheduler. The caller is responsible for
+launching workers and driving the overall workflow.
+
+## Current Model
+
+The current model for distributed vector indexing has two layers of parallelism.
+
+### Worker Build
+
+First, multiple workers build segments in parallel:
+
+1. on each worker, call a shard-build API such as
+   `create_index_builder(...).fragments(...).execute_uncommitted()`
+   or Python `create_index_uncommitted(..., fragment_ids=...)`
+2. each worker writes one segment under `indices/<segment_uuid>/`
+
+### Segment Merge
+
+Then the caller decides whether those existing segments should be committed as-is
+or merged into larger segments:
+
+1. keep the worker outputs as-is and commit them directly with
+   `commit_existing_index_segments(...)`, or
+2. group one or more existing segments and call
+   `merge_existing_index_segments(...)` for each caller-defined group
+3. commit the final segment list with `commit_existing_index_segments(...)`
+
+Within a single commit, built segments must have disjoint fragment coverage.
+
+## Internal Finalize Model
+
+Internally, Lance models distributed vector segment build as:
+
+1. **build** one uncommitted segment per worker
+2. **optionally merge** caller-defined groups of existing segments
+3. **commit** the resulting segments as one logical index
+
+The merge step is driven directly by the `IndexMetadata` returned from
+`execute_uncommitted()`.
+
+This is intentionally a storage-level model:
+
+- segments are worker outputs that are not yet published
+- physical segments are durable artifacts referenced by the manifest
+- the logical index identity is attached only at commit time
+
+## Segment Grouping
+
+The caller chooses the final segment grouping:
+
+- keep segment boundaries, so each worker output is committed directly
+- merge multiple existing segments into a larger segment before commit
+
+The grouping decision is separate from worker build. Workers only build
+segments; Lance applies the segment build policy when it plans
+physical segments.
+
+## Responsibility Boundaries
+
+The caller is expected to know:
+
+- which distributed build is ready for segment build
+- the segment metadata returned by worker builds
+- how the resulting physical segments should be published
+
+Lance is responsible for:
+
+- writing segment artifacts
+- planning physical segments from the supplied segment set
+- merging segment storage into physical segment artifacts
+- committing physical segments into the manifest
+
+If a staging root or built segment directory is never committed, it remains an
+unreferenced index directory under `_indices/`. These artifacts are cleaned up
+by `cleanup_old_versions(...)` using the same age-based rules as other
+unreferenced index files.
+
+This split keeps distributed scheduling outside the storage engine while still
+letting Lance own the on-disk index format.
diff --git a/docs/src/guide/json.md b/docs/src/guide/json.md
index 667b2596a37..7246c8fe08a 100644
--- a/docs/src/guide/json.md
+++ b/docs/src/guide/json.md
@@ -235,6 +235,75 @@ result = dataset.to_table(
 )
 ```
 
+## JSON Indexing
+
+Lance supports indexing JSON columns to accelerate filters on frequently queried paths.
+
+### Scalar Index on a JSON Path
+
+For `pa.json_()` columns, create a scalar index with `IndexConfig` and specify the JSON
+path to index. The query should use the same path literal that was indexed.
+
+```python
+import json
+import lance
+import pyarrow as pa
+from lance.indices import IndexConfig
+
+table = pa.table({
+    "id": [1, 2, 3, 4],
+    "data": pa.array([
+        json.dumps({"x": 7, "y": 10}),
+        json.dumps({"x": 11, "y": 22}),
+        json.dumps({"y": 0}),
+        json.dumps({"x": 10}),
+    ], type=pa.json_()),
+})
+
+lance.write_dataset(table, "json-index.lance")
+dataset = lance.dataset("json-index.lance")
+
+dataset.create_scalar_index(
+    "data",
+    IndexConfig(
+        index_type="json",
+        parameters={
+            "target_index_type": "btree",
+            "path": "x",
+        },
+    ),
+)
+
+result = dataset.to_table(filter="json_get_int(data, 'x') = 10")
+```
+
+!!! note
+    The JSON index matches queries by path literal. For example, if the index is built
+    with `path="x"`, then the filter should also use `"x"` with a function such as
+    `json_get_int(data, 'x')`. If the index is built with `path="$.user.name"`, then
+    the filter should use `json_extract(data, '$.user.name')`.
+
+### Full-Text Search on JSON Documents
+
+If you want text search over the contents of a JSON document instead of scalar filtering
+on a single path, create an `INVERTED` index on the JSON column.
+
+```python
+dataset.create_scalar_index(
+    "data",
+    index_type="INVERTED",
+    base_tokenizer="simple",
+    lower_case=True,
+    stem=True,
+    remove_stop_words=True,
+)
+```
+
+!!! note
+    JSON columns and nested struct columns are indexed differently. For nested struct
+    fields, use dot notation such as `meta.lang`. For `pa.json_()` columns, use the JSON
+    index shown above and query with `json_get_*` or `json_extract`.
+
 ## Usage Examples
 
 ### Working with Nested JSON
@@ -348,7 +417,7 @@ complex_projects = dataset.to_table(
 ## Performance Considerations
 
 1. **Choose the right function**: Use `json_get_*` functions for direct field access and type conversion; use `json_extract` for complex JSONPath queries.
-2. **Index frequently queried paths**: Consider creating computed columns for frequently accessed JSON paths to improve query performance.
+2. **Index frequently queried paths**: Use a JSON scalar index on frequently filtered paths before creating computed columns for the same fields.
 3. **Minimize deep nesting**: While Lance supports arbitrary nesting, flatter structures generally perform better.
 4. **Understand type conversion**: The `json_get_*` functions use strict type conversion, which may fail if types don't match. Plan your schema accordingly.
 5. **Array access**: When working with JSON arrays, you can access elements by index using numeric strings (e.g., "0", "1") with `json_get` functions.
diff --git a/docs/src/guide/migration.md b/docs/src/guide/migration.md
index 9b7471ed07c..23cab4e9745 100644
--- a/docs/src/guide/migration.md
+++ b/docs/src/guide/migration.md
@@ -6,6 +6,41 @@ stable and breaking changes should generally be communicated (via warnings) for
 give users a chance to migrate.  This page documents the breaking changes between releases and gives advice on how to
 migrate.
 
+## 5.0.0
+
+* The default data storage version changed from 2.0 to 2.1. This affects the `column_indices`
+  field in the `DataFile` protobuf message. In 2.0, every field (including non-leaf fields like
+  struct containers and list containers) was assigned a sequential column index. In 2.1, non-leaf
+  fields (unpacked structs, list containers) are assigned `-1` instead since their validity
+  information is now folded into repetition/definition levels. Only leaf fields and packed structs
+  are assigned column indices.
+
+    For example, given the schema:
+
+    ```
+    x: i32, y: [f32], z: { a: i32 }
+    ```
+
+    The fields (in depth-first order) are:
+
+    | Field ID | Field         |
+    |----------|---------------|
+    | 0        | `x` (i32)     |
+    | 1        | `y` (list)    |
+    | 2        | `y.item` (f32)|
+    | 3        | `z` (struct)  |
+    | 4        | `z.a` (i32)   |
+
+    In **2.0**, `column_indices` = `[0, 1, 2, 3, 4]` — every field gets a column.
+
+    In **2.1**, `column_indices` = `[0, -1, 1, -1, 2]` — non-leaf fields (`y` and `z`) get `-1`.
+
+* This change only affects advanced users who construct `DataFile` messages directly, for example
+  when building operations by hand for `Dataset.commit`. Normal read and write paths are
+  unaffected.
+
+* To opt back to 2.0 format, set `data_storage_version="2.0"` when creating a dataset.
+
 ## 1.0.0
 
 * The `SearchResult` returned by scalar indices must now output information about null values.
diff --git a/docs/src/guide/object_store.md b/docs/src/guide/object_store.md
index 21c1addcc62..1710e3b5100 100644
--- a/docs/src/guide/object_store.md
+++ b/docs/src/guide/object_store.md
@@ -38,8 +38,8 @@ These options apply to all object stores.
 | `proxy_url`                  | URL of a proxy server to use for requests. Default, `None`.                                                                                                                                                                                                                                             |
 | `proxy_ca_certificate`       | PEM-formatted CA certificate for proxy connections                                                                                                                                                                                                                                                      |
 | `proxy_excludes`             | List of hosts that bypass proxy. This is a comma separated list of domains and IP masks. Any subdomain of the provided domain will be bypassed. For example, `example.com, 192.168.1.0/24` would bypass `https://api.example.com`, `https://www.example.com`, and any IP in the range `192.168.1.0/24`. |
-| `client_max_retries`         | Number of times for a s3 client to retry the request. Default, `10`.                                                                                                                                                                                                                                    |
-| `client_retry_timeout`       | Timeout for a s3 client to retry the request in seconds. Default, `180`.                                                                                                                                                                                                                                |
+| `client_max_retries`         | Number of times for the object store client to retry the request. Default, `3`.                                                                                                                                                                                                                         |
+| `client_retry_timeout`       | Timeout for the object store client to retry the request in seconds. Default, `180`.                                                                                                                                                                                                                    |
 
 ## S3 Configuration
 
@@ -189,4 +189,32 @@ These keys can be used as both environment variables or keys in the `storage_opt
 | `azure_msi_resource_id` / `msi_resource_id` | Msi resource id for use with managed identity authentication. |
 | `azure_federated_token_file` / `federated_token_file` | File containing token for Azure AD workload identity federation. |
 | `azure_use_azure_cli` / `use_azure_cli` | Use azure cli for acquiring access token. |
-| `azure_disable_tagging` / `disable_tagging` | Disables tagging objects. This can be desirable if not supported by the backing store. | 
\ No newline at end of file
+| `azure_disable_tagging` / `disable_tagging` | Disables tagging objects. This can be desirable if not supported by the backing store. | 
+
+## AliCloud Object Storage Service Configuration
+
+OSS credentials can be set in the environment variables `OSS_ACCESS_KEY_ID`,
+`OSS_ACCESS_KEY_SECRET`, `OSS_REGION`, and `OSS_SECURITY_TOKEN`. Alternatively, they can be
+passed as parameters to the `storage_options` parameter:
+
+```python
+import lance
+ds = lance.dataset(
+    "oss://bucket/path",
+    storage_options={
+        "oss_region": "oss-region",
+        "oss_endpoint": "oss-endpoint",
+        "oss_access_key_id": "my-access-key",
+        "oss_secret_access_key": "my-secret-key",
+        "oss_security_token": "my-session-token",
+    }
+)
+```
+
+| Key | Description |
+|-----|-------------|
+| `oss_endpoint` | OSS endpoint. Required (for example, `https://oss-cn-hangzhou.aliyuncs.com`). |
+| `oss_access_key_id` | Access key ID used for OSS authentication. Optional if credentials are provided by environment. |
+| `oss_secret_access_key` | Access key secret used for OSS authentication. Optional if credentials are provided by environment. |
+| `oss_region` | OSS region (for example, `cn-hangzhou`). Optional. |
+| `oss_security_token` | Security token for temporary credentials (STS). Optional. |
diff --git a/docs/src/guide/performance.md b/docs/src/guide/performance.md
index ca1458834f1..4934b7c49f6 100644
--- a/docs/src/guide/performance.md
+++ b/docs/src/guide/performance.md
@@ -64,7 +64,7 @@ debugging query performance.
 Lance is designed to be thread-safe and performant. Lance APIs can be called concurrently unless
 explicitly stated otherwise. Users may create multiple tables and share tables between threads.
 Operations may run in parallel on the same table, but some operations may lead to conflicts. For
-details see [conflict resolution](../format/table/transaction.md#conflict-resolution).
+details see [conflict resolution](../format/table/transaction.md/#conflict-resolution).
 
 Most Lance operations will use multiple threads to perform work in parallel. There are two thread
 pools in lance: the IO thread pool and the compute thread pool. The IO thread pool is used for
@@ -163,11 +163,72 @@ In summary, scans could use up to `(2 * io_buffer_size) + (batch_size * num_comp
 Keep in mind that `io_buffer_size` is a soft limit (e.g. we cannot read less than one page at a time right now)
 and so it is not necessarily a bug if you see memory usage exceed this limit by a small margin.
 
-The above limits refer to limits per-scan. There is an additional limit on the number of IOPS that is applied
-across the entire process. This limit is specified by the `LANCE_PROCESS_IO_THREADS_LIMIT` environment variable.
-The default is 128 which is more than enough for most workloads. You can increase this limit if you are working
-with a high-throughput workload. You can even disable this limit entirely by setting it to zero. Note that this
-can often lead to issues with excessive retries and timeouts from the object store.
+### Cloud Store Throttling
+
+Cloud object stores (S3, GCS, Azure) are automatically wrapped with an AIMD (Additive Increase / Multiplicative
+Decrease) rate limiter. When the store returns throttle errors (HTTP 429/503), the request rate decreases
+multiplicatively. During sustained success, the rate increases additively. This applies to all operations
+(reads, writes, deletes, lists) and replaces the old `LANCE_PROCESS_IO_THREADS_LIMIT` process-wide cap.
+
+Local and in-memory stores are **not** throttled.
+
+The AIMD throttle can be tuned via storage options or environment variables. Storage options take precedence
+over environment variables:
+
+| Setting            | Storage Option Key              | Env Var                         | Default |
+| ------------------ | ------------------------------- | ------------------------------- | ------- |
+| Initial rate       | `lance_aimd_initial_rate`       | `LANCE_AIMD_INITIAL_RATE`       | 2000    |
+| Min rate           | `lance_aimd_min_rate`           | `LANCE_AIMD_MIN_RATE`           | 1       |
+| Max rate           | `lance_aimd_max_rate`           | `LANCE_AIMD_MAX_RATE`           | 5000    |
+| Decrease factor    | `lance_aimd_decrease_factor`    | `LANCE_AIMD_DECREASE_FACTOR`    | 0.5     |
+| Additive increment | `lance_aimd_additive_increment` | `LANCE_AIMD_ADDITIVE_INCREMENT` | 300     |
+| Burst capacity     | `lance_aimd_burst_capacity`     | `LANCE_AIMD_BURST_CAPACITY`     | 100     |
+
+These initial settings are balanced and should work for most
+use cases. For example, S3 can typically get up to 5000
+req/s and with these settings we should get there in about
+10 seconds.
+
+## Conflict Handling
+
+Lance supports concurrent operations on the same table using optimistic concurrency control. When two
+operations conflict, one of them must be retried. Retries are handled automatically but they repeat
+work that has already been done, which can hurt throughput. Understanding and minimizing conflicts is
+important for maintaining good performance in write-heavy workloads.
+
+Common sources of conflicts include:
+
+- Concurrent compaction and index building, since both need to modify the same indices
+- Update operations that affect the same fragments, since both need to rewrite the same data files
+
+For more details on which operations conflict with each other, see
+[conflict resolution](../format/table/transaction.md#conflict-resolution).
+
+### Fragment Reuse Index
+
+Compaction is one of the most expensive write operations because it rewrites data files and, by
+default, remaps all indices to reflect the new row addresses. When compaction and index building
+run concurrently, they often conflict because both need to modify the same indices. This typically
+causes the compaction to fail and retry, and repeated failures can cause table layout to degrade
+over time.
+
+The Fragment Reuse Index (FRI) solves this by allowing compaction to skip the index remap step.
+Instead of immediately updating indices, compaction records a mapping from old fragment row
+addresses to new ones. When indices are loaded into the cache, the FRI is applied to translate
+the old row addresses to the current ones. This adds a small cost to index load time but does
+not affect query performance once the index is cached.
+
+This decoupling means compaction and index building no longer conflict, which is especially
+valuable for tables that are continuously ingesting data while also maintaining indices.
+
+To enable the FRI, set `defer_index_remap=True` when compacting:
+
+```python
+dataset.optimize.compact_files(defer_index_remap=True)
+```
+
+For details on the index format and usage patterns, see the
+[Fragment Reuse Index specification](../format/table/index/system/frag_reuse.md).
 
 ## Indexes
 
@@ -239,3 +300,95 @@ currently extremely slow and the btree index is much faster for large range quer
 When a bitmap index is not fully loaded into the index cache, the search time will be controlled by the number of bitmaps that
 need to be loaded from disk and the speed of storage. The parts_loaded metric in the execution metrics can tell you how many
 bitmaps were loaded from disk to satisfy a query.
+
+### Vector Index
+
+Vector indexes (IVF_PQ, IVF_HNSW_SQ, etc.) are built in multiple phases, each with different memory requirements.
+
+#### IVF Training
+
+The IVF (Inverted File) phase clusters vectors into partitions using KMeans. To train the KMeans model, a sample of the
+dataset is loaded into memory. The size of this sample is determined by:
+
+```
+training_data = num_partitions * sample_rate * dimension * sizeof(data_type)
+```
+
+The default `sample_rate` is 256. For example, with 1024 partitions, 768-dimensional float32 vectors, and the default
+sample rate:
+
+```
+1024 * 256 * 768 * 4 bytes = 768 MiB
+```
+
+In addition to the training data, each KMeans iteration allocates membership and distance vectors proportional to the
+number of training vectors (8 bytes per vector). The centroids themselves require `num_partitions * dimension *
+sizeof(data_type)` bytes. In practice, the training data dominates and these additional allocations are small in
+comparison.
+
+If the dataset has fewer rows than `num_partitions * sample_rate`, the entire dataset is used for training instead.
+
+#### Quantizer Training
+
+After IVF training, a quantizer (e.g. PQ, SQ) is trained to compress vectors. This phase may sample some of the
+dataset, but the sample size is tied to properties of the quantizer and the vector dimension rather than the size of the
+dataset. As a result, quantizer training typically requires very little RAM compared to the IVF phase.
+
+#### Shuffling
+
+The final phase scans the entire vector column, transforms each vector (assigning it to an IVF partition and quantizing
+it), and writes the results into per-partition files on disk. This is a streaming operation — data is not accumulated in
+memory.
+
+The input scan uses a 2 GiB I/O readahead buffer by default (configurable via `LANCE_DEFAULT_IO_BUFFER_SIZE`) and reads
+batches of 8,192 rows. Incoming batches are transformed in parallel, with `num_cpus - 2` batches in flight at a time
+(configurable via `LANCE_CPU_THREADS`). Each batch is sorted by partition ID and the slices are written directly to the
+corresponding partition file. The in-flight memory during this phase is roughly:
+
+```
+io_readahead_buffer + num_cpu_threads * batch_size * (raw_vector_size + transformed_vector_size)
+```
+
+Each partition has an open file writer with roughly 8 MiB of accumulation buffer. In practice there shouldn't be that
+much data accumulated in a single partition anyways. Instead, the max accumulation will be roughly the final size of
+the partitions which comes out to `num_rows * (num_sub_vectors + 8) bytes`. For example, 100M rows with a 1536-dimensional
+vector will have 96 sub-vectors and so the max accumulation will be ~10GB. The additional 8 bytes per row is for the row ID.
+
+#### Storage Requirements
+
+The on-disk size of a vector index consists of the IVF centroids and the quantized vectors.
+
+The centroids require:
+
+```
+num_partitions * dimension * sizeof(data_type)
+```
+
+This is typically small. For example, 10K partitions with 768-dimensional float32 vectors is only 30 MiB.
+
+The quantized vectors make up the bulk of the index. Each row stores a quantized code plus an 8-byte row ID. The
+exact size depends on the quantizer:
+
+**PQ (Product Quantization):** Each sub-vector is quantized to a single byte, so each row requires
+`num_sub_vectors + 8` bytes. For example, 100M rows with 96 sub-vectors:
+
+```
+100M * (96 + 8) = ~9.7 GiB
+```
+
+**SQ (Scalar Quantization):** Each dimension is independently quantized to a single byte, so each row requires
+`dimension + 8` bytes. SQ preserves more information than PQ but requires more storage. For example, 100M rows with
+768-dimensional vectors:
+
+```
+100M * (768 + 8) = ~72.3 GiB
+```
+
+**RQ (RaBitQ):** Vectors are quantized to binary codes with a configurable number of bits per
+dimension. Each row also stores per-row scale and offset factors (4 bytes each) used for distance correction. Each
+row requires `dimension * num_bits / 8 + 16` bytes (8 bytes for the row ID plus 8 bytes for the factors). For
+example, 100M rows with 768 dimensions and 1 bit per dimension:
+
+```
+100M * (768 * 1 / 8 + 16) = ~10.8 GiB
+```
diff --git a/docs/src/guide/read_and_write.md b/docs/src/guide/read_and_write.md
index cbfb65a1e4c..ec6cde5173a 100644
--- a/docs/src/guide/read_and_write.md
+++ b/docs/src/guide/read_and_write.md
@@ -19,6 +19,8 @@ also supports `Iterator` of `pyarrow.RecordBatch` es.
 You will need to provide a `pyarrow.Schema` for the dataset in this case.
 
 ```python
+from typing import Iterator
+
 def producer() -> Iterator[pa.RecordBatch]:
     """An iterator of RecordBatches."""
     yield pa.RecordBatch.from_pylist([{"name": "Alice", "age": 20}])
diff --git a/docs/src/images/distributed_vector_segment_build.svg b/docs/src/images/distributed_vector_segment_build.svg
new file mode 100644
index 00000000000..4d52e2a0c82
--- /dev/null
+++ b/docs/src/images/distributed_vector_segment_build.svg
@@ -0,0 +1,121 @@
+<svg width="100%" viewBox="0 0 680 820" xmlns="http://www.w3.org/2000/svg">
+<defs>
+  <marker id="arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto-start-reverse">
+    <path d="M2 1L8 5L2 9" fill="none" stroke="context-stroke" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+  </marker>
+</defs>
+
+<!-- Phase 1: Caller -->
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="250" y="30" width="180" height="44" rx="8" stroke-width="0.5" style="fill:rgb(241, 239, 232);stroke:rgb(95, 94, 90);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="340" y="52" text-anchor="middle" dominant-baseline="central" style="fill:rgb(68, 68, 65);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">Caller</text>
+</g>
+
+<!-- Arrow down -->
+<line x1="340" y1="74" x2="340" y2="110" marker-end="url(#arrow)" style="fill:none;stroke:rgb(115, 114, 108);color:rgb(0, 0, 0);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+<text x="348" y="96" text-anchor="start" style="fill:rgb(61, 61, 58);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:12px;font-weight:400;text-anchor:start;dominant-baseline:auto">launch workers</text>
+
+<!-- Phase 2: Parallel workers container -->
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="40" y="110" width="600" height="200" rx="16" stroke-width="0.5" style="fill:rgb(238, 237, 254);stroke:rgb(83, 74, 183);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="340" y="138" text-anchor="middle" dominant-baseline="central" style="fill:rgb(60, 52, 137);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">Parallel index build (per worker)</text>
+</g>
+
+<!-- Worker 1 -->
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="68" y="158" width="160" height="56" rx="8" stroke-width="0.5" style="fill:rgb(225, 245, 238);stroke:rgb(15, 110, 86);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="148" y="178" text-anchor="middle" dominant-baseline="central" style="fill:rgb(8, 80, 65);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">Worker 1</text>
+  <text x="148" y="196" text-anchor="middle" dominant-baseline="central" style="fill:rgb(15, 110, 86);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">execute_uncommitted()</text>
+</g>
+<!-- Worker 2 -->
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="260" y="158" width="160" height="56" rx="8" stroke-width="0.5" style="fill:rgb(225, 245, 238);stroke:rgb(15, 110, 86);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="340" y="178" text-anchor="middle" dominant-baseline="central" style="fill:rgb(8, 80, 65);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">Worker 2</text>
+  <text x="340" y="196" text-anchor="middle" dominant-baseline="central" style="fill:rgb(15, 110, 86);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">execute_uncommitted()</text>
+</g>
+<!-- Worker N -->
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="452" y="158" width="160" height="56" rx="8" stroke-width="0.5" style="fill:rgb(225, 245, 238);stroke:rgb(15, 110, 86);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="532" y="178" text-anchor="middle" dominant-baseline="central" style="fill:rgb(8, 80, 65);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">Worker N</text>
+  <text x="532" y="196" text-anchor="middle" dominant-baseline="central" style="fill:rgb(15, 110, 86);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">execute_uncommitted()</text>
+</g>
+
+<!-- Partial shard outputs -->
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="68" y="234" width="160" height="36" rx="6" stroke-width="0.5" style="fill:rgb(241, 239, 232);stroke:rgb(95, 94, 90);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="148" y="252" text-anchor="middle" dominant-baseline="central" style="fill:rgb(95, 94, 90);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">partial_&lt;u1&gt;/</text>
+</g>
+<line x1="148" y1="214" x2="148" y2="234" marker-end="url(#arrow)" style="fill:none;stroke:rgb(115, 114, 108);color:rgb(0, 0, 0);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="260" y="234" width="160" height="36" rx="6" stroke-width="0.5" style="fill:rgb(241, 239, 232);stroke:rgb(95, 94, 90);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="340" y="252" text-anchor="middle" dominant-baseline="central" style="fill:rgb(95, 94, 90);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">partial_&lt;u2&gt;/</text>
+</g>
+<line x1="340" y1="214" x2="340" y2="234" marker-end="url(#arrow)" style="fill:none;stroke:rgb(115, 114, 108);color:rgb(0, 0, 0);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="452" y="234" width="160" height="36" rx="6" stroke-width="0.5" style="fill:rgb(241, 239, 232);stroke:rgb(95, 94, 90);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="532" y="252" text-anchor="middle" dominant-baseline="central" style="fill:rgb(95, 94, 90);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">partial_&lt;uN&gt;/</text>
+</g>
+<line x1="532" y1="214" x2="532" y2="234" marker-end="url(#arrow)" style="fill:none;stroke:rgb(115, 114, 108);color:rgb(0, 0, 0);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+
+<!-- Staging root label -->
+<text x="340" y="290" text-anchor="middle" dominant-baseline="central" opacity="0.7" style="fill:rgb(61, 61, 58);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:0.7;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">indices/&lt;staging_uuid&gt;/partial_&lt;u*&gt;/</text>
+
+<!-- Arrow to segment planner -->
+<line x1="340" y1="310" x2="340" y2="350" marker-end="url(#arrow)" style="fill:none;stroke:rgb(115, 114, 108);color:rgb(0, 0, 0);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+
+<!-- Phase 3: Caller-defined grouping -->
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="170" y="350" width="340" height="56" rx="8" stroke-width="0.5" style="fill:rgb(250, 236, 231);stroke:rgb(153, 60, 29);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="340" y="370" text-anchor="middle" dominant-baseline="central" style="fill:rgb(113, 43, 19);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">Caller-defined grouping</text>
+  <text x="340" y="388" text-anchor="middle" dominant-baseline="central" style="fill:rgb(153, 60, 29);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">merge_existing_index_segments(...)</text>
+</g>
+
+<!-- Arrow to plans -->
+<line x1="340" y1="406" x2="340" y2="440" marker-end="url(#arrow)" style="fill:none;stroke:rgb(115, 114, 108);color:rgb(0, 0, 0);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+<text x="348" y="428" text-anchor="start" style="fill:rgb(61, 61, 58);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:12px;font-weight:400;text-anchor:start;dominant-baseline:auto">Vec&lt;IndexMetadata&gt;</text>
+
+<!-- Phase 4: Optional segment merge -->
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="40" y="440" width="600" height="160" rx="16" stroke-width="0.5" style="fill:rgb(238, 237, 254);stroke:rgb(83, 74, 183);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="340" y="468" text-anchor="middle" dominant-baseline="central" style="fill:rgb(60, 52, 137);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">Optional segment merge</text>
+</g>
+
+<!-- Segment build 0 -->
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="68" y="488" width="160" height="56" rx="8" stroke-width="0.5" style="fill:rgb(225, 245, 238);stroke:rgb(15, 110, 86);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="148" y="508" text-anchor="middle" dominant-baseline="central" style="fill:rgb(8, 80, 65);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">merge(group[0])</text>
+  <text x="148" y="526" text-anchor="middle" dominant-baseline="central" style="fill:rgb(15, 110, 86);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">→ IndexMetadata 0</text>
+</g>
+<!-- Segment build 1 -->
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="260" y="488" width="160" height="56" rx="8" stroke-width="0.5" style="fill:rgb(225, 245, 238);stroke:rgb(15, 110, 86);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="340" y="508" text-anchor="middle" dominant-baseline="central" style="fill:rgb(8, 80, 65);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">merge(group[1])</text>
+  <text x="340" y="526" text-anchor="middle" dominant-baseline="central" style="fill:rgb(15, 110, 86);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">→ IndexMetadata 1</text>
+</g>
+<!-- Segment build N -->
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="452" y="488" width="160" height="56" rx="8" stroke-width="0.5" style="fill:rgb(225, 245, 238);stroke:rgb(15, 110, 86);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="532" y="508" text-anchor="middle" dominant-baseline="central" style="fill:rgb(8, 80, 65);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">merge(group[N])</text>
+  <text x="532" y="526" text-anchor="middle" dominant-baseline="central" style="fill:rgb(15, 110, 86);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">→ IndexMetadata N</text>
+</g>
+
+<!-- Arrow to commit -->
+<line x1="340" y1="600" x2="340" y2="650" marker-end="url(#arrow)" style="fill:none;stroke:rgb(115, 114, 108);color:rgb(0, 0, 0);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+
+<!-- Phase 5: Commit -->
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="140" y="650" width="400" height="44" rx="8" stroke-width="0.5" style="fill:rgb(250, 236, 231);stroke:rgb(153, 60, 29);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="340" y="672" text-anchor="middle" dominant-baseline="central" style="fill:rgb(113, 43, 19);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">commit_existing_index_segments(...)</text>
+</g>
+
+<!-- Arrow to logical index -->
+<line x1="340" y1="694" x2="340" y2="740" marker-end="url(#arrow)" style="fill:none;stroke:rgb(115, 114, 108);color:rgb(0, 0, 0);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+
+<!-- Final: Logical index -->
+<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
+  <rect x="230" y="740" width="220" height="44" rx="8" stroke-width="0.5" style="fill:rgb(234, 243, 222);stroke:rgb(59, 109, 17);color:rgb(0, 0, 0);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
+  <text x="340" y="762" text-anchor="middle" dominant-baseline="central" style="fill:rgb(39, 80, 10);stroke:none;color:rgb(0, 0, 0);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:&quot;Anthropic Sans&quot;, -apple-system, &quot;system-ui&quot;, &quot;Segoe UI&quot;, sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">Logical index</text>
+</g>
+</svg>
diff --git a/docs/src/images/lakehouse_stack.png b/docs/src/images/lakehouse_stack.png
index ff9546d9639..4fa98dbdcf6 100644
Binary files a/docs/src/images/lakehouse_stack.png and b/docs/src/images/lakehouse_stack.png differ
diff --git a/docs/src/quickstart/index.md b/docs/src/quickstart/index.md
index 6f2c9a8d147..00daa6a4ee4 100644
--- a/docs/src/quickstart/index.md
+++ b/docs/src/quickstart/index.md
@@ -25,8 +25,6 @@ pip install --pre --extra-index-url https://pypi.fury.io/lance-format/pylance
 
 > Note: Preview releases receive the same level of testing as regular releases.
 
-> Note: For versions prior to 1.0.0-beta.4, you can find them at https://pypi.fury.io/lancedb/pylance
-
 ## Set Up Your Environment
 
 First, let's import the necessary libraries:
diff --git a/java/dev/checkstyle.xml b/java/dev/checkstyle.xml
index dd5fe8cab62..9a0e1f5fb1a 100644
--- a/java/dev/checkstyle.xml
+++ b/java/dev/checkstyle.xml
@@ -134,6 +134,9 @@
         <module name="CommentsIndentation"/>
         <module name="UnusedImports"/>
         <module name="RedundantImport"/>
+        <module name="AvoidStarImport">
+            <property name="allowStaticMemberImports" value="true"/>
+        </module>
         <module name="RedundantModifier"/>
         <module name="RegexpSinglelineJava">
             <property name="format" value="throw new \w+Error\("/>
diff --git a/java/lance-jni/Cargo.lock b/java/lance-jni/Cargo.lock
index 7cd5c1cefec..22acba0d789 100644
--- a/java/lance-jni/Cargo.lock
+++ b/java/lance-jni/Cargo.lock
@@ -16,7 +16,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
 dependencies = [
  "cfg-if",
  "cipher",
- "cpufeatures",
+ "cpufeatures 0.2.17",
 ]
 
 [[package]]
@@ -74,9 +74,9 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.6.21"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
+checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
 dependencies = [
  "anstyle",
  "anstyle-parse",
@@ -89,15 +89,15 @@ dependencies = [
 
 [[package]]
 name = "anstyle"
-version = "1.0.13"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.7"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
+checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
 dependencies = [
  "utf8parse",
 ]
@@ -137,15 +137,6 @@ dependencies = [
  "num-traits",
 ]
 
-[[package]]
-name = "arc-swap"
-version = "1.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5"
-dependencies = [
- "rustversion",
-]
-
 [[package]]
 name = "arrayref"
 version = "0.3.9"
@@ -286,7 +277,7 @@ dependencies = [
  "arrow-schema",
  "arrow-select",
  "flatbuffers",
- "lz4_flex 0.12.0",
+ "lz4_flex",
  "zstd",
 ]
 
@@ -513,9 +504,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-rs"
-version = "1.16.1"
+version = "1.16.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf"
+checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f"
 dependencies = [
  "aws-lc-sys",
  "zeroize",
@@ -523,9 +514,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-sys"
-version = "0.38.0"
+version = "0.40.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e"
+checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7"
 dependencies = [
  "cc",
  "cmake",
@@ -928,9 +919,9 @@ dependencies = [
 
 [[package]]
 name = "bitflags"
-version = "2.11.0"
+version = "2.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
+checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
 
 [[package]]
 name = "bitpacking"
@@ -964,16 +955,16 @@ dependencies = [
 
 [[package]]
 name = "blake3"
-version = "1.8.3"
+version = "1.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d"
+checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e"
 dependencies = [
  "arrayref",
  "arrayvec",
  "cc",
  "cfg-if",
  "constant_time_eq",
- "cpufeatures",
+ "cpufeatures 0.3.0",
 ]
 
 [[package]]
@@ -994,31 +985,6 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "bon"
-version = "3.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d13a61f2963b88eef9c1be03df65d42f6996dfeac1054870d950fcf66686f83"
-dependencies = [
- "bon-macros",
- "rustversion",
-]
-
-[[package]]
-name = "bon-macros"
-version = "3.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d314cc62af2b6b0c65780555abb4d02a03dd3b799cd42419044f0c38d99738c0"
-dependencies = [
- "darling",
- "ident_case",
- "prettyplease",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "brotli"
 version = "8.0.2"
@@ -1085,9 +1051,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.56"
+version = "1.2.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
+checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1095,12 +1061,6 @@ dependencies = [
  "shlex",
 ]
 
-[[package]]
-name = "census"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0"
-
 [[package]]
 name = "cesu8"
 version = "1.1.0"
@@ -1155,18 +1115,18 @@ dependencies = [
 
 [[package]]
 name = "cmake"
-version = "0.1.57"
+version = "0.1.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d"
+checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
 dependencies = [
  "cc",
 ]
 
 [[package]]
 name = "colorchoice"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
+checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
 
 [[package]]
 name = "combine"
@@ -1271,6 +1231,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "cpufeatures"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "crc32c"
 version = "0.6.8"
@@ -1379,40 +1348,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "darling"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
-dependencies = [
- "darling_core",
- "darling_macro",
-]
-
-[[package]]
-name = "darling_core"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
-dependencies = [
- "ident_case",
- "proc-macro2",
- "quote",
- "strsim",
- "syn 2.0.117",
-]
-
-[[package]]
-name = "darling_macro"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
-dependencies = [
- "darling_core",
- "quote",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "dashmap"
 version = "6.1.0"
@@ -1429,9 +1364,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503f1f4a9060ae6e650d3dff5dc7a21266fea1302d890768d45b4b28586e830f"
+checksum = "7541353e77dc7262b71ca27be07d8393661737e3a73b5d1b1c6f7d814c64fa2a"
 dependencies = [
  "arrow",
  "arrow-schema",
@@ -1469,7 +1404,7 @@ dependencies = [
  "object_store",
  "parking_lot",
  "parquet",
- "rand 0.9.2",
+ "rand 0.9.4",
  "regex",
  "sqlparser",
  "tempfile",
@@ -1480,9 +1415,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14417a3ee4ae3d092b56cd6c1d32e8ff3e2c9ec130ecb2276ec91c89fd599399"
+checksum = "9997731f90fa5398ef831ad0e69600f92c861b79c0d38bd1a29b6f0e3a0ce4c8"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1505,9 +1440,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog-listing"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d0eba824adb45a4b3ac6f0251d40df3f6a9382371cad136f4f14ac9ebc6bc10"
+checksum = "2b30a3dd50dec860c9559275c8d97d9de602e611237a6ecfbda0b3b63b872352"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1528,9 +1463,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0039deefbd00c56adf5168b7ca58568fb058e4ba4c5a03b09f8be371b4e434b6"
+checksum = "d551054acec0398ca604512310b77ce05c46f66e54b54d48200a686e385cca4e"
 dependencies = [
  "ahash",
  "arrow",
@@ -1551,9 +1486,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common-runtime"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ec7e3e60b813048331f8fb9673583173e5d2dd8fef862834ee871fc98b57ca7"
+checksum = "567d40e285f5b79f8737b576605721cd6c1133b5d2b00bdbd5d9838d90d0812f"
 dependencies = [
  "futures",
  "log",
@@ -1562,9 +1497,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "802068957f620302ecf05f84ff4019601aeafd36f5f3f1334984af2e34265129"
+checksum = "27d2668f51b3b30befae2207472569e37807fdedd1d14da58acc6f8ca6257eae"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1584,16 +1519,16 @@ dependencies = [
  "itertools 0.14.0",
  "log",
  "object_store",
- "rand 0.9.2",
+ "rand 0.9.4",
  "tokio",
  "url",
 ]
 
 [[package]]
 name = "datafusion-datasource-arrow"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90fc387d5067c62d494a6647d29c5ad4fcdd5a6e50ab4ea1d2568caa2d66f2cc"
+checksum = "e02e1b3e3a8ec55f1f62de4252b0407c8567363d056078769a197e24fc834a0f"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -1615,9 +1550,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-csv"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efd5e20579bb6c8bd4e6c620253972fb723822030c280dd6aa047f660d09eeba"
+checksum = "b559d7bf87d4f900f847baba8509634f838d9718695389e903604cdcccdb01f3"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1638,9 +1573,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-json"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0788b0d48fcef31880a02013ea3cc18e5a4e0eacc3b0abdd2cd0597b99dc96e"
+checksum = "250e2d7591ba8b638f063854650faa40bca4e8bd4059b2ece8836f6388d02db4"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1660,9 +1595,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-parquet"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66639b70f1f363f5f0950733170100e588f1acfacac90c1894e231194aa35957"
+checksum = "0b043149f2c3557ca94abc58de40f68a8d412ff53365c06126ed234f8596399d"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1690,15 +1625,15 @@ dependencies = [
 
 [[package]]
 name = "datafusion-doc"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e44b41f3e8267c6cf3eec982d63f34db9f1dd5f30abfd2e1f124f0871708952e"
+checksum = "b9496cb0db222dbb9a3735760ceca7fc56f35e1d5502c38d0caa77a81e9c1f6a"
 
 [[package]]
 name = "datafusion-execution"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e456f60e5d38db45335e84617006d90af14a8c8c5b8e959add708b2daaa0e2c"
+checksum = "dc45d23c516ed8d3637751e44e09e21b45b3f58b473c802dddd1f1ad4fe435ff"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1710,16 +1645,16 @@ dependencies = [
  "log",
  "object_store",
  "parking_lot",
- "rand 0.9.2",
+ "rand 0.9.4",
  "tempfile",
  "url",
 ]
 
 [[package]]
 name = "datafusion-expr"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6507c719804265a58043134580c1c20767e7c23ba450724393f03ec982769ad9"
+checksum = "63dd30526d2db4fda6440806a41e4676334a94bc0596cc9cc2a0efed20ef2c44"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1739,9 +1674,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-expr-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a413caa9c5885072b539337aed68488f0291653e8edd7d676c92df2480f6cab0"
+checksum = "1b486b5f6255d40976b88bb83813b0d035a8333e0ec39864824e78068cf42fa6"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -1752,9 +1687,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "189256495dc9cbbb8e20dbcf161f60422e628d201a78df8207e44bd4baefadb6"
+checksum = "07356c94118d881130dd0ffbff127540407d969c8978736e324edcd6c41cd48f"
 dependencies = [
  "arrow",
  "arrow-buffer",
@@ -1774,7 +1709,7 @@ dependencies = [
  "log",
  "md-5",
  "num-traits",
- "rand 0.9.2",
+ "rand 0.9.4",
  "regex",
  "sha2",
  "unicode-segmentation",
@@ -1783,9 +1718,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-aggregate"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12e73dfee4cd67c4a507ffff4c5a711d39983adf544adbc09c09bf06f789f413"
+checksum = "b644f9cf696df9233ce6958b9807666d78563b56f923267474dd6c07795f1f8f"
 dependencies = [
  "ahash",
  "arrow",
@@ -1804,9 +1739,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-aggregate-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87727bd9e65f4f9ac6d608c9810b7da9eaa3b18b26a4a4b76520592d49020acf"
+checksum = "c1de2deaaabe8923ce9ea9f29c47bbb4ee14f67ea2fe1ab5398d9bbebcf86e56"
 dependencies = [
  "ahash",
  "arrow",
@@ -1817,9 +1752,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-nested"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e5ef761359224b7c2b5a1bfad6296ac63225f8583d08ad18af9ba1a89ac3887"
+checksum = "552f8d92e4331ee91d23c02d12bb6acf32cbfd5215117e01c0fb63cd4b15af1a"
 dependencies = [
  "arrow",
  "arrow-ord",
@@ -1840,9 +1775,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-table"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b17dac25dfda2d2a90ff0ad1c054a11fb1523766226bec6e9bd8c410daee2ae"
+checksum = "970fd0cdd3df8802b9a9975ff600998289ba9d46682a4f7285cba4820c9ada78"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1856,9 +1791,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-window"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c594a29ddb22cbdbce500e4d99b5b2392c5cecb4c1086298b41d1ffec14dbb77"
+checksum = "40b4c21a7c8a986a1866c0a87ab756d0bbf7b5f41f306009fa2d9af79c52ed31"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -1874,9 +1809,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-window-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aa1b15ed81c7543f62264a30dd49dec4b1b0b698053b968f53be32dfba4f729"
+checksum = "b1210ad73b8b3211aeaf4a42bef9bd7a2b7fce3ec119a478831f18c6ff7f7b93"
 dependencies = [
  "datafusion-common",
  "datafusion-physical-expr-common",
@@ -1884,9 +1819,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-macros"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c00c31c4795597aa25b74cab5174ac07a53051f27ce1e011ecaffa9eaeecef81"
+checksum = "aaa566a963013a38681ad82a727a654bc7feb19632426aea8c3412d415d200c5"
 dependencies = [
  "datafusion-doc",
  "quote",
@@ -1895,9 +1830,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-optimizer"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80ccf60767c09302b2e0fc3afebb3761a6d508d07316fab8c5e93312728a21bb"
+checksum = "ff9aa82b240252a88dee118372f9b9757c545ab9e53c0736bebab2e7da0ef1f2"
 dependencies = [
  "arrow",
  "chrono",
@@ -1914,9 +1849,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c64b7f277556944e4edd3558da01d9e9ff9f5416f1c0aa7fee088e57bd141a7e"
+checksum = "7d48022b8af9988c1d852644f9e8b5584c490659769a550c5e8d39457a1da0a5"
 dependencies = [
  "ahash",
  "arrow",
@@ -1937,9 +1872,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-adapter"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7abaee372ea2d19c016ee9ef8629c4415257d291cdd152bc7f0b75f28af1b63"
+checksum = "ae7a8abc0b4fe624000972a9b145b30b7f1b680bffaa950ea53f78d9b21c27c3"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -1952,9 +1887,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42237efe621f92adc22d111b531fdbc2cc38ca9b5e02327535628fb103ae2157"
+checksum = "147253ca3e6b9d59c162de64c02800973018660e13340dd1886dd038d17ac429"
 dependencies = [
  "ahash",
  "arrow",
@@ -1969,9 +1904,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-optimizer"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd093498bd1319c6e5c76e9dfa905e78486f01b34579ce97f2e3a49f84c37fac"
+checksum = "689156bb2282107b6239db8d7ef44b4dab10a9b33d3491a0c74acac5e4fedd72"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -1987,9 +1922,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-plan"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cbe61b12daf81a9f20ba03bd3541165d51f86e004ef37426b11881330eed261"
+checksum = "68253dc0ee5330aa558b2549c9b0da5af9fc17d753ae73022939014ad616fc28"
 dependencies = [
  "ahash",
  "arrow",
@@ -2018,9 +1953,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-pruning"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0124331116db7f79df92ebfd2c3b11a8f90240f253555c9bb084f10b6fecf1dd"
+checksum = "0fcad240a54d0b1d3e8f668398900260a53122d522b2102ab57218590decacd6"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2035,9 +1970,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-session"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1673e3c58ba618a6ea0568672f00664087b8982c581e9afd5aa6c3c79c9b431f"
+checksum = "f58e83a68bb67007a8fcbf005c44cefe441270c7ee7f6dee10c0e0109b556f6d"
 dependencies = [
  "async-trait",
  "datafusion-common",
@@ -2049,9 +1984,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-sql"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5272d256dab5347bb39d2040589f45d8c6b715b27edcb5fffe88cc8b9c3909cb"
+checksum = "be53e9eb55db0fbb8980bb6d87f2435b0524acf4c718ed54a57cabbb299b2ab3"
 dependencies = [
  "arrow",
  "bigdecimal",
@@ -2066,9 +2001,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-substrait"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "199790fd96e852997b30da4ff11109378c944841757d93875ea85fc69587ec91"
+checksum = "7dadf18107366b3470e47bdde8eeb9294b36444e451830d8e5fc8dc7b4ad9c58"
 dependencies = [
  "async-recursion",
  "async-trait",
@@ -2122,7 +2057,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
 dependencies = [
  "powerfmt",
- "serde_core",
 ]
 
 [[package]]
@@ -2178,12 +2112,6 @@ dependencies = [
  "const-random",
 ]
 
-[[package]]
-name = "downcast-rs"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc"
-
 [[package]]
 name = "dunce"
 version = "1.0.5"
@@ -2223,9 +2151,9 @@ dependencies = [
 
 [[package]]
 name = "env_filter"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f"
+checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef"
 dependencies = [
  "log",
  "regex",
@@ -2233,9 +2161,9 @@ dependencies = [
 
 [[package]]
 name = "env_logger"
-version = "0.11.9"
+version = "0.11.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d"
+checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a"
 dependencies = [
  "anstream",
  "anstyle",
@@ -2293,17 +2221,11 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
 
-[[package]]
-name = "fastdivide"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471"
-
 [[package]]
 name = "fastrand"
-version = "2.3.0"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
+checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
 
 [[package]]
 name = "find-msvc-tools"
@@ -2371,16 +2293,6 @@ dependencies = [
  "percent-encoding",
 ]
 
-[[package]]
-name = "fs4"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8"
-dependencies = [
- "rustix 0.38.44",
- "windows-sys 0.52.0",
-]
-
 [[package]]
 name = "fs_extra"
 version = "1.3.0"
@@ -2389,10 +2301,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-array",
- "rand 0.9.2",
+ "rand 0.9.4",
 ]
 
 [[package]]
@@ -2552,9 +2464,9 @@ dependencies = [
 
 [[package]]
 name = "geo-types"
-version = "0.7.18"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24f8647af4005fa11da47cd56252c6ef030be8fa97bdbf355e7dfb6348f0a82c"
+checksum = "94776032c45f950d30a13af6113c2ad5625316c9abfbccee4dd5a6695f8fe0f5"
 dependencies = [
  "approx",
  "num-traits",
@@ -2755,8 +2667,6 @@ version = "0.15.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
 dependencies = [
- "allocator-api2",
- "equivalent",
  "foldhash 0.1.5",
 ]
 
@@ -2771,6 +2681,12 @@ dependencies = [
  "foldhash 0.2.0",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"
+
 [[package]]
 name = "heapless"
 version = "0.8.0"
@@ -2817,12 +2733,6 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
-[[package]]
-name = "htmlescape"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
-
 [[package]]
 name = "http"
 version = "0.2.12"
@@ -2898,9 +2808,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
 
 [[package]]
 name = "hyper"
-version = "1.8.1"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"
+checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -2913,7 +2823,6 @@ dependencies = [
  "httpdate",
  "itoa",
  "pin-project-lite",
- "pin-utils",
  "smallvec",
  "tokio",
  "want",
@@ -2921,16 +2830,15 @@ dependencies = [
 
 [[package]]
 name = "hyper-rustls"
-version = "0.27.7"
+version = "0.27.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58"
+checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
 dependencies = [
  "http 1.4.0",
  "hyper",
  "hyper-util",
  "rustls",
  "rustls-native-certs",
- "rustls-pki-types",
  "tokio",
  "tokio-rustls",
  "tower-service",
@@ -3038,12 +2946,13 @@ dependencies = [
 
 [[package]]
 name = "icu_collections"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43"
+checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"
 dependencies = [
  "displaydoc",
  "potential_utf",
+ "utf8_iter",
  "yoke",
  "zerofrom",
  "zerovec",
@@ -3051,9 +2960,9 @@ dependencies = [
 
 [[package]]
 name = "icu_locale_core"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6"
+checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
 dependencies = [
  "displaydoc",
  "litemap",
@@ -3064,9 +2973,9 @@ dependencies = [
 
 [[package]]
 name = "icu_normalizer"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599"
+checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
 dependencies = [
  "icu_collections",
  "icu_normalizer_data",
@@ -3078,15 +2987,15 @@ dependencies = [
 
 [[package]]
 name = "icu_normalizer_data"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"
+checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
 
 [[package]]
 name = "icu_properties"
-version = "2.1.2"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec"
+checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
 dependencies = [
  "icu_collections",
  "icu_locale_core",
@@ -3098,15 +3007,15 @@ dependencies = [
 
 [[package]]
 name = "icu_properties_data"
-version = "2.1.2"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af"
+checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
 
 [[package]]
 name = "icu_provider"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614"
+checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
 dependencies = [
  "displaydoc",
  "icu_locale_core",
@@ -3123,12 +3032,6 @@ version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
 
-[[package]]
-name = "ident_case"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
-
 [[package]]
 name = "idna"
 version = "1.1.0"
@@ -3152,12 +3055,12 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.13.0"
+version = "2.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
+checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
 dependencies = [
  "equivalent",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.0",
  "serde",
  "serde_core",
 ]
@@ -3178,6 +3081,17 @@ version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
+[[package]]
+name = "io-uring"
+version = "0.7.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdd7bddefd0a8833b88a4b68f90dae22c7450d11b354198baee3874fd811b344"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "libc",
+]
+
 [[package]]
 name = "ipnet"
 version = "2.12.0"
@@ -3186,9 +3100,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
 
 [[package]]
 name = "iri-string"
-version = "0.7.10"
+version = "0.7.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a"
+checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20"
 dependencies = [
  "memchr",
  "serde",
@@ -3229,9 +3143,9 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.17"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "jiff"
@@ -3283,7 +3197,7 @@ dependencies = [
  "cesu8",
  "cfg-if",
  "combine",
- "jni-sys",
+ "jni-sys 0.3.1",
  "log",
  "thiserror 1.0.69",
  "walkdir",
@@ -3292,9 +3206,31 @@ dependencies = [
 
 [[package]]
 name = "jni-sys"
-version = "0.3.0"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"
+dependencies = [
+ "jni-sys 0.4.1",
+]
+
+[[package]]
+name = "jni-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
+dependencies = [
+ "jni-sys-macros",
+]
+
+[[package]]
+name = "jni-sys-macros"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130"
+checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
+dependencies = [
+ "quote",
+ "syn 2.0.117",
+]
 
 [[package]]
 name = "jobserver"
@@ -3308,32 +3244,34 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.91"
+version = "0.3.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c"
+checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca"
 dependencies = [
+ "cfg-if",
+ "futures-util",
  "once_cell",
  "wasm-bindgen",
 ]
 
 [[package]]
 name = "jsonb"
-version = "0.5.5"
+version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a901f06163d352fbe41c3c2ff5e08b75330a003cc941e988fb501022f5421e6"
+checksum = "eb98fb29636087c40ad0d1274d9a30c0c1e83e03ae93f6e7e89247b37fcc6953"
 dependencies = [
  "byteorder",
  "ethnum",
  "fast-float2",
  "itoa",
  "jiff",
- "nom 8.0.0",
+ "nom",
  "num-traits",
- "ordered-float 5.1.0",
- "rand 0.9.2",
- "ryu",
+ "ordered-float 5.3.0",
+ "rand 0.9.4",
  "serde",
  "serde_json",
+ "zmij",
 ]
 
 [[package]]
@@ -3353,12 +3291,13 @@ dependencies = [
 
 [[package]]
 name = "lance"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-arith",
  "arrow-array",
  "arrow-buffer",
+ "arrow-cast",
  "arrow-ipc",
  "arrow-ord",
  "arrow-row",
@@ -3395,20 +3334,21 @@ dependencies = [
  "lance-linalg",
  "lance-namespace",
  "lance-table",
+ "lance-tokenizer",
  "log",
  "moka",
  "object_store",
  "permutation",
  "pin-project",
  "prost",
+ "prost-build",
  "prost-types",
- "rand 0.9.2",
+ "rand 0.9.4",
  "roaring",
  "semver",
  "serde",
  "serde_json",
  "snafu",
- "tantivy",
  "tokio",
  "tokio-stream",
  "tokio-util",
@@ -3419,12 +3359,13 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-cast",
  "arrow-data",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
@@ -3434,12 +3375,12 @@ dependencies = [
  "half",
  "jsonb",
  "num-traits",
- "rand 0.9.2",
+ "rand 0.9.4",
 ]
 
 [[package]]
 name = "lance-bitpacking"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrayref",
  "paste",
@@ -3448,7 +3389,7 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -3471,7 +3412,7 @@ dependencies = [
  "object_store",
  "pin-project",
  "prost",
- "rand 0.9.2",
+ "rand 0.9.4",
  "roaring",
  "serde_json",
  "snafu",
@@ -3485,11 +3426,12 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-array",
  "arrow-buffer",
+ "arrow-cast",
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
@@ -3511,13 +3453,14 @@ dependencies = [
  "prost",
  "prost-build",
  "snafu",
+ "substrait",
  "tokio",
  "tracing",
 ]
 
 [[package]]
 name = "lance-datagen"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -3527,15 +3470,15 @@ dependencies = [
  "futures",
  "half",
  "hex",
- "rand 0.9.2",
- "rand_distr 0.5.1",
+ "rand 0.9.4",
+ "rand_distr",
  "rand_xoshiro",
  "random_word",
 ]
 
 [[package]]
 name = "lance-encoding"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -3561,7 +3504,7 @@ dependencies = [
  "prost",
  "prost-build",
  "prost-types",
- "rand 0.9.2",
+ "rand 0.9.4",
  "snafu",
  "strum",
  "tokio",
@@ -3572,7 +3515,7 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -3604,7 +3547,7 @@ dependencies = [
 
 [[package]]
 name = "lance-geo"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "datafusion",
  "geo-traits",
@@ -3618,7 +3561,7 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -3632,6 +3575,7 @@ dependencies = [
  "bitpacking",
  "bitvec",
  "bytes",
+ "chrono",
  "crossbeam-queue",
  "datafusion",
  "datafusion-common",
@@ -3658,6 +3602,7 @@ dependencies = [
  "lance-io",
  "lance-linalg",
  "lance-table",
+ "lance-tokenizer",
  "libm",
  "log",
  "ndarray",
@@ -3666,8 +3611,8 @@ dependencies = [
  "prost",
  "prost-build",
  "prost-types",
- "rand 0.9.2",
- "rand_distr 0.5.1",
+ "rand 0.9.4",
+ "rand_distr",
  "rangemap",
  "rayon",
  "roaring",
@@ -3675,7 +3620,6 @@ dependencies = [
  "serde_json",
  "smallvec",
  "snafu",
- "tantivy",
  "tempfile",
  "tokio",
  "tracing",
@@ -3685,7 +3629,7 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -3705,17 +3649,20 @@ dependencies = [
  "deepsize",
  "futures",
  "http 1.4.0",
+ "io-uring",
  "lance-arrow",
  "lance-core",
  "lance-namespace",
+ "libc",
  "log",
+ "moka",
  "object_store",
  "object_store_opendal",
  "opendal",
  "path_abs",
  "pin-project",
  "prost",
- "rand 0.9.2",
+ "rand 0.9.4",
  "serde",
  "snafu",
  "tempfile",
@@ -3726,13 +3673,15 @@ dependencies = [
 
 [[package]]
 name = "lance-jni"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
+ "arrow-array",
  "arrow-schema",
  "async-trait",
  "bytes",
  "chrono",
+ "datafusion-common",
  "env_logger",
  "jni",
  "lance",
@@ -3760,7 +3709,7 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -3771,24 +3720,25 @@ dependencies = [
  "lance-arrow",
  "lance-core",
  "num-traits",
- "rand 0.9.2",
+ "rand 0.9.4",
 ]
 
 [[package]]
 name = "lance-namespace"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "async-trait",
  "bytes",
  "lance-core",
  "lance-namespace-reqwest-client",
+ "serde",
  "snafu",
 ]
 
 [[package]]
 name = "lance-namespace-impls"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -3802,11 +3752,12 @@ dependencies = [
  "lance-core",
  "lance-index",
  "lance-io",
+ "lance-linalg",
  "lance-namespace",
  "lance-table",
  "log",
  "object_store",
- "rand 0.9.2",
+ "rand 0.9.4",
  "reqwest",
  "serde",
  "serde_json",
@@ -3819,9 +3770,9 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-reqwest-client"
-version = "0.5.2"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ad4c947349acd6e37e984eba0254588bd894e6128434338b9e6904e56fb4633"
+checksum = "ee2e48de899e2931afb67fcddd0a08e439bf5d8b6ea2a2ed9cb8f4df669bd5cc"
 dependencies = [
  "reqwest",
  "serde",
@@ -3832,7 +3783,7 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -3854,7 +3805,7 @@ dependencies = [
  "prost",
  "prost-build",
  "prost-types",
- "rand 0.9.2",
+ "rand 0.9.4",
  "rangemap",
  "roaring",
  "semver",
@@ -3867,6 +3818,15 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "lance-tokenizer"
+version = "6.0.0-beta.1"
+dependencies = [
+ "rust-stemmers",
+ "serde",
+ "unicode-normalization",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.5.0"
@@ -3882,12 +3842,6 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
 
-[[package]]
-name = "levenshtein_automata"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
-
 [[package]]
 name = "lexical-core"
 version = "1.0.6"
@@ -3947,9 +3901,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.182"
+version = "0.2.185"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
+checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"
 
 [[package]]
 name = "libm"
@@ -3959,19 +3913,13 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 
 [[package]]
 name = "libredox"
-version = "0.1.14"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a"
+checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c"
 dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "linux-raw-sys"
-version = "0.4.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
-
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -3980,9 +3928,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
 
 [[package]]
 name = "litemap"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
+checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
 
 [[package]]
 name = "lock_api"
@@ -4012,15 +3960,6 @@ dependencies = [
  "tracing-subscriber",
 ]
 
-[[package]]
-name = "lru"
-version = "0.12.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
-dependencies = [
- "hashbrown 0.15.5",
-]
-
 [[package]]
 name = "lru-slab"
 version = "0.1.2"
@@ -4048,15 +3987,9 @@ dependencies = [
 
 [[package]]
 name = "lz4_flex"
-version = "0.11.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a"
-
-[[package]]
-name = "lz4_flex"
-version = "0.12.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e"
+checksum = "98c23545df7ecf1b16c303910a69b079e8e251d60f7dd2cc9b4177f2afaf1746"
 dependencies = [
  "twox-hash",
 ]
@@ -4099,30 +4032,12 @@ dependencies = [
  "digest",
 ]
 
-[[package]]
-name = "measure_time"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e"
-dependencies = [
- "log",
-]
-
 [[package]]
 name = "memchr"
 version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
 
-[[package]]
-name = "memmap2"
-version = "0.9.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -4139,12 +4054,6 @@ dependencies = [
  "unicase",
 ]
 
-[[package]]
-name = "minimal-lexical"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
-
 [[package]]
 name = "miniz_oxide"
 version = "0.8.9"
@@ -4157,9 +4066,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "1.1.1"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc"
+checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
 dependencies = [
  "libc",
  "wasi",
@@ -4174,9 +4083,9 @@ checksum = "dce6dd36094cac388f119d2e9dc82dc730ef91c32a6222170d630e5414b956e6"
 
 [[package]]
 name = "moka"
-version = "0.12.14"
+version = "0.12.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85f8024e1c8e71c778968af91d43700ce1d11b219d127d79fb2934153b82b42b"
+checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046"
 dependencies = [
  "async-lock",
  "crossbeam-channel",
@@ -4198,12 +4107,6 @@ version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
 
-[[package]]
-name = "murmurhash32"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
-
 [[package]]
 name = "ndarray"
 version = "0.16.1"
@@ -4219,16 +4122,6 @@ dependencies = [
  "rawpointer",
 ]
 
-[[package]]
-name = "nom"
-version = "7.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
-dependencies = [
- "memchr",
- "minimal-lexical",
-]
-
 [[package]]
 name = "nom"
 version = "8.0.0"
@@ -4284,9 +4177,9 @@ dependencies = [
 
 [[package]]
 name = "num-conv"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
+checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
 
 [[package]]
 name = "num-integer"
@@ -4330,9 +4223,9 @@ dependencies = [
 
 [[package]]
 name = "num_enum"
-version = "0.7.5"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c"
+checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26"
 dependencies = [
  "num_enum_derive",
  "rustversion",
@@ -4340,9 +4233,9 @@ dependencies = [
 
 [[package]]
 name = "num_enum_derive"
-version = "0.7.5"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7"
+checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8"
 dependencies = [
  "proc-macro-crate",
  "proc-macro2",
@@ -4372,7 +4265,7 @@ dependencies = [
  "parking_lot",
  "percent-encoding",
  "quick-xml 0.38.4",
- "rand 0.9.2",
+ "rand 0.9.4",
  "reqwest",
  "ring",
  "rustls-pemfile",
@@ -4406,9 +4299,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.21.3"
+version = "1.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "once_cell_polyfill"
@@ -4416,12 +4309,6 @@ version = "1.70.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
 
-[[package]]
-name = "oneshot"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
-
 [[package]]
 name = "opendal"
 version = "0.55.0"
@@ -4475,9 +4362,9 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "5.1.0"
+version = "5.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d"
+checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e"
 dependencies = [
  "num-traits",
 ]
@@ -4498,15 +4385,6 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
 
-[[package]]
-name = "ownedbytes"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e"
-dependencies = [
- "stable_deref_trait",
-]
-
 [[package]]
 name = "parking"
 version = "2.2.1"
@@ -4558,7 +4436,7 @@ dependencies = [
  "futures",
  "half",
  "hashbrown 0.16.1",
- "lz4_flex 0.12.0",
+ "lz4_flex",
  "num-bigint",
  "num-integer",
  "num-traits",
@@ -4771,9 +4649,9 @@ dependencies = [
 
 [[package]]
 name = "pkg-config"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
 
 [[package]]
 name = "portable-atomic"
@@ -4783,18 +4661,18 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
 
 [[package]]
 name = "portable-atomic-util"
-version = "0.2.5"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5"
+checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3"
 dependencies = [
  "portable-atomic",
 ]
 
 [[package]]
 name = "potential_utf"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77"
+checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"
 dependencies = [
  "zerovec",
 ]
@@ -4826,9 +4704,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro-crate"
-version = "3.4.0"
+version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983"
+checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
 dependencies = [
  "toml_edit",
 ]
@@ -4935,14 +4813,14 @@ dependencies = [
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.13"
+version = "0.11.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31"
+checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
 dependencies = [
  "bytes",
  "getrandom 0.3.4",
  "lru-slab",
- "rand 0.9.2",
+ "rand 0.9.4",
  "ring",
  "rustc-hash",
  "rustls",
@@ -5008,9 +4886,9 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.9.2"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
+checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
 dependencies = [
  "rand_chacha 0.9.0",
  "rand_core 0.9.5",
@@ -5054,16 +4932,6 @@ dependencies = [
  "getrandom 0.3.4",
 ]
 
-[[package]]
-name = "rand_distr"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
-dependencies = [
- "num-traits",
- "rand 0.8.5",
-]
-
 [[package]]
 name = "rand_distr"
 version = "0.5.1"
@@ -5071,7 +4939,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
 dependencies = [
  "num-traits",
- "rand 0.9.2",
+ "rand 0.9.4",
 ]
 
 [[package]]
@@ -5092,7 +4960,7 @@ dependencies = [
  "ahash",
  "brotli",
  "paste",
- "rand 0.9.2",
+ "rand 0.9.4",
  "unicase",
 ]
 
@@ -5110,9 +4978,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
 
 [[package]]
 name = "rayon"
-version = "1.11.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
 dependencies = [
  "either",
  "rayon-core",
@@ -5355,9 +5223,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-hash"
-version = "2.1.1"
+version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
 
 [[package]]
 name = "rustc_version"
@@ -5368,19 +5236,6 @@ dependencies = [
  "semver",
 ]
 
-[[package]]
-name = "rustix"
-version = "0.38.44"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
-dependencies = [
- "bitflags",
- "errno",
- "libc",
- "linux-raw-sys 0.4.15",
- "windows-sys 0.59.0",
-]
-
 [[package]]
 name = "rustix"
 version = "1.1.4"
@@ -5390,15 +5245,15 @@ dependencies = [
  "bitflags",
  "errno",
  "libc",
- "linux-raw-sys 0.12.1",
+ "linux-raw-sys",
  "windows-sys 0.61.2",
 ]
 
 [[package]]
 name = "rustls"
-version = "0.23.37"
+version = "0.23.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
+checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21"
 dependencies = [
  "aws-lc-rs",
  "once_cell",
@@ -5442,9 +5297,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.103.9"
+version = "0.103.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53"
+checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06"
 dependencies = [
  "aws-lc-rs",
  "ring",
@@ -5484,9 +5339,9 @@ dependencies = [
 
 [[package]]
 name = "schannel"
-version = "0.1.28"
+version = "0.1.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1"
+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
 dependencies = [
  "windows-sys 0.61.2",
 ]
@@ -5563,9 +5418,9 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.27"
+version = "1.0.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
 dependencies = [
  "serde",
  "serde_core",
@@ -5697,7 +5552,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "digest",
 ]
 
@@ -5708,7 +5563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "digest",
 ]
 
@@ -5749,9 +5604,9 @@ dependencies = [
 
 [[package]]
 name = "simd-adler32"
-version = "0.3.8"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
+checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
 
 [[package]]
 name = "simdutf8"
@@ -5777,15 +5632,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
 
-[[package]]
-name = "sketches-ddsketch"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "slab"
 version = "0.4.12"
@@ -5827,21 +5673,21 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
 
 [[package]]
 name = "socket2"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0"
+checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
 dependencies = [
  "libc",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
 name = "spade"
-version = "2.15.0"
+version = "2.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb313e1c8afee5b5647e00ee0fe6855e3d529eb863a0fdae1d60006c4d1e9990"
+checksum = "9699399fd9349b00b184f5635b074f9ec93afffef30c853f8c875b32c0f8c7fa"
 dependencies = [
- "hashbrown 0.15.5",
+ "hashbrown 0.16.1",
  "num-traits",
  "robust",
  "smallvec",
@@ -5902,12 +5748,6 @@ version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978"
 
-[[package]]
-name = "strsim"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
-
 [[package]]
 name = "strum"
 version = "0.26.3"
@@ -6009,152 +5849,6 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
 
-[[package]]
-name = "tantivy"
-version = "0.24.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64a966cb0e76e311f09cf18507c9af192f15d34886ee43d7ba7c7e3803660c43"
-dependencies = [
- "aho-corasick",
- "arc-swap",
- "base64",
- "bitpacking",
- "bon",
- "byteorder",
- "census",
- "crc32fast",
- "crossbeam-channel",
- "downcast-rs",
- "fastdivide",
- "fnv",
- "fs4",
- "htmlescape",
- "hyperloglogplus",
- "itertools 0.14.0",
- "levenshtein_automata",
- "log",
- "lru",
- "lz4_flex 0.11.5",
- "measure_time",
- "memmap2",
- "once_cell",
- "oneshot",
- "rayon",
- "regex",
- "rust-stemmers",
- "rustc-hash",
- "serde",
- "serde_json",
- "sketches-ddsketch",
- "smallvec",
- "tantivy-bitpacker",
- "tantivy-columnar",
- "tantivy-common",
- "tantivy-fst",
- "tantivy-query-grammar",
- "tantivy-stacker",
- "tantivy-tokenizer-api",
- "tempfile",
- "thiserror 2.0.18",
- "time",
- "uuid",
- "winapi",
-]
-
-[[package]]
-name = "tantivy-bitpacker"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1adc286a39e089ae9938935cd488d7d34f14502544a36607effd2239ff0e2494"
-dependencies = [
- "bitpacking",
-]
-
-[[package]]
-name = "tantivy-columnar"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6300428e0c104c4f7db6f95b466a6f5c1b9aece094ec57cdd365337908dc7344"
-dependencies = [
- "downcast-rs",
- "fastdivide",
- "itertools 0.14.0",
- "serde",
- "tantivy-bitpacker",
- "tantivy-common",
- "tantivy-sstable",
- "tantivy-stacker",
-]
-
-[[package]]
-name = "tantivy-common"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e91b6ea6090ce03dc72c27d0619e77185d26cc3b20775966c346c6d4f7e99d7f"
-dependencies = [
- "async-trait",
- "byteorder",
- "ownedbytes",
- "serde",
- "time",
-]
-
-[[package]]
-name = "tantivy-fst"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
-dependencies = [
- "byteorder",
- "regex-syntax",
- "utf8-ranges",
-]
-
-[[package]]
-name = "tantivy-query-grammar"
-version = "0.24.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e810cdeeebca57fc3f7bfec5f85fdbea9031b2ac9b990eb5ff49b371d52bbe6a"
-dependencies = [
- "nom 7.1.3",
- "serde",
- "serde_json",
-]
-
-[[package]]
-name = "tantivy-sstable"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "709f22c08a4c90e1b36711c1c6cad5ae21b20b093e535b69b18783dd2cb99416"
-dependencies = [
- "futures-util",
- "itertools 0.14.0",
- "tantivy-bitpacker",
- "tantivy-common",
- "tantivy-fst",
- "zstd",
-]
-
-[[package]]
-name = "tantivy-stacker"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bcdebb267671311d1e8891fd9d1301803fdb8ad21ba22e0a30d0cab49ba59c1"
-dependencies = [
- "murmurhash32",
- "rand_distr 0.4.3",
- "tantivy-common",
-]
-
-[[package]]
-name = "tantivy-tokenizer-api"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfa942fcee81e213e09715bbce8734ae2180070b97b33839a795ba1de201547d"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "tap"
 version = "1.0.1"
@@ -6163,14 +5857,14 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "tempfile"
-version = "3.26.0"
+version = "3.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0"
+checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
  "getrandom 0.4.2",
  "once_cell",
- "rustix 1.1.4",
+ "rustix",
  "windows-sys 0.61.2",
 ]
 
@@ -6285,9 +5979,9 @@ dependencies = [
 
 [[package]]
 name = "tinystr"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869"
+checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
 dependencies = [
  "displaydoc",
  "zerovec",
@@ -6295,9 +5989,9 @@ dependencies = [
 
 [[package]]
 name = "tinyvec"
-version = "1.10.0"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
+checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
 dependencies = [
  "tinyvec_macros",
 ]
@@ -6310,9 +6004,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.50.0"
+version = "1.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d"
+checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776"
 dependencies = [
  "bytes",
  "libc",
@@ -6327,9 +6021,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "2.6.1"
+version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c"
+checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -6372,18 +6066,18 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "0.7.5+spec-1.1.0"
+version = "1.1.1+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
 dependencies = [
  "serde_core",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.23.10+spec-1.0.0"
+version = "0.25.11+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269"
+checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b"
 dependencies = [
  "indexmap",
  "toml_datetime",
@@ -6393,9 +6087,9 @@ dependencies = [
 
 [[package]]
 name = "toml_parser"
-version = "1.0.9+spec-1.1.0"
+version = "1.1.2+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4"
+checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
 dependencies = [
  "winnow",
 ]
@@ -6514,9 +6208,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-subscriber"
-version = "0.3.22"
+version = "0.3.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e"
+checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
 dependencies = [
  "matchers",
  "nu-ansi-term",
@@ -6542,7 +6236,7 @@ version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
 dependencies = [
- "rand 0.9.2",
+ "rand 0.9.4",
 ]
 
 [[package]]
@@ -6610,11 +6304,20 @@ version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
 
+[[package]]
+name = "unicode-normalization"
+version = "0.1.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
+dependencies = [
+ "tinyvec",
+]
+
 [[package]]
 name = "unicode-segmentation"
-version = "1.12.0"
+version = "1.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
+checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c"
 
 [[package]]
 name = "unicode-width"
@@ -6678,9 +6381,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.21.0"
+version = "1.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb"
+checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9"
 dependencies = [
  "getrandom 0.4.2",
  "js-sys",
@@ -6751,9 +6454,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.114"
+version = "0.2.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e"
+checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -6764,23 +6467,19 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.64"
+version = "0.4.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8"
+checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8"
 dependencies = [
- "cfg-if",
- "futures-util",
  "js-sys",
- "once_cell",
  "wasm-bindgen",
- "web-sys",
 ]
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.114"
+version = "0.2.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6"
+checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -6788,9 +6487,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.114"
+version = "0.2.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3"
+checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904"
 dependencies = [
  "bumpalo",
  "proc-macro2",
@@ -6801,9 +6500,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.114"
+version = "0.2.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16"
+checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129"
 dependencies = [
  "unicode-ident",
 ]
@@ -6857,9 +6556,9 @@ dependencies = [
 
 [[package]]
 name = "web-sys"
-version = "0.3.91"
+version = "0.3.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9"
+checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -6884,22 +6583,6 @@ dependencies = [
  "rustls-pki-types",
 ]
 
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
 [[package]]
 name = "winapi-util"
 version = "0.1.11"
@@ -6909,12 +6592,6 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-
 [[package]]
 name = "windows-core"
 version = "0.62.2"
@@ -6992,15 +6669,6 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
-[[package]]
-name = "windows-sys"
-version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
-dependencies = [
- "windows-targets 0.52.6",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.60.2"
@@ -7207,9 +6875,9 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
 
 [[package]]
 name = "winnow"
-version = "0.7.14"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829"
+checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5"
 dependencies = [
  "memchr",
 ]
@@ -7329,9 +6997,9 @@ dependencies = [
 
 [[package]]
 name = "writeable"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
+checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
 
 [[package]]
 name = "wyz"
@@ -7356,9 +7024,9 @@ checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
 
 [[package]]
 name = "yoke"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954"
+checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
 dependencies = [
  "stable_deref_trait",
  "yoke-derive",
@@ -7367,9 +7035,9 @@ dependencies = [
 
 [[package]]
 name = "yoke-derive"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
+checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -7379,18 +7047,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.40"
+version = "0.8.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5"
+checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.40"
+version = "0.8.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953"
+checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -7399,18 +7067,18 @@ dependencies = [
 
 [[package]]
 name = "zerofrom"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"
+checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df"
 dependencies = [
  "zerofrom-derive",
 ]
 
 [[package]]
 name = "zerofrom-derive"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
+checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -7426,9 +7094,9 @@ checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
 
 [[package]]
 name = "zerotrie"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851"
+checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
 dependencies = [
  "displaydoc",
  "yoke",
@@ -7437,9 +7105,9 @@ dependencies = [
 
 [[package]]
 name = "zerovec"
-version = "0.11.5"
+version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002"
+checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
 dependencies = [
  "yoke",
  "zerofrom",
@@ -7448,9 +7116,9 @@ dependencies = [
 
 [[package]]
 name = "zerovec-derive"
-version = "0.11.2"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
+checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
 dependencies = [
  "proc-macro2",
  "quote",
diff --git a/java/lance-jni/Cargo.toml b/java/lance-jni/Cargo.toml
index 62a6ff82a52..6657c70e002 100644
--- a/java/lance-jni/Cargo.toml
+++ b/java/lance-jni/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lance-jni"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 edition = "2024"
 authors = ["Lance Devs <dev@lance.org>"]
 rust-version = "1.91"
@@ -28,7 +28,9 @@ lance-core = { path = "../../rust/lance-core" }
 lance-file = { path = "../../rust/lance-file" }
 lance-table = { path = "../../rust/lance-table" }
 arrow = { version = "57.1", features = ["ffi"] }
+arrow-array = "57.1"
 arrow-schema = "57.1"
+datafusion-common = "52.1.0"
 object_store = { version = "0.12.2" }
 tokio = { version = "1.23", features = [
     "rt-multi-thread",
diff --git a/java/lance-jni/src/async_scanner.rs b/java/lance-jni/src/async_scanner.rs
new file mode 100644
index 00000000000..eada9287c47
--- /dev/null
+++ b/java/lance-jni/src/async_scanner.rs
@@ -0,0 +1,352 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::sync::Arc;
+
+use crate::RT;
+use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET};
+use crate::blocking_scanner::{ScannerOptions, build_scanner_with_options};
+use crate::dispatcher::{DISPATCHER, DispatcherMessage};
+use crate::error::Result;
+use crate::task_tracker::{TASK_TRACKER, TaskInfo};
+use arrow::ffi::FFI_ArrowSchema;
+use jni::JNIEnv;
+use jni::objects::JObject;
+use jni::sys::{jboolean, jint, jlong};
+use lance::dataset::scanner::Scanner;
+use lance_io::ffi::to_ffi_arrow_array_stream;
+
+pub const NATIVE_ASYNC_SCANNER: &str = "nativeAsyncScannerHandle";
+
+/// Async scanner that spawns Tokio tasks for non-blocking I/O
+pub struct AsyncScanner {
+    pub(crate) inner: Arc<Scanner>,
+}
+
+/// RAII guard that ensures task cleanup even on panic or early return
+///
+/// This guard prevents memory leaks in the task tracker by guaranteeing
+/// that task_id is removed from the HashMap when the guard is dropped,
+/// regardless of how the async task terminates (normal completion, panic,
+/// or cancellation).
+struct TaskCleanupGuard {
+    task_id: u64,
+}
+
+impl TaskCleanupGuard {
+    fn new(task_id: u64) -> Self {
+        Self { task_id }
+    }
+}
+
+impl Drop for TaskCleanupGuard {
+    fn drop(&mut self) {
+        // GUARANTEED to run when guard goes out of scope
+        // Works even if the task panics or returns early
+        //
+        // Note: We spawn a detached task instead of using block_on()
+        // because Drop may be called from within a tokio runtime context
+        let task_id = self.task_id;
+        RT.spawn(async move {
+            TASK_TRACKER.complete(task_id).await;
+            log::debug!("Task {} cleaned up via RAII guard", task_id);
+        });
+    }
+}
+
+impl AsyncScanner {
+    pub fn create(scanner: Scanner) -> Self {
+        Self {
+            inner: Arc::new(scanner),
+        }
+    }
+
+    /// Start an async scan task (static method to avoid holding locks)
+    pub fn start_scan_with_scanner(
+        scanner: Arc<Scanner>,
+        task_id: u64,
+        scanner_global_ref: jni::objects::GlobalRef,
+    ) {
+        // Two-phase registration to prevent race condition:
+        // 1. Pre-register with placeholder handle BEFORE spawning
+        // 2. Spawn the actual task
+        // 3. Update registration with real handle
+        // This ensures task is registered before cleanup can run
+
+        // Clone for the spawned task
+        let global_ref_for_task = scanner_global_ref.clone();
+
+        // Step 1: Pre-register with placeholder handle
+        let placeholder_handle = RT.spawn(async {
+            // Placeholder task that does nothing
+            // Will be aborted when real handle is registered
+        });
+
+        RT.block_on(async {
+            TASK_TRACKER
+                .register(
+                    task_id,
+                    TaskInfo {
+                        scanner_global_ref: scanner_global_ref.clone(),
+                        cancel_handle: placeholder_handle,
+                    },
+                )
+                .await;
+        });
+
+        // Step 2: Spawn the actual task
+        let handle = RT.spawn(async move {
+            // RAII guard ensures cleanup on normal exit, panic, or cancellation
+            let _cleanup_guard = TaskCleanupGuard::new(task_id);
+
+            let result = match scanner.try_into_stream().await {
+                Ok(stream) => {
+                    // Convert to FFI pointer
+                    match to_ffi_arrow_array_stream(stream, RT.handle().clone()) {
+                        Ok(ffi_stream) => {
+                            let ptr = Box::into_raw(Box::new(ffi_stream)) as i64;
+                            Ok(ptr)
+                        }
+                        Err(e) => Err(e.to_string()),
+                    }
+                }
+                Err(e) => Err(e.to_string()),
+            };
+
+            // Send result to dispatcher for Java completion
+            let dispatcher = match DISPATCHER.get() {
+                Some(d) => d,
+                None => {
+                    log::error!(
+                        "Dispatcher not initialized - cannot complete task {}. \
+                         This indicates a critical initialization failure.",
+                        task_id
+                    );
+                    // Clean up the FFI stream pointer to prevent memory leak
+                    if let Ok(ptr) = result {
+                        unsafe {
+                            drop(Box::from_raw(
+                                ptr as *mut arrow::ffi_stream::FFI_ArrowArrayStream,
+                            ));
+                        }
+                        log::debug!("Cleaned up FFI stream pointer for task {}", task_id);
+                    }
+                    return;
+                }
+            };
+
+            // Save the pointer before sending so we can clean up on failure
+            let result_ptr = result.as_ref().ok().copied();
+
+            if let Err(e) = dispatcher.send(DispatcherMessage {
+                scanner_global_ref: global_ref_for_task,
+                task_id,
+                result,
+            }) {
+                log::error!(
+                    "Failed to send completion message for task {}: {}",
+                    task_id,
+                    e
+                );
+                // Clean up the FFI stream pointer to prevent memory leak
+                if let Some(ptr) = result_ptr {
+                    unsafe {
+                        drop(Box::from_raw(
+                            ptr as *mut arrow::ffi_stream::FFI_ArrowArrayStream,
+                        ));
+                    }
+                    log::debug!("Cleaned up FFI stream pointer for task {}", task_id);
+                }
+            }
+
+            // _cleanup_guard.drop() called here automatically, removing task from tracker
+        });
+
+        // Step 3: Update registration with real handle
+        RT.block_on(async {
+            TASK_TRACKER.update_handle(task_id, handle).await;
+        });
+    }
+}
+
+// JNI Exports
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_ipc_AsyncScanner_createAsyncScanner<'local>(
+    mut env: JNIEnv<'local>,
+    _class: JObject<'local>,
+    jdataset: JObject<'local>,
+    fragment_ids_obj: JObject<'local>,
+    columns_obj: JObject<'local>,
+    substrait_filter_obj: JObject<'local>,
+    filter_obj: JObject<'local>,
+    batch_size_obj: JObject<'local>,
+    limit_obj: JObject<'local>,
+    offset_obj: JObject<'local>,
+    query_obj: JObject<'local>,
+    fts_query_obj: JObject<'local>,
+    prefilter: jboolean,
+    with_row_id: jboolean,
+    with_row_address: jboolean,
+    batch_readahead: jint,
+    column_orderings: JObject<'local>,
+    use_scalar_index: jboolean,
+    substrait_aggregate_obj: JObject<'local>,
+) -> JObject<'local> {
+    crate::ok_or_throw!(
+        env,
+        inner_create_async_scanner(
+            &mut env,
+            jdataset,
+            fragment_ids_obj,
+            columns_obj,
+            substrait_filter_obj,
+            filter_obj,
+            batch_size_obj,
+            limit_obj,
+            offset_obj,
+            query_obj,
+            fts_query_obj,
+            prefilter,
+            with_row_id,
+            with_row_address,
+            batch_readahead,
+            column_orderings,
+            use_scalar_index,
+            substrait_aggregate_obj,
+        )
+    )
+}
+
+#[allow(clippy::too_many_arguments)]
+fn inner_create_async_scanner<'local>(
+    env: &mut JNIEnv<'local>,
+    jdataset: JObject<'local>,
+    fragment_ids_obj: JObject<'local>,
+    columns_obj: JObject<'local>,
+    substrait_filter_obj: JObject<'local>,
+    filter_obj: JObject<'local>,
+    batch_size_obj: JObject<'local>,
+    limit_obj: JObject<'local>,
+    offset_obj: JObject<'local>,
+    query_obj: JObject<'local>,
+    fts_query_obj: JObject<'local>,
+    prefilter: jboolean,
+    with_row_id: jboolean,
+    with_row_address: jboolean,
+    batch_readahead: jint,
+    column_orderings: JObject<'local>,
+    use_scalar_index: jboolean,
+    substrait_aggregate_obj: JObject<'local>,
+) -> Result<JObject<'local>> {
+    let dataset_guard =
+        unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?;
+    let dataset = dataset_guard.inner.clone();
+    drop(dataset_guard);
+
+    let options = ScannerOptions {
+        fragment_ids_obj,
+        columns_obj,
+        substrait_filter_obj,
+        filter_obj,
+        batch_size_obj,
+        limit_obj,
+        offset_obj,
+        query_obj,
+        fts_query_obj,
+        prefilter,
+        with_row_id,
+        with_row_address,
+        batch_readahead,
+        column_orderings,
+        use_scalar_index,
+        substrait_aggregate_obj,
+    };
+
+    let scanner = build_scanner_with_options(env, &dataset, options)?;
+
+    let async_scanner = AsyncScanner::create(scanner);
+
+    // Create Java AsyncScanner object
+    let j_scanner = env.new_object("org/lance/ipc/AsyncScanner", "()V", &[])?;
+
+    // Attach native handle
+    unsafe { env.set_rust_field(&j_scanner, NATIVE_ASYNC_SCANNER, async_scanner)? };
+
+    Ok(j_scanner)
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_ipc_AsyncScanner_nativeStartScan(
+    mut env: JNIEnv,
+    j_scanner: JObject,
+    task_id: jlong,
+) {
+    ok_or_throw_without_return!(env, inner_start_scan(&mut env, j_scanner, task_id as u64));
+}
+
+fn inner_start_scan(env: &mut JNIEnv, j_scanner: JObject, task_id: u64) -> Result<()> {
+    // Create global reference first, before borrowing scanner
+    let scanner_global_ref = env.new_global_ref(&j_scanner)?;
+
+    // Clone the Arc<Scanner> and drop the MutexGuard before calling start_scan,
+    // which does block_on internally. Holding the guard across block_on risks deadlock.
+    let scanner = {
+        let guard =
+            unsafe { env.get_rust_field::<_, _, AsyncScanner>(&j_scanner, NATIVE_ASYNC_SCANNER)? };
+        guard.inner.clone()
+    };
+
+    AsyncScanner::start_scan_with_scanner(scanner, task_id, scanner_global_ref);
+    Ok(())
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_ipc_AsyncScanner_nativeCancelTask(
+    _env: JNIEnv,
+    _j_scanner: JObject,
+    task_id: jlong,
+) {
+    RT.block_on(async {
+        TASK_TRACKER.cancel(task_id as u64).await;
+    });
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_ipc_AsyncScanner_releaseNativeScanner(
+    mut env: JNIEnv,
+    j_scanner: JObject,
+) {
+    ok_or_throw_without_return!(env, inner_release_async_scanner(&mut env, j_scanner));
+}
+
+fn inner_release_async_scanner(env: &mut JNIEnv, j_scanner: JObject) -> Result<()> {
+    let _: AsyncScanner = unsafe { env.take_rust_field(j_scanner, NATIVE_ASYNC_SCANNER) }?;
+    Ok(())
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_ipc_AsyncScanner_importFfiSchema(
+    mut env: JNIEnv,
+    j_scanner: JObject,
+    schema_addr: jlong,
+) {
+    ok_or_throw_without_return!(
+        env,
+        inner_import_async_ffi_schema(&mut env, j_scanner, schema_addr)
+    );
+}
+
+fn inner_import_async_ffi_schema(
+    env: &mut JNIEnv,
+    j_scanner: JObject,
+    schema_addr: jlong,
+) -> Result<()> {
+    let scanner_guard =
+        unsafe { env.get_rust_field::<_, _, AsyncScanner>(j_scanner, NATIVE_ASYNC_SCANNER)? };
+
+    let schema = RT.block_on(scanner_guard.inner.schema())?;
+    let ffi_schema = FFI_ArrowSchema::try_from(&*schema)?;
+    unsafe { std::ptr::write_unaligned(schema_addr as *mut FFI_ArrowSchema, ffi_schema) }
+    Ok(())
+}
diff --git a/java/lance-jni/src/blocking_blob.rs b/java/lance-jni/src/blocking_blob.rs
index 4222e6b89d6..002fa817cf6 100755
--- a/java/lance-jni/src/blocking_blob.rs
+++ b/java/lance-jni/src/blocking_blob.rs
@@ -177,6 +177,41 @@ fn inner_blob_read_up_to<'local>(
     Ok(arr)
 }
 
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_BlobFile_nativeReadRange<'local>(
+    mut env: JNIEnv<'local>,
+    jblob: JObject,
+    offset: jlong,
+    len: jint,
+) -> jbyteArray {
+    ok_or_throw_with_return!(
+        env,
+        inner_blob_read_range(&mut env, jblob, offset, len).map(|arr| arr.into_raw()),
+        JByteArray::default().into_raw()
+    )
+}
+
+fn inner_blob_read_range<'local>(
+    env: &mut JNIEnv<'local>,
+    jblob: JObject,
+    offset: jlong,
+    len: jint,
+) -> Result<JByteArray<'local>> {
+    let end = (offset as u64)
+        .checked_add(len as u64)
+        .ok_or_else(|| lance_core::Error::invalid_input("offset + len overflowed".to_string()))?;
+    let bytes = {
+        let blob = unsafe { env.get_rust_field::<_, _, BlockingBlobFile>(jblob, NATIVE_BLOB) }?;
+        RT.block_on(blob.inner.read_range(offset as u64..end))?
+    };
+    let arr = env.new_byte_array(bytes.len() as jint)?;
+    let u8_slice: &[u8] = bytes.as_ref();
+    let i8_slice: &[i8] = unsafe { transmute(u8_slice) };
+
+    env.set_byte_array_region(&arr, 0, i8_slice)?;
+    Ok(arr)
+}
+
 #[unsafe(no_mangle)]
 pub extern "system" fn Java_org_lance_BlobFile_nativeSeek(
     mut env: JNIEnv,
diff --git a/java/lance-jni/src/blocking_dataset.rs b/java/lance-jni/src/blocking_dataset.rs
index 5edfffe7112..ab2333266da 100644
--- a/java/lance-jni/src/blocking_dataset.rs
+++ b/java/lance-jni/src/blocking_dataset.rs
@@ -7,8 +7,7 @@ use crate::namespace::{
     BlockingDirectoryNamespace, BlockingRestNamespace, create_java_lance_namespace,
 };
 use crate::session::{handle_from_session, session_from_handle};
-use crate::storage_options::JavaStorageOptionsProvider;
-use crate::traits::{FromJObjectWithEnv, FromJString, export_vec, import_vec};
+use crate::traits::{FromJObjectWithEnv, FromJString, export_vec, import_vec, import_vec_to_rust};
 use crate::utils::{
     build_compaction_options, extract_storage_options, extract_write_params,
     get_scalar_index_params, get_vector_index_params, to_rust_map,
@@ -38,6 +37,7 @@ use lance::dataset::{
     ColumnAlteration, CommitBuilder, Dataset, NewColumnTransform, ProjectionRequest, ReadParams,
     Version, WriteParams,
 };
+use lance::index::DatasetIndexExt;
 use lance::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore;
 use lance::io::{ObjectStore, ObjectStoreParams};
 use lance::session::Session as LanceSession;
@@ -45,13 +45,13 @@ use lance::table::format::IndexMetadata;
 use lance::table::format::{BasePath, Fragment};
 use lance_core::datatypes::Schema as LanceSchema;
 use lance_file::version::LanceFileVersion;
-use lance_index::DatasetIndexExt;
 use lance_index::IndexCriteria as RustIndexCriteria;
 use lance_index::optimize::OptimizeOptions;
+use lance_index::progress::noop_progress;
 use lance_index::scalar::btree::BTreeParameters;
 use lance_index::{IndexParams, IndexType};
 use lance_io::object_store::ObjectStoreRegistry;
-use lance_io::object_store::StorageOptionsProvider;
+use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsProvider};
 use lance_namespace::LanceNamespace;
 use lance_table::io::commit::CommitHandler;
 use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;
@@ -149,6 +149,7 @@ impl BlockingDataset {
         session: Option<Arc<LanceSession>>,
         namespace: Option<Arc<dyn LanceNamespace>>,
         table_id: Option<Vec<String>>,
+        namespace_client_managed_versioning: bool,
     ) -> Result<Self> {
         // Create storage options accessor from storage_options and provider
         let accessor = match (storage_options.is_empty(), storage_options_provider) {
@@ -190,9 +191,11 @@ impl BlockingDataset {
             builder = builder.with_serialized_manifest(serialized_manifest)?;
         }
 
-        // Set up namespace commit handler if namespace and table_id are provided
-        if let (Some(ns), Some(tid)) = (namespace, table_id) {
-            let external_store = LanceNamespaceExternalManifestStore::new(ns, tid);
+        // Set up namespace commit handler only if namespace manages versioning
+        if namespace_client_managed_versioning
+            && let (Some(namespace_client), Some(tid)) = (namespace, table_id)
+        {
+            let external_store = LanceNamespaceExternalManifestStore::new(namespace_client, tid);
             let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
                 external_manifest_store: Arc::new(external_store),
             });
@@ -236,6 +239,10 @@ impl BlockingDataset {
         Ok(version)
     }
 
+    pub fn version_id(&self) -> u64 {
+        self.inner.version_id()
+    }
+
     pub fn list_versions(&self) -> Result<Vec<Version>> {
         let versions = RT.block_on(self.inner.versions())?;
         Ok(versions)
@@ -341,6 +348,7 @@ impl BlockingDataset {
         storage_format: Option<LanceFileVersion>,
         max_retries: u32,
         skip_auto_cleanup: bool,
+        commit_handler: Option<Arc<dyn CommitHandler>>,
     ) -> Result<Self> {
         let mut builder = CommitBuilder::new(Arc::new(self.clone().inner))
             .with_store_params(store_params)
@@ -358,6 +366,9 @@ impl BlockingDataset {
         if skip_auto_cleanup {
             builder = builder.with_skip_auto_cleanup(true);
         }
+        if let Some(handler) = commit_handler {
+            builder = builder.with_commit_handler(handler);
+        }
         let new_dataset = RT.block_on(builder.execute(transaction))?;
         Ok(BlockingDataset { inner: new_dataset })
     }
@@ -402,6 +413,8 @@ pub extern "system" fn Java_org_lance_Dataset_createWithFfiSchema<'local>(
     storage_options_obj: JObject,      // Map<String, String>
     initial_bases: JObject,
     target_bases: JObject,
+    allow_external_blob_outside_bases: JObject, // Optional<Boolean>
+    blob_pack_file_size_threshold: JObject,     // Optional<Long>
 ) -> JObject<'local> {
     ok_or_throw!(
         env,
@@ -419,6 +432,8 @@ pub extern "system" fn Java_org_lance_Dataset_createWithFfiSchema<'local>(
             storage_options_obj,
             initial_bases,
             target_bases,
+            allow_external_blob_outside_bases,
+            blob_pack_file_size_threshold,
         )
     )
 }
@@ -438,6 +453,8 @@ fn inner_create_with_ffi_schema<'local>(
     storage_options_obj: JObject,      // Map<String, String>
     initial_bases: JObject,
     target_bases: JObject,
+    allow_external_blob_outside_bases: JObject, // Optional<Boolean>
+    blob_pack_file_size_threshold: JObject,     // Optional<Long>
 ) -> Result<JObject<'local>> {
     let c_schema_ptr = arrow_schema_addr as *mut FFI_ArrowSchema;
     let c_schema = unsafe { FFI_ArrowSchema::from_raw(c_schema_ptr) };
@@ -455,11 +472,13 @@ fn inner_create_with_ffi_schema<'local>(
         data_storage_version,
         enable_v2_manifest_paths,
         storage_options_obj,
-        JObject::null(), // No provider for schema-only creation
         initial_bases,
         target_bases,
+        allow_external_blob_outside_bases,
+        blob_pack_file_size_threshold,
         reader,
-        None, // No namespace for schema-only creation
+        None,  // No namespace for schema-only creation
+        false, // No managed versioning for schema-only creation
     )
 }
 
@@ -495,66 +514,28 @@ fn inner_native_migrate_manifest_paths_v2(env: &mut JNIEnv, java_dataset: JObjec
     Ok(())
 }
 
-#[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_Dataset_createWithFfiStream<'local>(
-    mut env: JNIEnv<'local>,
-    _obj: JObject,
-    arrow_array_stream_addr: jlong,
-    path: JString,
-    max_rows_per_file: JObject,        // Optional<Integer>
-    max_rows_per_group: JObject,       // Optional<Integer>
-    max_bytes_per_file: JObject,       // Optional<Long>
-    mode: JObject,                     // Optional<String>
-    enable_stable_row_ids: JObject,    // Optional<Boolean>
-    data_storage_version: JObject,     // Optional<String>
-    enable_v2_manifest_paths: JObject, // Optional<Boolean>
-    storage_options_obj: JObject,      // Map<String, String>
-    initial_bases: JObject,
-    target_bases: JObject,
-) -> JObject<'local> {
-    ok_or_throw!(
-        env,
-        inner_create_with_ffi_stream(
-            &mut env,
-            arrow_array_stream_addr,
-            path,
-            max_rows_per_file,
-            max_rows_per_group,
-            max_bytes_per_file,
-            mode,
-            enable_stable_row_ids,
-            data_storage_version,
-            enable_v2_manifest_paths,
-            storage_options_obj,
-            JObject::null(),
-            initial_bases,
-            target_bases,
-            JObject::null(), // No namespace
-            JObject::null(), // No table_id
-        )
-    )
-}
-
 #[unsafe(no_mangle)]
 #[allow(clippy::too_many_arguments)]
-pub extern "system" fn Java_org_lance_Dataset_createWithFfiStreamAndProvider<'local>(
+pub extern "system" fn Java_org_lance_Dataset_createWithFfiStream<'local>(
     mut env: JNIEnv<'local>,
     _obj: JObject,
     arrow_array_stream_addr: jlong,
     path: JString,
-    max_rows_per_file: JObject,            // Optional<Integer>
-    max_rows_per_group: JObject,           // Optional<Integer>
-    max_bytes_per_file: JObject,           // Optional<Long>
-    mode: JObject,                         // Optional<String>
-    enable_stable_row_ids: JObject,        // Optional<Boolean>
-    data_storage_version: JObject,         // Optional<String>
-    enable_v2_manifest_paths: JObject,     // Optional<Boolean>
-    storage_options_obj: JObject,          // Map<String, String>
-    storage_options_provider_obj: JObject, // Optional<StorageOptionsProvider>
-    initial_bases: JObject,                // Optional<List<BasePath>>
-    target_bases: JObject,                 // Optional<List<String>>
-    namespace_obj: JObject,                // LanceNamespace (can be null)
-    table_id_obj: JObject,                 // List<String> (can be null)
+    max_rows_per_file: JObject,                    // Optional<Integer>
+    max_rows_per_group: JObject,                   // Optional<Integer>
+    max_bytes_per_file: JObject,                   // Optional<Long>
+    mode: JObject,                                 // Optional<String>
+    enable_stable_row_ids: JObject,                // Optional<Boolean>
+    data_storage_version: JObject,                 // Optional<String>
+    enable_v2_manifest_paths: JObject,             // Optional<Boolean>
+    storage_options_obj: JObject,                  // Map<String, String>
+    initial_bases: JObject,                        // Optional<List<BasePath>>
+    target_bases: JObject,                         // Optional<List<String>>
+    allow_external_blob_outside_bases: JObject,    // Optional<Boolean>
+    blob_pack_file_size_threshold: JObject,        // Optional<Long>
+    namespace_obj: JObject,                        // LanceNamespace (can be null)
+    table_id_obj: JObject,                         // List<String> (can be null)
+    namespace_client_managed_versioning: jboolean, // Whether namespace manages versioning
 ) -> JObject<'local> {
     ok_or_throw!(
         env,
@@ -570,11 +551,13 @@ pub extern "system" fn Java_org_lance_Dataset_createWithFfiStreamAndProvider<'lo
             data_storage_version,
             enable_v2_manifest_paths,
             storage_options_obj,
-            storage_options_provider_obj,
             initial_bases,
             target_bases,
+            allow_external_blob_outside_bases,
+            blob_pack_file_size_threshold,
             namespace_obj,
             table_id_obj,
+            namespace_client_managed_versioning != 0,
         )
     )
 }
@@ -584,19 +567,21 @@ fn inner_create_with_ffi_stream<'local>(
     env: &mut JNIEnv<'local>,
     arrow_array_stream_addr: jlong,
     path: JString,
-    max_rows_per_file: JObject,            // Optional<Integer>
-    max_rows_per_group: JObject,           // Optional<Integer>
-    max_bytes_per_file: JObject,           // Optional<Long>
-    mode: JObject,                         // Optional<String>
-    enable_stable_row_ids: JObject,        // Optional<Boolean>
-    data_storage_version: JObject,         // Optional<String>
-    enable_v2_manifest_paths: JObject,     // Optional<Boolean>
-    storage_options_obj: JObject,          // Map<String, String>
-    storage_options_provider_obj: JObject, // Optional<StorageOptionsProvider>
-    initial_bases: JObject,                // Optional<List<BasePath>>
-    target_bases: JObject,                 // Optional<List<String>>
-    namespace_obj: JObject,                // LanceNamespace (can be null)
-    table_id_obj: JObject,                 // List<String> (can be null)
+    max_rows_per_file: JObject,                 // Optional<Integer>
+    max_rows_per_group: JObject,                // Optional<Integer>
+    max_bytes_per_file: JObject,                // Optional<Long>
+    mode: JObject,                              // Optional<String>
+    enable_stable_row_ids: JObject,             // Optional<Boolean>
+    data_storage_version: JObject,              // Optional<String>
+    enable_v2_manifest_paths: JObject,          // Optional<Boolean>
+    storage_options_obj: JObject,               // Map<String, String>
+    initial_bases: JObject,                     // Optional<List<BasePath>>
+    target_bases: JObject,                      // Optional<List<String>>
+    allow_external_blob_outside_bases: JObject, // Optional<Boolean>
+    blob_pack_file_size_threshold: JObject,     // Optional<Long>
+    namespace_obj: JObject,                     // LanceNamespace (can be null)
+    table_id_obj: JObject,                      // List<String> (can be null)
+    namespace_client_managed_versioning: bool,  // Whether namespace manages versioning
 ) -> Result<JObject<'local>> {
     let stream_ptr = arrow_array_stream_addr as *mut FFI_ArrowArrayStream;
     let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?;
@@ -615,14 +600,21 @@ fn inner_create_with_ffi_stream<'local>(
         data_storage_version,
         enable_v2_manifest_paths,
         storage_options_obj,
-        storage_options_provider_obj,
         initial_bases,
         target_bases,
+        allow_external_blob_outside_bases,
+        blob_pack_file_size_threshold,
         reader,
         namespace_info,
+        namespace_client_managed_versioning,
     )
 }
 
+/// Creates a dataset from a record batch reader.
+///
+/// When `namespace_info` is provided, sets up the storage options provider for
+/// credential refresh. When `namespace_client_managed_versioning` is true, also sets up the
+/// commit handler for namespace-managed versioning.
 #[allow(clippy::too_many_arguments)]
 fn create_dataset<'local>(
     env: &mut JNIEnv<'local>,
@@ -635,11 +627,13 @@ fn create_dataset<'local>(
     data_storage_version: JObject,
     enable_v2_manifest_paths: JObject,
     storage_options_obj: JObject,
-    storage_options_provider_obj: JObject, // Optional<StorageOptionsProvider>
     initial_bases: JObject,
     target_bases: JObject,
+    allow_external_blob_outside_bases: JObject,
+    blob_pack_file_size_threshold: JObject,
     reader: impl RecordBatchReader + Send + 'static,
     namespace_info: Option<(Arc<dyn LanceNamespace>, Vec<String>)>,
+    namespace_client_managed_versioning: bool,
 ) -> Result<JObject<'local>> {
     let path_str = path.extract(env)?;
 
@@ -653,18 +647,47 @@ fn create_dataset<'local>(
         &data_storage_version,
         Some(&enable_v2_manifest_paths),
         &storage_options_obj,
-        &storage_options_provider_obj,
         &initial_bases,
         &target_bases,
+        &allow_external_blob_outside_bases,
+        &blob_pack_file_size_threshold,
     )?;
 
-    // Set up namespace commit handler if provided
+    // Set up namespace commit handler and storage options provider if namespace is provided
     if let Some((namespace, table_id)) = namespace_info {
-        let external_store = LanceNamespaceExternalManifestStore::new(namespace, table_id);
-        let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
-            external_manifest_store: Arc::new(external_store),
+        // Set up commit handler only if namespace manages versioning
+        if namespace_client_managed_versioning {
+            let external_store =
+                LanceNamespaceExternalManifestStore::new(namespace.clone(), table_id.clone());
+            let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
+                external_manifest_store: Arc::new(external_store),
+            });
+            write_params.commit_handler = Some(commit_handler);
+        }
+
+        // Set up storage options provider for credential refresh
+        let provider: Arc<dyn StorageOptionsProvider> = Arc::new(
+            LanceNamespaceStorageOptionsProvider::new(namespace, table_id),
+        );
+
+        // Get existing storage options to combine with provider
+        let storage_options: HashMap<String, String> =
+            extract_storage_options(env, &storage_options_obj)?;
+
+        let accessor = if storage_options.is_empty() {
+            Arc::new(lance::io::StorageOptionsAccessor::with_provider(provider))
+        } else {
+            Arc::new(
+                lance::io::StorageOptionsAccessor::with_initial_and_provider(
+                    storage_options,
+                    provider,
+                ),
+            )
+        };
+        write_params.store_params = Some(ObjectStoreParams {
+            storage_options_accessor: Some(accessor),
+            ..Default::default()
         });
-        write_params.commit_handler = Some(commit_handler);
     }
 
     let dataset = BlockingDataset::write(reader, &path_str, Some(write_params))?;
@@ -1060,12 +1083,83 @@ fn inner_merge_index_metadata(
     RT.block_on(async {
         dataset_guard
             .inner
-            .merge_index_metadata(&index_uuid, index_type, batch_readhead)
+            .merge_index_metadata(&index_uuid, index_type, batch_readhead, noop_progress())
             .await
     })?;
     Ok(())
 }
 
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_Dataset_nativeMergeExistingIndexSegments<'local>(
+    mut env: JNIEnv<'local>,
+    java_dataset: JObject,
+    java_segments: JObject,
+) -> JObject<'local> {
+    ok_or_throw!(
+        env,
+        inner_merge_existing_index_segments(&mut env, java_dataset, java_segments)
+    )
+}
+
+fn inner_merge_existing_index_segments<'local>(
+    env: &mut JNIEnv<'local>,
+    java_dataset: JObject,
+    java_segments: JObject,
+) -> Result<JObject<'local>> {
+    let segments = import_vec_to_rust(env, &java_segments, |env, obj| obj.extract_object(env))?;
+    let merged_segment = {
+        let dataset_guard =
+            unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?;
+        RT.block_on(dataset_guard.inner.merge_existing_index_segments(segments))?
+    };
+    (&merged_segment).into_java(env)
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_Dataset_nativeCommitExistingIndexSegments<'local>(
+    mut env: JNIEnv<'local>,
+    java_dataset: JObject,
+    index_name: JString,
+    column: JString,
+    java_segments: JObject,
+) -> JObject<'local> {
+    ok_or_throw!(
+        env,
+        inner_commit_existing_index_segments(
+            &mut env,
+            java_dataset,
+            index_name,
+            column,
+            java_segments
+        )
+    )
+}
+
+fn inner_commit_existing_index_segments<'local>(
+    env: &mut JNIEnv<'local>,
+    java_dataset: JObject,
+    index_name: JString,
+    column: JString,
+    java_segments: JObject,
+) -> Result<JObject<'local>> {
+    let index_name = index_name.extract(env)?;
+    let column = column.extract(env)?;
+    let segments = import_vec_to_rust(env, &java_segments, |env, obj| obj.extract_object(env))?;
+
+    let committed = {
+        let mut dataset_guard =
+            unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?;
+        RT.block_on(dataset_guard.inner.commit_existing_index_segments(
+            &index_name,
+            &column,
+            segments,
+        ))?;
+        RT.block_on(dataset_guard.inner.load_indices_by_name(&index_name))?
+    };
+
+    export_vec(env, &committed)
+}
+
 #[unsafe(no_mangle)]
 pub extern "system" fn Java_org_lance_Dataset_nativeOptimizeIndices(
     mut env: JNIEnv,
@@ -1126,12 +1220,12 @@ pub extern "system" fn Java_org_lance_Dataset_openNative<'local>(
     block_size_obj: JObject, // Optional<Integer>
     index_cache_size_bytes: jlong,
     metadata_cache_size_bytes: jlong,
-    storage_options_obj: JObject,          // Map<String, String>
-    serialized_manifest: JObject,          // Optional<ByteBuffer>
-    storage_options_provider_obj: JObject, // Optional<StorageOptionsProvider>
-    session_handle: jlong,                 // Session handle, 0 means no session
-    namespace_obj: JObject,                // LanceNamespace object, null if no namespace
-    table_id_obj: JObject,                 // List<String>, null if no namespace
+    storage_options_obj: JObject,                  // Map<String, String>
+    serialized_manifest: JObject,                  // Optional<ByteBuffer>
+    session_handle: jlong,                         // Session handle, 0 means no session
+    namespace_obj: JObject,                        // LanceNamespace object, null if no namespace
+    table_id_obj: JObject,                         // List<String>, null if no namespace
+    namespace_client_managed_versioning: jboolean, // Whether namespace manages versioning
 ) -> JObject<'local> {
     ok_or_throw!(
         env,
@@ -1144,10 +1238,10 @@ pub extern "system" fn Java_org_lance_Dataset_openNative<'local>(
             metadata_cache_size_bytes,
             storage_options_obj,
             serialized_manifest,
-            storage_options_provider_obj,
             session_handle,
             namespace_obj,
             table_id_obj,
+            namespace_client_managed_versioning != 0,
         )
     )
 }
@@ -1160,12 +1254,12 @@ fn inner_open_native<'local>(
     block_size_obj: JObject, // Optional<Integer>
     index_cache_size_bytes: jlong,
     metadata_cache_size_bytes: jlong,
-    storage_options_obj: JObject,          // Map<String, String>
-    serialized_manifest: JObject,          // Optional<ByteBuffer>
-    storage_options_provider_obj: JObject, // Optional<StorageOptionsProvider>
-    session_handle: jlong,                 // Session handle, 0 means no session
-    namespace_obj: JObject,                // LanceNamespace object, null if no namespace
-    table_id_obj: JObject,                 // List<String>, null if no namespace
+    storage_options_obj: JObject,              // Map<String, String>
+    serialized_manifest: JObject,              // Optional<ByteBuffer>
+    session_handle: jlong,                     // Session handle, 0 means no session
+    namespace_obj: JObject,                    // LanceNamespace object, null if no namespace
+    table_id_obj: JObject,                     // List<String>, null if no namespace
+    namespace_client_managed_versioning: bool, // Whether namespace manages versioning
 ) -> Result<JObject<'local>> {
     let path_str: String = path.extract(env)?;
     let version = env.get_u64_opt(&version_obj)?;
@@ -1173,15 +1267,6 @@ fn inner_open_native<'local>(
     let jmap = JMap::from_env(env, &storage_options_obj)?;
     let storage_options = to_rust_map(env, &jmap)?;
 
-    // Extract storage options provider first (before get_bytes_opt which borrows env)
-    let storage_options_provider = env
-        .get_optional(&storage_options_provider_obj, |env, provider_obj| {
-            JavaStorageOptionsProvider::new(env, provider_obj)
-        })?;
-
-    let storage_options_provider_arc =
-        storage_options_provider.map(|v| Arc::new(v) as Arc<dyn StorageOptionsProvider>);
-
     // Extract namespace and table_id if provided (before get_bytes_opt which holds borrow)
     let namespace_info = extract_namespace_info(env, &namespace_obj, &table_id_obj)?;
     let (namespace, table_id) = match namespace_info {
@@ -1189,6 +1274,15 @@ fn inner_open_native<'local>(
         None => (None, None),
     };
 
+    // When namespace is provided, automatically create a storage options provider
+    // for credential refresh
+    let storage_options_provider_arc: Option<Arc<dyn StorageOptionsProvider>> =
+        if let (Some(ns), Some(tid)) = (namespace.clone(), table_id.clone()) {
+            Some(Arc::new(LanceNamespaceStorageOptionsProvider::new(ns, tid)))
+        } else {
+            None
+        };
+
     let serialized_manifest = env.get_bytes_opt(&serialized_manifest)?;
 
     // Convert session handle to Arc<LanceSession> if provided
@@ -1206,6 +1300,7 @@ fn inner_open_native<'local>(
         session,
         namespace,
         table_id,
+        namespace_client_managed_versioning,
     )?;
     dataset.into_java(env)
 }
@@ -1252,20 +1347,21 @@ pub(crate) fn extract_namespace_info(
         return Ok(None);
     }
 
-    let namespace: Arc<dyn LanceNamespace> = if is_directory_namespace(env, namespace_obj)? {
+    let namespace_client: Arc<dyn LanceNamespace> = if is_directory_namespace(env, namespace_obj)? {
         let native_handle = get_native_namespace_handle(env, namespace_obj)?;
-        let ns = unsafe { &*(native_handle as *const BlockingDirectoryNamespace) };
-        ns.inner.clone()
+        let dir_namespace_client =
+            unsafe { &*(native_handle as *const BlockingDirectoryNamespace) };
+        dir_namespace_client.inner.clone()
     } else if is_rest_namespace(env, namespace_obj)? {
         let native_handle = get_native_namespace_handle(env, namespace_obj)?;
-        let ns = unsafe { &*(native_handle as *const BlockingRestNamespace) };
-        ns.inner.clone()
+        let rest_namespace_client = unsafe { &*(native_handle as *const BlockingRestNamespace) };
+        rest_namespace_client.inner.clone()
     } else {
         create_java_lance_namespace(env, namespace_obj)?
     };
 
     let table_id = env.get_strings(table_id_obj)?;
-    Ok(Some((namespace, table_id)))
+    Ok(Some((namespace_client, table_id)))
 }
 
 #[unsafe(no_mangle)]
@@ -1445,6 +1541,20 @@ fn inner_get_version<'local>(
     version.into_java(env)
 }
 
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_Dataset_nativeGetVersionId(
+    mut env: JNIEnv,
+    java_dataset: JObject,
+) -> jlong {
+    ok_or_throw_with_return!(env, inner_get_version_id(&mut env, java_dataset), -1) as jlong
+}
+
+fn inner_get_version_id(env: &mut JNIEnv, java_dataset: JObject) -> Result<u64> {
+    let dataset_guard =
+        unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?;
+    Ok(dataset_guard.version_id())
+}
+
 #[unsafe(no_mangle)]
 pub extern "system" fn Java_org_lance_Dataset_nativeGetLatestVersionId(
     mut env: JNIEnv,
@@ -1769,6 +1879,20 @@ fn inner_get_config<'local>(
     Ok(java_hashmap)
 }
 
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_Dataset_nativeHasStableRowIds(
+    mut env: JNIEnv,
+    java_dataset: JObject,
+) -> jboolean {
+    ok_or_throw_with_return!(env, inner_has_stable_row_ids(&mut env, java_dataset), 0u8)
+}
+
+fn inner_has_stable_row_ids(env: &mut JNIEnv, java_dataset: JObject) -> Result<u8> {
+    let dataset_guard =
+        unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?;
+    Ok(dataset_guard.inner.manifest().uses_stable_row_ids() as u8)
+}
+
 #[unsafe(no_mangle)]
 pub extern "system" fn Java_org_lance_Dataset_nativeGetLanceFileFormatVersion<'local>(
     mut env: JNIEnv<'local>,
@@ -2321,12 +2445,32 @@ fn inner_list_branches<'local>(
         } else {
             JObject::null()
         };
+        let jbranch_identifier = env.new_object(
+            "java/util/ArrayList",
+            "(I)V",
+            &[JValue::Int(contents.identifier.version_mapping.len() as i32)],
+        )?;
+        for (version, uuid) in contents.identifier.version_mapping.iter() {
+            let juuid = env.new_string(uuid)?;
+            let jmapping = env.new_object(
+                "org/lance/Branch$BranchVersionMapping",
+                "(JLjava/lang/String;)V",
+                &[JValue::Long(*version as i64), JValue::Object(&juuid)],
+            )?;
+            env.call_method(
+                &jbranch_identifier,
+                "add",
+                "(Ljava/lang/Object;)Z",
+                &[JValue::Object(&jmapping)],
+            )?;
+        }
         let jbranch = env.new_object(
             "org/lance/Branch",
-            "(Ljava/lang/String;Ljava/lang/String;JJI)V",
+            "(Ljava/lang/String;Ljava/lang/String;Ljava/util/List;JJI)V",
             &[
                 JValue::Object(&jname),
                 JValue::Object(&jparent),
+                JValue::Object(&jbranch_identifier),
                 JValue::Long(contents.parent_version as i64),
                 JValue::Long(contents.create_at as i64),
                 JValue::Int(contents.manifest_size as i32),
@@ -2541,7 +2685,12 @@ fn inner_compact(
     java_dataset: JObject,
     compaction_options: JObject, // CompactionOptions
 ) -> Result<()> {
-    let rust_options = convert_java_compaction_options_to_rust(env, compaction_options)?;
+    let config = {
+        let dataset =
+            unsafe { env.get_rust_field::<_, _, BlockingDataset>(&java_dataset, NATIVE_DATASET) }?;
+        dataset.inner.manifest.config.clone()
+    };
+    let rust_options = convert_java_compaction_options_to_rust(env, compaction_options, &config)?;
     let mut dataset_guard =
         unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?;
     dataset_guard.compact(rust_options)?;
@@ -2551,6 +2700,7 @@ fn inner_compact(
 fn convert_java_compaction_options_to_rust(
     env: &mut JNIEnv,
     java_options: JObject,
+    config: &std::collections::HashMap<String, String>,
 ) -> Result<RustCompactionOptions> {
     let target_rows_per_fragment = env
         .call_method(
@@ -2627,6 +2777,14 @@ fn convert_java_compaction_options_to_rust(
             &[],
         )?
         .l()?;
+    let max_source_fragments = env
+        .call_method(
+            &java_options,
+            "getMaxSourceFragments",
+            "()Ljava/util/Optional;",
+            &[],
+        )?
+        .l()?;
 
     build_compaction_options(
         env,
@@ -2640,6 +2798,8 @@ fn convert_java_compaction_options_to_rust(
         &defer_index_remap,
         &compaction_mode,
         &binary_copy_read_batch_bytes,
+        &max_source_fragments,
+        config,
     )
 }
 
@@ -2931,3 +3091,196 @@ fn inner_get_session_handle(env: &mut JNIEnv, java_dataset: JObject) -> Result<j
     let session = dataset_guard.inner.session();
     Ok(handle_from_session(session))
 }
+
+/////////////////////////////////
+// Zonemap Stats Methods       //
+/////////////////////////////////
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_Dataset_nativeGetZonemapStats<'local>(
+    mut env: JNIEnv<'local>,
+    java_dataset: JObject,
+    jcolumn_name: JString,
+) -> JObject<'local> {
+    ok_or_throw!(
+        env,
+        inner_get_zonemap_stats(&mut env, java_dataset, jcolumn_name)
+    )
+}
+
+fn inner_get_zonemap_stats<'local>(
+    env: &mut JNIEnv<'local>,
+    java_dataset: JObject,
+    jcolumn_name: JString,
+) -> Result<JObject<'local>> {
+    use arrow_array::Array;
+    use datafusion_common::ScalarValue;
+    use lance::dataset::index::LanceIndexStoreExt;
+    use lance::index::DatasetIndexExt;
+    use lance_index::scalar::IndexStore;
+    use lance_index::scalar::lance_format::LanceIndexStore;
+
+    let column_name: String = jcolumn_name.extract(env)?;
+
+    // 1. Get the dataset and find the zonemap index for this column
+    let zonemap_data = {
+        let dataset = {
+            let dataset_guard = unsafe {
+                env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET)
+            }?;
+            dataset_guard.inner.clone()
+        };
+        // Guard is dropped here — dataset is owned
+
+        // Find the field for the requested column (validates it exists)
+        dataset.schema().field(&column_name).ok_or_else(|| {
+            Error::input_error(format!(
+                "Column '{}' not found in dataset schema",
+                column_name
+            ))
+        })?;
+
+        // Do all async work in a single block_on call to avoid nested runtime issues
+        RT.block_on(async {
+            // Find the zonemap index for this column using describe_indices
+            let descriptions = dataset
+                .describe_indices(Some(lance_index::IndexCriteria {
+                    for_column: Some(&column_name),
+                    has_name: None,
+                    must_support_fts: false,
+                    must_support_exact_equality: false,
+                }))
+                .await
+                .map_err(Error::from)?;
+
+            let zonemap_desc = descriptions
+                .iter()
+                .find(|desc| desc.index_type().to_lowercase().contains("zonemap"));
+
+            match zonemap_desc {
+                Some(desc) => {
+                    let indices = dataset.load_indices().await.map_err(Error::from)?;
+                    let index_meta = indices.iter().find(|idx| idx.name == desc.name());
+                    match index_meta {
+                        Some(index) => {
+                            let index_store = Arc::new(
+                                LanceIndexStore::from_dataset_for_existing(&dataset, index)
+                                    .map_err(Error::from)?,
+                            );
+                            let index_file = index_store
+                                .open_index_file("zonemap.lance")
+                                .await
+                                .map_err(Error::from)?;
+                            let record_batch = index_file
+                                .read_range(0..index_file.num_rows(), None)
+                                .await
+                                .map_err(Error::from)?;
+                            Ok::<_, Error>(Some(record_batch))
+                        }
+                        None => Ok(None),
+                    }
+                }
+                None => Ok(None),
+            }
+        })?
+    };
+
+    // 3. Convert the RecordBatch to a Java ArrayList<ZoneStats>
+    let array_list = env.new_object("java/util/ArrayList", "()V", &[])?;
+
+    let record_batch = match zonemap_data {
+        Some(batch) => batch,
+        None => return Ok(array_list), // empty list if no zonemap index
+    };
+
+    if record_batch.num_rows() == 0 {
+        return Ok(array_list);
+    }
+
+    let min_col = record_batch
+        .column_by_name("min")
+        .ok_or_else(|| Error::input_error("ZoneMap index file missing 'min' column".to_string()))?;
+    let max_col = record_batch
+        .column_by_name("max")
+        .ok_or_else(|| Error::input_error("ZoneMap index file missing 'max' column".to_string()))?;
+    let null_count_col = record_batch
+        .column_by_name("null_count")
+        .ok_or_else(|| {
+            Error::input_error("ZoneMap index file missing 'null_count' column".to_string())
+        })?
+        .as_any()
+        .downcast_ref::<arrow_array::UInt32Array>()
+        .ok_or_else(|| {
+            Error::input_error("ZoneMap 'null_count' column is not UInt32".to_string())
+        })?;
+    let fragment_id_col = record_batch
+        .column_by_name("fragment_id")
+        .ok_or_else(|| {
+            Error::input_error("ZoneMap index file missing 'fragment_id' column".to_string())
+        })?
+        .as_any()
+        .downcast_ref::<arrow_array::UInt64Array>()
+        .ok_or_else(|| {
+            Error::input_error("ZoneMap 'fragment_id' column is not UInt64".to_string())
+        })?;
+    let zone_start_col = record_batch
+        .column_by_name("zone_start")
+        .ok_or_else(|| {
+            Error::input_error("ZoneMap index file missing 'zone_start' column".to_string())
+        })?
+        .as_any()
+        .downcast_ref::<arrow_array::UInt64Array>()
+        .ok_or_else(|| {
+            Error::input_error("ZoneMap 'zone_start' column is not UInt64".to_string())
+        })?;
+    let zone_length_col = record_batch
+        .column_by_name("zone_length")
+        .ok_or_else(|| {
+            Error::input_error("ZoneMap index file missing 'zone_length' column".to_string())
+        })?
+        .as_any()
+        .downcast_ref::<arrow_array::UInt64Array>()
+        .ok_or_else(|| {
+            Error::input_error("ZoneMap 'zone_length' column is not UInt64".to_string())
+        })?;
+
+    for i in 0..record_batch.num_rows() {
+        let fragment_id = fragment_id_col.value(i) as i32;
+        let zone_start = zone_start_col.value(i) as i64;
+        let zone_length = zone_length_col.value(i) as i64;
+        let null_count = null_count_col.value(i) as i64;
+
+        // Convert min/max ScalarValues to Java Comparable objects
+        let min_scalar = ScalarValue::try_from_array(min_col, i).map_err(|e| {
+            Error::input_error(format!("Failed to read min value at row {}: {}", i, e))
+        })?;
+        let max_scalar = ScalarValue::try_from_array(max_col, i).map_err(|e| {
+            Error::input_error(format!("Failed to read max value at row {}: {}", i, e))
+        })?;
+
+        let j_min = crate::utils::scalar_value_to_java(env, &min_scalar)?;
+        let j_max = crate::utils::scalar_value_to_java(env, &max_scalar)?;
+
+        let zone_stats = env.new_object(
+            "org/lance/index/scalar/ZoneStats",
+            "(IJJLjava/lang/Comparable;Ljava/lang/Comparable;J)V",
+            &[
+                JValue::Int(fragment_id),
+                JValue::Long(zone_start),
+                JValue::Long(zone_length),
+                JValue::Object(&j_min),
+                JValue::Object(&j_max),
+                JValue::Long(null_count),
+            ],
+        )?;
+
+        env.call_method(
+            &array_list,
+            "add",
+            "(Ljava/lang/Object;)Z",
+            &[JValue::Object(&zone_stats)],
+        )?;
+    }
+
+    Ok(array_list)
+}
diff --git a/java/lance-jni/src/blocking_scanner.rs b/java/lance-jni/src/blocking_scanner.rs
index 93a441f3902..5a369b98a73 100644
--- a/java/lance-jni/src/blocking_scanner.rs
+++ b/java/lance-jni/src/blocking_scanner.rs
@@ -58,7 +58,16 @@ impl BlockingScanner {
     }
 }
 
-fn build_full_text_search_query<'a>(env: &mut JNIEnv<'a>, java_obj: JObject) -> Result<FtsQuery> {
+///////////////////
+// Shared Helpers //
+///////////////////
+
+/// Build FTS query from Java FullTextQuery object
+/// Made pub(crate) to be reused by async_scanner
+pub(crate) fn build_full_text_search_query<'a>(
+    env: &mut JNIEnv<'a>,
+    java_obj: JObject,
+) -> Result<FtsQuery> {
     let type_obj = env
         .call_method(
             &java_obj,
@@ -193,88 +202,40 @@ fn build_full_text_search_query<'a>(env: &mut JNIEnv<'a>, java_obj: JObject) ->
     }
 }
 
-///////////////////
-// Write Methods //
-///////////////////
-#[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_ipc_LanceScanner_createScanner<'local>(
-    mut env: JNIEnv<'local>,
-    _reader: JObject,
-    jdataset: JObject,
-    fragment_ids_obj: JObject,        // Optional<List<Integer>>
-    columns_obj: JObject,             // Optional<List<String>>
-    substrait_filter_obj: JObject,    // Optional<ByteBuffer>
-    filter_obj: JObject,              // Optional<String>
-    batch_size_obj: JObject,          // Optional<Long>
-    limit_obj: JObject,               // Optional<Integer>
-    offset_obj: JObject,              // Optional<Integer>
-    query_obj: JObject,               // Optional<Query>
-    fts_query_obj: JObject,           // Optional<FullTextQuery>
-    prefilter: jboolean,              // boolean
-    with_row_id: jboolean,            // boolean
-    with_row_address: jboolean,       // boolean
-    batch_readahead: jint,            // int
-    column_orderings: JObject,        // Optional<List<ColumnOrdering>>
-    use_scalar_index: jboolean,       // boolean
-    substrait_aggregate_obj: JObject, // Optional<ByteBuffer>
-) -> JObject<'local> {
-    ok_or_throw!(
-        env,
-        inner_create_scanner(
-            &mut env,
-            jdataset,
-            fragment_ids_obj,
-            columns_obj,
-            substrait_filter_obj,
-            filter_obj,
-            batch_size_obj,
-            limit_obj,
-            offset_obj,
-            query_obj,
-            fts_query_obj,
-            prefilter,
-            with_row_id,
-            with_row_address,
-            batch_readahead,
-            column_orderings,
-            use_scalar_index,
-            substrait_aggregate_obj
-        )
-    )
+/// Scanner options passed from JNI - shared between blocking and async scanners
+pub(crate) struct ScannerOptions<'a> {
+    pub fragment_ids_obj: JObject<'a>,
+    pub columns_obj: JObject<'a>,
+    pub substrait_filter_obj: JObject<'a>,
+    pub filter_obj: JObject<'a>,
+    pub batch_size_obj: JObject<'a>,
+    pub limit_obj: JObject<'a>,
+    pub offset_obj: JObject<'a>,
+    pub query_obj: JObject<'a>,
+    pub fts_query_obj: JObject<'a>,
+    pub prefilter: jboolean,
+    pub with_row_id: jboolean,
+    pub with_row_address: jboolean,
+    pub batch_readahead: jint,
+    pub column_orderings: JObject<'a>,
+    pub use_scalar_index: jboolean,
+    pub substrait_aggregate_obj: JObject<'a>,
 }
 
-#[allow(clippy::too_many_arguments)]
-fn inner_create_scanner<'local>(
-    env: &mut JNIEnv<'local>,
-    jdataset: JObject,
-    fragment_ids_obj: JObject,
-    columns_obj: JObject,
-    substrait_filter_obj: JObject,
-    filter_obj: JObject,
-    batch_size_obj: JObject,
-    limit_obj: JObject,
-    offset_obj: JObject,
-    query_obj: JObject,
-    fts_query_obj: JObject,
-    prefilter: jboolean,
-    with_row_id: jboolean,
-    with_row_address: jboolean,
-    batch_readahead: jint,
-    column_orderings: JObject,
-    use_scalar_index: jboolean,
-    substrait_aggregate_obj: JObject,
-) -> Result<JObject<'local>> {
-    let fragment_ids_opt = env.get_ints_opt(&fragment_ids_obj)?;
-    let dataset_guard =
-        unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?;
-
-    let mut scanner = dataset_guard.inner.scan();
+/// Build a scanner with options applied - shared by blocking and async scanners
+pub(crate) fn build_scanner_with_options<'a>(
+    env: &mut JNIEnv<'a>,
+    dataset: &lance::Dataset,
+    options: ScannerOptions<'a>,
+) -> Result<Scanner> {
+    let mut scanner = dataset.scan();
 
     // handle fragment_ids
+    let fragment_ids_opt = env.get_ints_opt(&options.fragment_ids_obj)?;
     if let Some(fragment_ids) = fragment_ids_opt {
         let mut fragments = Vec::with_capacity(fragment_ids.len());
         for fragment_id in fragment_ids {
-            let Some(fragment) = dataset_guard.inner.get_fragment(fragment_id as usize) else {
+            let Some(fragment) = dataset.get_fragment(fragment_id as usize) else {
                 return Err(Error::input_error(format!(
                     "Fragment {fragment_id} not found"
                 )));
@@ -283,49 +244,48 @@ fn inner_create_scanner<'local>(
         }
         scanner.with_fragments(fragments);
     }
-    drop(dataset_guard);
 
-    let columns_opt = env.get_strings_opt(&columns_obj)?;
+    let columns_opt = env.get_strings_opt(&options.columns_obj)?;
     if let Some(columns) = columns_opt {
         scanner.project(&columns)?;
     };
 
-    let substrait_opt = env.get_bytes_opt(&substrait_filter_obj)?;
+    let substrait_opt = env.get_bytes_opt(&options.substrait_filter_obj)?;
     if let Some(substrait) = substrait_opt {
         RT.block_on(async { scanner.filter_substrait(substrait) })?;
     }
 
-    let filter_opt = env.get_string_opt(&filter_obj)?;
+    let filter_opt = env.get_string_opt(&options.filter_obj)?;
     if let Some(filter) = filter_opt {
         scanner.filter(filter.as_str())?;
     }
 
-    let batch_size_opt = env.get_long_opt(&batch_size_obj)?;
+    let batch_size_opt = env.get_long_opt(&options.batch_size_obj)?;
     if let Some(batch_size) = batch_size_opt {
         scanner.batch_size(batch_size as usize);
     }
 
-    let limit_opt = env.get_long_opt(&limit_obj)?;
-    let offset_opt = env.get_long_opt(&offset_obj)?;
+    let limit_opt = env.get_long_opt(&options.limit_obj)?;
+    let offset_opt = env.get_long_opt(&options.offset_obj)?;
     scanner
         .limit(limit_opt, offset_opt)
         .map_err(|err| Error::input_error(err.to_string()))?;
 
-    if with_row_id == JNI_TRUE {
+    if options.with_row_id == JNI_TRUE {
         scanner.with_row_id();
     }
 
-    if with_row_address == JNI_TRUE {
+    if options.with_row_address == JNI_TRUE {
         scanner.with_row_address();
     }
 
-    if prefilter == JNI_TRUE {
+    if options.prefilter == JNI_TRUE {
         scanner.prefilter(true);
     }
 
-    scanner.use_scalar_index(use_scalar_index == JNI_TRUE);
+    scanner.use_scalar_index(options.use_scalar_index == JNI_TRUE);
 
-    env.get_optional(&query_obj, |env, java_obj| {
+    env.get_optional(&options.query_obj, |env, java_obj| {
         // Set column and key for nearest search
         let column = env.get_string_from_method(&java_obj, "getColumn")?;
         let key_array = env.get_vec_f32_from_method(&java_obj, "getKey")?;
@@ -363,16 +323,16 @@ fn inner_create_scanner<'local>(
         Ok(())
     })?;
 
-    env.get_optional(&fts_query_obj, |env, java_obj| {
+    env.get_optional(&options.fts_query_obj, |env, java_obj| {
         let fts_query = build_full_text_search_query(env, java_obj)?;
         let full_text_query = FullTextSearchQuery::new_query(fts_query);
         scanner.full_text_search(full_text_query)?;
         Ok(())
     })?;
 
-    scanner.batch_readahead(batch_readahead as usize);
+    scanner.batch_readahead(options.batch_readahead as usize);
 
-    env.get_optional(&column_orderings, |env, java_obj| {
+    env.get_optional(&options.column_orderings, |env, java_obj| {
         let list = env.get_list(&java_obj)?;
         let mut iter = list.iter(env)?;
         let mut results = Vec::with_capacity(list.size(env)? as usize);
@@ -391,11 +351,111 @@ fn inner_create_scanner<'local>(
         Ok(())
     })?;
 
-    let substrait_aggregate_opt = env.get_bytes_opt(&substrait_aggregate_obj)?;
+    let substrait_aggregate_opt = env.get_bytes_opt(&options.substrait_aggregate_obj)?;
     if let Some(substrait_aggregate) = substrait_aggregate_opt {
         scanner.aggregate(AggregateExpr::substrait(substrait_aggregate))?;
     }
 
+    Ok(scanner)
+}
+
+///////////////////
+// Write Methods //
+///////////////////
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_ipc_LanceScanner_createScanner<'local>(
+    mut env: JNIEnv<'local>,
+    _reader: JObject<'local>,
+    jdataset: JObject<'local>,
+    fragment_ids_obj: JObject<'local>, // Optional<List<Integer>>
+    columns_obj: JObject<'local>,      // Optional<List<String>>
+    substrait_filter_obj: JObject<'local>, // Optional<ByteBuffer>
+    filter_obj: JObject<'local>,       // Optional<String>
+    batch_size_obj: JObject<'local>,   // Optional<Long>
+    limit_obj: JObject<'local>,        // Optional<Integer>
+    offset_obj: JObject<'local>,       // Optional<Integer>
+    query_obj: JObject<'local>,        // Optional<Query>
+    fts_query_obj: JObject<'local>,    // Optional<FullTextQuery>
+    prefilter: jboolean,               // boolean
+    with_row_id: jboolean,             // boolean
+    with_row_address: jboolean,        // boolean
+    batch_readahead: jint,             // int
+    column_orderings: JObject<'local>, // Optional<List<ColumnOrdering>>
+    use_scalar_index: jboolean,        // boolean
+    substrait_aggregate_obj: JObject<'local>, // Optional<ByteBuffer>
+) -> JObject<'local> {
+    ok_or_throw!(
+        env,
+        inner_create_scanner(
+            &mut env,
+            jdataset,
+            fragment_ids_obj,
+            columns_obj,
+            substrait_filter_obj,
+            filter_obj,
+            batch_size_obj,
+            limit_obj,
+            offset_obj,
+            query_obj,
+            fts_query_obj,
+            prefilter,
+            with_row_id,
+            with_row_address,
+            batch_readahead,
+            column_orderings,
+            use_scalar_index,
+            substrait_aggregate_obj
+        )
+    )
+}
+
+#[allow(clippy::too_many_arguments)]
+fn inner_create_scanner<'local>(
+    env: &mut JNIEnv<'local>,
+    jdataset: JObject<'local>,
+    fragment_ids_obj: JObject<'local>,
+    columns_obj: JObject<'local>,
+    substrait_filter_obj: JObject<'local>,
+    filter_obj: JObject<'local>,
+    batch_size_obj: JObject<'local>,
+    limit_obj: JObject<'local>,
+    offset_obj: JObject<'local>,
+    query_obj: JObject<'local>,
+    fts_query_obj: JObject<'local>,
+    prefilter: jboolean,
+    with_row_id: jboolean,
+    with_row_address: jboolean,
+    batch_readahead: jint,
+    column_orderings: JObject<'local>,
+    use_scalar_index: jboolean,
+    substrait_aggregate_obj: JObject<'local>,
+) -> Result<JObject<'local>> {
+    let dataset_guard =
+        unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?;
+    let dataset = dataset_guard.inner.clone();
+    drop(dataset_guard);
+
+    let options = ScannerOptions {
+        fragment_ids_obj,
+        columns_obj,
+        substrait_filter_obj,
+        filter_obj,
+        batch_size_obj,
+        limit_obj,
+        offset_obj,
+        query_obj,
+        fts_query_obj,
+        prefilter,
+        with_row_id,
+        with_row_address,
+        batch_readahead,
+        column_orderings,
+        use_scalar_index,
+        substrait_aggregate_obj,
+    };
+
+    let scanner = build_scanner_with_options(env, &dataset, options)?;
+
     let scanner = BlockingScanner::create(scanner);
     scanner.into_java(env)
 }
diff --git a/java/lance-jni/src/dispatcher.rs b/java/lance-jni/src/dispatcher.rs
new file mode 100644
index 00000000000..a5efadc8cea
--- /dev/null
+++ b/java/lance-jni/src/dispatcher.rs
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use jni::JavaVM;
+use jni::objects::GlobalRef;
+use std::sync::{Arc, OnceLock};
+use tokio::sync::mpsc;
+
+/// Message sent from Tokio tasks to the dispatcher thread
+pub struct DispatcherMessage {
+    pub scanner_global_ref: GlobalRef,
+    pub task_id: u64,
+    pub result: Result<i64, String>, // Ok(stream_ptr) or Err(error_msg)
+}
+
+/// Global dispatcher instance initialized in JNI_OnLoad
+pub static DISPATCHER: OnceLock<Arc<Dispatcher>> = OnceLock::new();
+
+/// Dispatcher manages a persistent JNI thread for completing Java futures
+#[derive(Debug)]
+pub struct Dispatcher {
+    tx: mpsc::UnboundedSender<DispatcherMessage>,
+}
+
+impl Dispatcher {
+    /// Initialize the dispatcher with a persistent JNI thread
+    pub fn initialize(jvm: Arc<JavaVM>) -> Arc<Self> {
+        let (tx, mut rx) = mpsc::unbounded_channel::<DispatcherMessage>();
+
+        // Spawn persistent dispatcher thread
+        std::thread::Builder::new()
+            .name("lance-jni-dispatcher".to_string())
+            .spawn(move || {
+                // Attach ONCE and never detach - this is the key optimization
+                let mut env = jvm
+                    .attach_current_thread_permanently()
+                    .expect("Failed to attach dispatcher to JVM");
+
+                log::info!("JNI dispatcher thread started");
+
+                // Cache method IDs for completeTask and failTask
+                let async_scanner_class = env
+                    .find_class("org/lance/ipc/AsyncScanner")
+                    .expect("AsyncScanner class not found");
+                let complete_method = env
+                    .get_method_id(&async_scanner_class, "completeTask", "(JJ)V")
+                    .expect("completeTask method not found");
+                let fail_method = env
+                    .get_method_id(&async_scanner_class, "failTask", "(JLjava/lang/String;)V")
+                    .expect("failTask method not found");
+
+                // Event loop: block waiting for completions
+                while let Some(msg) = rx.blocking_recv() {
+                    let scanner_obj = msg.scanner_global_ref.as_obj();
+
+                    match msg.result {
+                        Err(error) => {
+                            handle_error(&mut env, scanner_obj, fail_method, msg.task_id, &error)
+                        }
+                        Ok(result_ptr) => handle_success(
+                            &mut env,
+                            scanner_obj,
+                            complete_method,
+                            msg.task_id,
+                            result_ptr,
+                        ),
+                    }
+                }
+
+                log::info!("JNI dispatcher thread shutting down");
+            })
+            .expect("Failed to spawn dispatcher thread");
+
+        Arc::new(Self { tx })
+    }
+
+    /// Send a completion message to the dispatcher
+    pub fn send(&self, msg: DispatcherMessage) -> std::result::Result<(), String> {
+        self.tx
+            .send(msg)
+            .map_err(|e| format!("Failed to send message to dispatcher: {}", e))
+    }
+}
+
+/// Handle error completion by calling failTask on Java side
+fn handle_error(
+    env: &mut jni::JNIEnv,
+    scanner_obj: &jni::objects::JObject,
+    fail_method: jni::objects::JMethodID,
+    task_id: u64,
+    error: &str,
+) {
+    let error_jstr = match env.new_string(error) {
+        Ok(s) => s,
+        Err(e) => {
+            log::error!("Failed to create JString for error: {:?}", e);
+            let _ = env.exception_clear();
+            return;
+        }
+    };
+
+    let result = unsafe {
+        env.call_method_unchecked(
+            scanner_obj,
+            fail_method,
+            jni::signature::ReturnType::Primitive(jni::signature::Primitive::Void),
+            &[
+                jni::sys::jvalue { j: task_id as i64 },
+                jni::sys::jvalue {
+                    l: error_jstr.as_raw(),
+                },
+            ],
+        )
+    };
+
+    if let Err(e) = result {
+        log::error!("Failed to call failTask: {:?}", e);
+        // Clear any pending JNI exception to protect the dispatcher loop
+        let _ = env.exception_clear();
+    }
+}
+
+/// Handle success completion by calling completeTask on Java side
+fn handle_success(
+    env: &mut jni::JNIEnv,
+    scanner_obj: &jni::objects::JObject,
+    complete_method: jni::objects::JMethodID,
+    task_id: u64,
+    result_ptr: i64,
+) {
+    let result = unsafe {
+        env.call_method_unchecked(
+            scanner_obj,
+            complete_method,
+            jni::signature::ReturnType::Primitive(jni::signature::Primitive::Void),
+            &[
+                jni::sys::jvalue { j: task_id as i64 },
+                jni::sys::jvalue { j: result_ptr },
+            ],
+        )
+    };
+
+    if let Err(e) = result {
+        log::error!("Failed to call completeTask: {:?}", e);
+        // Clear any pending JNI exception to protect the dispatcher loop
+        let _ = env.exception_clear();
+        // Clean up the FFI stream since Java won't receive it
+        unsafe {
+            drop(Box::from_raw(
+                result_ptr as *mut arrow::ffi_stream::FFI_ArrowArrayStream,
+            ));
+        }
+        log::debug!(
+            "Cleaned up FFI stream pointer for task {} after completeTask failure",
+            task_id
+        );
+    }
+}
diff --git a/java/lance-jni/src/error.rs b/java/lance-jni/src/error.rs
index 8518dcf55be..e02203a9567 100644
--- a/java/lance-jni/src/error.rs
+++ b/java/lance-jni/src/error.rs
@@ -120,60 +120,29 @@ impl Error {
         env: &mut JNIEnv,
         code: u32,
     ) -> std::result::Result<(), ()> {
-        // Try to find and call the LanceNamespaceException constructor
-        // that takes ErrorCode and message
-        let class_name = "org/lance/namespace/errors/LanceNamespaceException";
-        let error_code_class = "org/lance/namespace/errors/ErrorCode";
+        // Use ErrorFactory.fromErrorCode(code, message) to get the specific exception subclass
+        // (e.g., TableNotFoundException, NamespaceNotFoundException, etc.)
+        let factory_class = "org/lance/namespace/errors/ErrorFactory";
 
-        // Find the ErrorCode.fromCode method
-        let error_code_cls = env.find_class(error_code_class).map_err(|_| ())?;
-        let from_code_method = env
+        let factory_cls = env.find_class(factory_class).map_err(|_| ())?;
+        let from_error_code_method = env
             .get_static_method_id(
-                &error_code_cls,
-                "fromCode",
-                "(I)Lorg/lance/namespace/errors/ErrorCode;",
+                &factory_cls,
+                "fromErrorCode",
+                "(ILjava/lang/String;)Lorg/lance/namespace/errors/LanceNamespaceException;",
             )
             .map_err(|_| ())?;
-        let error_code_obj = unsafe {
-            env.call_static_method_unchecked(
-                &error_code_cls,
-                from_code_method,
-                jni::signature::ReturnType::Object,
-                &[jni::sys::jvalue {
-                    i: code as jni::sys::jint,
-                }],
-            )
-        }
-        .map_err(|_| ())?;
-
-        let error_code = match error_code_obj {
-            jni::objects::JValueGen::Object(obj) => obj,
-            _ => return Err(()),
-        };
-
-        // Find the LanceNamespaceException class
-        let exception_cls = env.find_class(class_name).map_err(|_| ())?;
 
-        // Create message JString
         let message_str = env.new_string(&self.message).map_err(|_| ())?;
 
-        // Find constructor (ErrorCode, String)
-        let constructor = env
-            .get_method_id(
-                &exception_cls,
-                "<init>",
-                "(Lorg/lance/namespace/errors/ErrorCode;Ljava/lang/String;)V",
-            )
-            .map_err(|_| ())?;
-
-        // Create the exception object
         let exception_obj = unsafe {
-            env.new_object_unchecked(
-                &exception_cls,
-                constructor,
+            env.call_static_method_unchecked(
+                &factory_cls,
+                from_error_code_method,
+                jni::signature::ReturnType::Object,
                 &[
                     jni::sys::jvalue {
-                        l: error_code.as_raw(),
+                        i: code as jni::sys::jint,
                     },
                     jni::sys::jvalue {
                         l: message_str.as_raw(),
@@ -183,8 +152,13 @@ impl Error {
         }
         .map_err(|_| ())?;
 
+        let exception = match exception_obj {
+            jni::objects::JValueGen::Object(obj) => obj,
+            _ => return Err(()),
+        };
+
         // Throw the exception
-        env.throw(jni::objects::JThrowable::from(exception_obj))
+        env.throw(jni::objects::JThrowable::from(exception))
             .map_err(|_| ())?;
 
         Ok(())
diff --git a/java/lance-jni/src/fragment.rs b/java/lance-jni/src/fragment.rs
index 05d71946a16..6ef10ee2f7d 100644
--- a/java/lance-jni/src/fragment.rs
+++ b/java/lance-jni/src/fragment.rs
@@ -17,11 +17,17 @@ use lance_io::utils::CachedFileSize;
 use std::iter::once;
 
 use lance::dataset::fragment::FileFragment;
+use lance::io::ObjectStoreParams;
 use lance_datafusion::utils::StreamingWriteSource;
+use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsProvider};
+use std::collections::HashMap;
+use std::sync::Arc;
 
+use crate::blocking_dataset::extract_namespace_info;
 use crate::error::{Error, Result};
 use crate::ffi::JNIEnvExt;
 use crate::traits::{FromJObjectWithEnv, IntoJava, JLance, export_vec, import_vec};
+use crate::utils::extract_storage_options;
 use crate::{
     RT,
     blocking_dataset::{BlockingDataset, NATIVE_DATASET},
@@ -83,14 +89,17 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiArray<'local>(
     dataset_uri: JString,
     arrow_array_addr: jlong,
     arrow_schema_addr: jlong,
-    max_rows_per_file: JObject,            // Optional<Integer>
-    max_rows_per_group: JObject,           // Optional<Integer>
-    max_bytes_per_file: JObject,           // Optional<Long>
-    mode: JObject,                         // Optional<String>
-    enable_stable_row_ids: JObject,        // Optional<Boolean>
-    data_storage_version: JObject,         // Optional<String>
-    storage_options_obj: JObject,          // Map<String, String>
-    storage_options_provider_obj: JObject, // Optional<StorageOptionsProvider>
+    max_rows_per_file: JObject,                 // Optional<Integer>
+    max_rows_per_group: JObject,                // Optional<Integer>
+    max_bytes_per_file: JObject,                // Optional<Long>
+    mode: JObject,                              // Optional<String>
+    enable_stable_row_ids: JObject,             // Optional<Boolean>
+    data_storage_version: JObject,              // Optional<String>
+    storage_options_obj: JObject,               // Map<String, String>
+    namespace_obj: JObject,                     // LanceNamespace (can be null)
+    table_id_obj: JObject,                      // List<String> (can be null)
+    allow_external_blob_outside_bases: JObject, // Optional<Boolean>
+    blob_pack_file_size_threshold: JObject,     // Optional<Long>
 ) -> JObject<'local> {
     ok_or_throw_with_return!(
         env,
@@ -106,7 +115,10 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiArray<'local>(
             enable_stable_row_ids,
             data_storage_version,
             storage_options_obj,
-            storage_options_provider_obj,
+            namespace_obj,
+            table_id_obj,
+            allow_external_blob_outside_bases,
+            blob_pack_file_size_threshold,
         ),
         JObject::default()
     )
@@ -118,14 +130,17 @@ fn inner_create_with_ffi_array<'local>(
     dataset_uri: JString,
     arrow_array_addr: jlong,
     arrow_schema_addr: jlong,
-    max_rows_per_file: JObject,            // Optional<Integer>
-    max_rows_per_group: JObject,           // Optional<Integer>
-    max_bytes_per_file: JObject,           // Optional<Long>
-    mode: JObject,                         // Optional<String>
-    enable_stable_row_ids: JObject,        // Optional<Boolean>
-    data_storage_version: JObject,         // Optional<String>
-    storage_options_obj: JObject,          // Map<String, String>
-    storage_options_provider_obj: JObject, // Optional<StorageOptionsProvider>
+    max_rows_per_file: JObject,                 // Optional<Integer>
+    max_rows_per_group: JObject,                // Optional<Integer>
+    max_bytes_per_file: JObject,                // Optional<Long>
+    mode: JObject,                              // Optional<String>
+    enable_stable_row_ids: JObject,             // Optional<Boolean>
+    data_storage_version: JObject,              // Optional<String>
+    storage_options_obj: JObject,               // Map<String, String>
+    namespace_obj: JObject,                     // LanceNamespace (can be null)
+    table_id_obj: JObject,                      // List<String> (can be null)
+    allow_external_blob_outside_bases: JObject, // Optional<Boolean>
+    blob_pack_file_size_threshold: JObject,     // Optional<Long>
 ) -> Result<JObject<'local>> {
     let c_array_ptr = arrow_array_addr as *mut FFI_ArrowArray;
     let c_schema_ptr = arrow_schema_addr as *mut FFI_ArrowSchema;
@@ -150,7 +165,10 @@ fn inner_create_with_ffi_array<'local>(
         enable_stable_row_ids,
         data_storage_version,
         storage_options_obj,
-        storage_options_provider_obj,
+        namespace_obj,
+        table_id_obj,
+        allow_external_blob_outside_bases,
+        blob_pack_file_size_threshold,
         reader,
     )
 }
@@ -161,14 +179,17 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiStream<'a>(
     _obj: JObject,
     dataset_uri: JString,
     arrow_array_stream_addr: jlong,
-    max_rows_per_file: JObject,            // Optional<Integer>
-    max_rows_per_group: JObject,           // Optional<Integer>
-    max_bytes_per_file: JObject,           // Optional<Long>
-    mode: JObject,                         // Optional<String>
-    enable_stable_row_ids: JObject,        // Optional<Boolean>
-    data_storage_version: JObject,         // Optional<String>
-    storage_options_obj: JObject,          // Map<String, String>
-    storage_options_provider_obj: JObject, // Optional<StorageOptionsProvider>
+    max_rows_per_file: JObject,                 // Optional<Integer>
+    max_rows_per_group: JObject,                // Optional<Integer>
+    max_bytes_per_file: JObject,                // Optional<Long>
+    mode: JObject,                              // Optional<String>
+    enable_stable_row_ids: JObject,             // Optional<Boolean>
+    data_storage_version: JObject,              // Optional<String>
+    storage_options_obj: JObject,               // Map<String, String>
+    namespace_obj: JObject,                     // LanceNamespace (can be null)
+    table_id_obj: JObject,                      // List<String> (can be null)
+    allow_external_blob_outside_bases: JObject, // Optional<Boolean>
+    blob_pack_file_size_threshold: JObject,     // Optional<Long>
 ) -> JObject<'a> {
     ok_or_throw_with_return!(
         env,
@@ -183,7 +204,10 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiStream<'a>(
             enable_stable_row_ids,
             data_storage_version,
             storage_options_obj,
-            storage_options_provider_obj,
+            namespace_obj,
+            table_id_obj,
+            allow_external_blob_outside_bases,
+            blob_pack_file_size_threshold,
         ),
         JObject::null()
     )
@@ -194,14 +218,17 @@ fn inner_create_with_ffi_stream<'local>(
     env: &mut JNIEnv<'local>,
     dataset_uri: JString,
     arrow_array_stream_addr: jlong,
-    max_rows_per_file: JObject,            // Optional<Integer>
-    max_rows_per_group: JObject,           // Optional<Integer>
-    max_bytes_per_file: JObject,           // Optional<Long>
-    mode: JObject,                         // Optional<String>
-    enable_stable_row_ids: JObject,        // Optional<Boolean>
-    data_storage_version: JObject,         // Optional<String>
-    storage_options_obj: JObject,          // Map<String, String>
-    storage_options_provider_obj: JObject, // Optional<StorageOptionsProvider>
+    max_rows_per_file: JObject,                 // Optional<Integer>
+    max_rows_per_group: JObject,                // Optional<Integer>
+    max_bytes_per_file: JObject,                // Optional<Long>
+    mode: JObject,                              // Optional<String>
+    enable_stable_row_ids: JObject,             // Optional<Boolean>
+    data_storage_version: JObject,              // Optional<String>
+    storage_options_obj: JObject,               // Map<String, String>
+    namespace_obj: JObject,                     // LanceNamespace (can be null)
+    table_id_obj: JObject,                      // List<String> (can be null)
+    allow_external_blob_outside_bases: JObject, // Optional<Boolean>
+    blob_pack_file_size_threshold: JObject,     // Optional<Long>
 ) -> Result<JObject<'local>> {
     let stream_ptr = arrow_array_stream_addr as *mut FFI_ArrowArrayStream;
     let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?;
@@ -216,7 +243,10 @@ fn inner_create_with_ffi_stream<'local>(
         enable_stable_row_ids,
         data_storage_version,
         storage_options_obj,
-        storage_options_provider_obj,
+        namespace_obj,
+        table_id_obj,
+        allow_external_blob_outside_bases,
+        blob_pack_file_size_threshold,
         reader,
     )
 }
@@ -225,19 +255,22 @@ fn inner_create_with_ffi_stream<'local>(
 fn create_fragment<'a>(
     env: &mut JNIEnv<'a>,
     dataset_uri: JString,
-    max_rows_per_file: JObject,            // Optional<Integer>
-    max_rows_per_group: JObject,           // Optional<Integer>
-    max_bytes_per_file: JObject,           // Optional<Long>
-    mode: JObject,                         // Optional<String>
-    enable_stable_row_ids: JObject,        // Optional<Boolean>
-    data_storage_version: JObject,         // Optional<String>
-    storage_options_obj: JObject,          // Map<String, String>
-    storage_options_provider_obj: JObject, // Optional<StorageOptionsProvider>
+    max_rows_per_file: JObject,                 // Optional<Integer>
+    max_rows_per_group: JObject,                // Optional<Integer>
+    max_bytes_per_file: JObject,                // Optional<Long>
+    mode: JObject,                              // Optional<String>
+    enable_stable_row_ids: JObject,             // Optional<Boolean>
+    data_storage_version: JObject,              // Optional<String>
+    storage_options_obj: JObject,               // Map<String, String>
+    namespace_obj: JObject,                     // LanceNamespace (can be null)
+    table_id_obj: JObject,                      // List<String> (can be null)
+    allow_external_blob_outside_bases: JObject, // Optional<Boolean>
+    blob_pack_file_size_threshold: JObject,     // Optional<Long>
     source: impl StreamingWriteSource,
 ) -> Result<JObject<'a>> {
     let path_str = dataset_uri.extract(env)?;
 
-    let write_params = extract_write_params(
+    let mut write_params = extract_write_params(
         env,
         &max_rows_per_file,
         &max_rows_per_group,
@@ -247,11 +280,38 @@ fn create_fragment<'a>(
         &data_storage_version,
         None,
         &storage_options_obj,
-        &storage_options_provider_obj,
         &JObject::null(), // not used when creating fragments
         &JObject::null(), // not used when creating fragments
+        &allow_external_blob_outside_bases,
+        &blob_pack_file_size_threshold,
     )?;
 
+    // Set up storage options provider if namespace is provided
+    let namespace_info = extract_namespace_info(env, &namespace_obj, &table_id_obj)?;
+    if let Some((namespace, table_id)) = namespace_info {
+        let provider: Arc<dyn StorageOptionsProvider> = Arc::new(
+            LanceNamespaceStorageOptionsProvider::new(namespace, table_id),
+        );
+
+        let storage_options: HashMap<String, String> =
+            extract_storage_options(env, &storage_options_obj)?;
+
+        let accessor = if storage_options.is_empty() {
+            Arc::new(lance::io::StorageOptionsAccessor::with_provider(provider))
+        } else {
+            Arc::new(
+                lance::io::StorageOptionsAccessor::with_initial_and_provider(
+                    storage_options,
+                    provider,
+                ),
+            )
+        };
+        write_params.store_params = Some(ObjectStoreParams {
+            storage_options_accessor: Some(accessor),
+            ..Default::default()
+        });
+    }
+
     let fragments = RT.block_on(FileFragment::create_fragments(
         &path_str,
         source,
@@ -486,8 +546,8 @@ impl IntoJava for &FragmentUpdateResult {
 impl IntoJava for &DataFile {
     fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> Result<JObject<'a>> {
         let path = env.new_string(self.path.clone())?.into();
-        let fields = JLance(self.fields.clone()).into_java(env)?;
-        let column_indices = JLance(self.column_indices.clone()).into_java(env)?;
+        let fields = JLance(self.fields.to_vec()).into_java(env)?;
+        let column_indices = JLance(self.column_indices.to_vec()).into_java(env)?;
         let file_size_bytes = match self.file_size_bytes.get() {
             Some(f) => JLance(u64::from(f) as i64).into_java(env)?,
             None => JObject::null(),
@@ -725,8 +785,8 @@ impl FromJObjectWithEnv<DataFile> for JObject<'_> {
         let base_id = get_base_id(env, self)?;
         Ok(DataFile {
             path,
-            fields,
-            column_indices,
+            fields: fields.into(),
+            column_indices: column_indices.into(),
             file_major_version,
             file_minor_version,
             file_size_bytes,
diff --git a/java/lance-jni/src/lib.rs b/java/lance-jni/src/lib.rs
index 53ce125aca8..7b3d40e6c00 100644
--- a/java/lance-jni/src/lib.rs
+++ b/java/lance-jni/src/lib.rs
@@ -39,10 +39,12 @@ macro_rules! ok_or_throw_with_return {
     };
 }
 
+mod async_scanner;
 mod blocking_blob;
 mod blocking_dataset;
 mod blocking_scanner;
 mod delta;
+mod dispatcher;
 pub mod error;
 pub mod ffi;
 mod file_reader;
@@ -56,6 +58,7 @@ mod schema;
 mod session;
 mod sql;
 mod storage_options;
+mod task_tracker;
 pub mod traits;
 mod transaction;
 pub mod utils;
@@ -64,7 +67,6 @@ mod vector_trainer;
 pub use error::Error;
 pub use error::Result;
 pub use ffi::JNIEnvExt;
-pub use storage_options::JavaStorageOptionsProvider;
 
 use env_logger::{Builder, Env};
 use std::env;
@@ -151,3 +153,23 @@ pub extern "system" fn Java_org_lance_JniLoader_initLanceLogger() {
     log::set_max_level(max_level);
     // todo: add tracing
 }
+
+/// JNI_OnLoad - Called when the JVM loads the native library
+/// Initializes the global dispatcher for async operations
+#[unsafe(no_mangle)]
+pub extern "system" fn JNI_OnLoad(
+    vm: jni::JavaVM,
+    _reserved: *mut std::ffi::c_void,
+) -> jni::sys::jint {
+    let jvm_arc = Arc::new(vm);
+
+    // Initialize global dispatcher with persistent thread
+    let dispatcher = dispatcher::Dispatcher::initialize(jvm_arc);
+
+    // Set the global DISPATCHER (will panic if called more than once)
+    dispatcher::DISPATCHER
+        .set(dispatcher)
+        .expect("Dispatcher already initialized");
+
+    jni::sys::JNI_VERSION_1_8
+}
diff --git a/java/lance-jni/src/namespace.rs b/java/lance-jni/src/namespace.rs
index 856904f343a..cc913761bf8 100644
--- a/java/lance-jni/src/namespace.rs
+++ b/java/lance-jni/src/namespace.rs
@@ -8,12 +8,12 @@ use async_trait::async_trait;
 use bytes::Bytes;
 use jni::JNIEnv;
 use jni::objects::{GlobalRef, JByteArray, JMap, JObject, JString, JValue};
-use jni::sys::{jbyteArray, jlong, jstring};
+use jni::sys::{jbyteArray, jlong, jobject, jstring};
 use lance_namespace::LanceNamespace as LanceNamespaceTrait;
 use lance_namespace::models::*;
 use lance_namespace_impls::{
-    ConnectBuilder, DirectoryNamespaceBuilder, DynamicContextProvider, OperationInfo, RestAdapter,
-    RestAdapterConfig, RestNamespaceBuilder,
+    ConnectBuilder, DirectoryNamespace, DirectoryNamespaceBuilder, DynamicContextProvider,
+    OperationInfo, RestAdapter, RestAdapterConfig, RestNamespace, RestNamespaceBuilder,
 };
 use serde::{Deserialize, Serialize};
 
@@ -117,14 +117,34 @@ fn convert_java_map_to_hashmap(
     Ok(result)
 }
 
+/// Convert a Rust HashMap<String, u64> to a Java HashMap<String, Long>
+fn rust_hashmap_to_java_map<'local>(
+    env: &mut JNIEnv<'local>,
+    map: &HashMap<String, u64>,
+) -> Result<JObject<'local>> {
+    let java_map = env.new_object("java/util/HashMap", "()V", &[])?;
+    for (k, v) in map {
+        let jkey = env.new_string(k)?;
+        // Create a java.lang.Long object from the u64 value
+        let jval = env.new_object("java/lang/Long", "(J)V", &[JValue::Long(*v as i64)])?;
+        env.call_method(
+            &java_map,
+            "put",
+            "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;",
+            &[JValue::Object(&jkey), JValue::Object(&jval)],
+        )?;
+    }
+    Ok(java_map)
+}
+
 /// Blocking wrapper for DirectoryNamespace
 pub struct BlockingDirectoryNamespace {
-    pub(crate) inner: Arc<dyn LanceNamespaceTrait>,
+    pub(crate) inner: Arc<DirectoryNamespace>,
 }
 
 /// Blocking wrapper for RestNamespace
 pub struct BlockingRestNamespace {
-    pub(crate) inner: Arc<dyn LanceNamespaceTrait>,
+    pub(crate) inner: Arc<RestNamespace>,
 }
 
 // ============================================================================
@@ -1610,8 +1630,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listNamespace
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.list_namespaces(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.list_namespaces(req))
         }),
         std::ptr::null_mut()
     )
@@ -1627,8 +1647,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeNames
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.describe_namespace(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.describe_namespace(req))
         }),
         std::ptr::null_mut()
     )
@@ -1644,8 +1664,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createNamespa
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.create_namespace(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.create_namespace(req))
         }),
         std::ptr::null_mut()
     )
@@ -1661,8 +1681,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_dropNamespace
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.drop_namespace(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.drop_namespace(req))
         }),
         std::ptr::null_mut()
     )
@@ -1678,8 +1698,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_namespaceExis
 ) {
     ok_or_throw_without_return!(
         env,
-        call_namespace_void_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.namespace_exists(req))
+        call_namespace_void_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.namespace_exists(req))
         })
     )
 }
@@ -1693,8 +1713,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listTablesNat
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.list_tables(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.list_tables(req))
         }),
         std::ptr::null_mut()
     )
@@ -1710,8 +1730,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeTable
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.describe_table(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.describe_table(req))
         }),
         std::ptr::null_mut()
     )
@@ -1727,8 +1747,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_registerTable
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.register_table(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.register_table(req))
         }),
         std::ptr::null_mut()
     )
@@ -1744,8 +1764,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_tableExistsNa
 ) {
     ok_or_throw_without_return!(
         env,
-        call_namespace_void_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.table_exists(req))
+        call_namespace_void_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.table_exists(req))
         })
     )
 }
@@ -1759,8 +1779,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_dropTableNati
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.drop_table(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.drop_table(req))
         }),
         std::ptr::null_mut()
     )
@@ -1776,8 +1796,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_deregisterTab
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.deregister_table(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.deregister_table(req))
         }),
         std::ptr::null_mut()
     )
@@ -1813,7 +1833,9 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableNa
             handle,
             request_json,
             request_data,
-            |ns, req, data| { RT.block_on(ns.inner.create_table(req, data)) }
+            |namespace_client, req, data| {
+                RT.block_on(namespace_client.inner.create_table(req, data))
+            }
         ),
         std::ptr::null_mut()
     )
@@ -1829,8 +1851,25 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_declareTableN
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.declare_table(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.declare_table(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_renameTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.rename_table(req))
         }),
         std::ptr::null_mut()
     )
@@ -1852,7 +1891,9 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_insertIntoTab
             handle,
             request_json,
             request_data,
-            |ns, req, data| { RT.block_on(ns.inner.insert_into_table(req, data)) }
+            |namespace_client, req, data| {
+                RT.block_on(namespace_client.inner.insert_into_table(req, data))
+            }
         ),
         std::ptr::null_mut()
     )
@@ -1874,7 +1915,9 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_mergeInsertIn
             handle,
             request_json,
             request_data,
-            |ns, req, data| { RT.block_on(ns.inner.merge_insert_into_table(req, data)) }
+            |namespace_client, req, data| {
+                RT.block_on(namespace_client.inner.merge_insert_into_table(req, data))
+            }
         ),
         std::ptr::null_mut()
     )
@@ -1890,8 +1933,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_updateTableNa
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.update_table(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.update_table(req))
         }),
         std::ptr::null_mut()
     )
@@ -1907,8 +1950,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_deleteFromTab
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.delete_from_table(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.delete_from_table(req))
         }),
         std::ptr::null_mut()
     )
@@ -1939,8 +1982,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableIn
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.create_table_index(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.create_table_index(req))
         }),
         std::ptr::null_mut()
     )
@@ -1956,8 +1999,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listTableIndi
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.list_table_indices(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.list_table_indices(req))
         }),
         std::ptr::null_mut()
     )
@@ -1973,8 +2016,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeTable
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.describe_table_index_stats(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.describe_table_index_stats(req))
         }),
         std::ptr::null_mut()
     )
@@ -1990,8 +2033,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeTrans
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.describe_transaction(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.describe_transaction(req))
         }),
         std::ptr::null_mut()
     )
@@ -2007,8 +2050,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTransact
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.alter_transaction(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.alter_transaction(req))
         }),
         std::ptr::null_mut()
     )
@@ -2024,8 +2067,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listTableVers
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.list_table_versions(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.list_table_versions(req))
         }),
         std::ptr::null_mut()
     )
@@ -2041,8 +2084,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableVe
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.create_table_version(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.create_table_version(req))
         }),
         std::ptr::null_mut()
     )
@@ -2058,8 +2101,8 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeTable
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.describe_table_version(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.describe_table_version(req))
         }),
         std::ptr::null_mut()
     )
@@ -2075,107 +2118,67 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_batchDeleteTa
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.batch_delete_table_versions(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.batch_delete_table_versions(req))
         }),
         std::ptr::null_mut()
     )
     .into_raw()
 }
 
-// ============================================================================
-// RestNamespace JNI Functions
-// ============================================================================
-
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_createNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableScalarIndexNative(
     mut env: JNIEnv,
     _obj: JObject,
-    properties_map: JObject,
-) -> jlong {
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
     ok_or_throw_with_return!(
         env,
-        create_rest_namespace_internal(&mut env, properties_map, None),
-        0
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.create_table_scalar_index(req))
+        }),
+        std::ptr::null_mut()
     )
+    .into_raw()
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_createNativeWithProvider(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_dropTableIndexNative(
     mut env: JNIEnv,
     _obj: JObject,
-    properties_map: JObject,
-    context_provider: JObject,
-) -> jlong {
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
     ok_or_throw_with_return!(
         env,
-        create_rest_namespace_internal(&mut env, properties_map, Some(context_provider)),
-        0
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.drop_table_index(req))
+        }),
+        std::ptr::null_mut()
     )
-}
-
-fn create_rest_namespace_internal(
-    env: &mut JNIEnv,
-    properties_map: JObject,
-    context_provider: Option<JObject>,
-) -> Result<jlong> {
-    // Convert Java HashMap to Rust HashMap
-    let jmap = JMap::from_env(env, &properties_map)?;
-    let properties = to_rust_map(env, &jmap)?;
-
-    // Build RestNamespace using builder
-    let mut builder = RestNamespaceBuilder::from_properties(properties).map_err(|e| {
-        Error::runtime_error(format!("Failed to create RestNamespaceBuilder: {}", e))
-    })?;
-
-    // Add context provider if provided
-    if let Some(provider_obj) = context_provider
-        && !provider_obj.is_null()
-    {
-        let java_provider = JavaDynamicContextProvider::new(env, &provider_obj)?;
-        builder = builder.context_provider(Arc::new(java_provider));
-    }
-
-    let namespace = builder.build();
-
-    let blocking_namespace = BlockingRestNamespace {
-        inner: Arc::new(namespace),
-    };
-    let handle = Box::into_raw(Box::new(blocking_namespace)) as jlong;
-    Ok(handle)
-}
-
-#[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_releaseNative(
-    _env: JNIEnv,
-    _obj: JObject,
-    handle: jlong,
-) {
-    if handle != 0 {
-        unsafe {
-            let _ = Box::from_raw(handle as *mut BlockingRestNamespace);
-        }
-    }
+    .into_raw()
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_namespaceIdNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listAllTablesNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
+    request_json: JString,
 ) -> jstring {
-    let namespace = unsafe { &*(handle as *const BlockingRestNamespace) };
-    let namespace_id = namespace.inner.namespace_id();
     ok_or_throw_with_return!(
         env,
-        env.new_string(namespace_id).map_err(Error::from),
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.list_all_tables(req))
+        }),
         std::ptr::null_mut()
     )
     .into_raw()
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_listNamespacesNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_restoreTableNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2183,8 +2186,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listNamespacesNati
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.list_namespaces(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.restore_table(req))
         }),
         std::ptr::null_mut()
     )
@@ -2192,7 +2195,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listNamespacesNati
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeNamespaceNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_updateTableSchemaMetadataNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2200,8 +2203,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeNamespaceN
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.describe_namespace(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.update_table_schema_metadata(req))
         }),
         std::ptr::null_mut()
     )
@@ -2209,7 +2212,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeNamespaceN
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_createNamespaceNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_getTableStatsNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2217,8 +2220,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createNamespaceNat
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.create_namespace(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.get_table_stats(req))
         }),
         std::ptr::null_mut()
     )
@@ -2226,7 +2229,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createNamespaceNat
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropNamespaceNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_explainTableQueryPlanNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2234,8 +2237,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropNamespaceNativ
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.drop_namespace(req))
+        call_namespace_string_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.explain_table_query_plan(req))
         }),
         std::ptr::null_mut()
     )
@@ -2243,22 +2246,24 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropNamespaceNativ
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_namespaceExistsNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_analyzeTableQueryPlanNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
     request_json: JString,
-) {
-    ok_or_throw_without_return!(
+) -> jstring {
+    ok_or_throw_with_return!(
         env,
-        call_rest_namespace_void_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.namespace_exists(req))
-        })
+        call_namespace_string_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.analyze_table_query_plan(req))
+        }),
+        std::ptr::null_mut()
     )
+    .into_raw()
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTablesNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTableAddColumnsNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2266,8 +2271,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTablesNative(
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.list_tables(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.alter_table_add_columns(req))
         }),
         std::ptr::null_mut()
     )
@@ -2275,7 +2280,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTablesNative(
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTableAlterColumnsNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2283,8 +2288,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableNativ
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.describe_table(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.alter_table_alter_columns(req))
         }),
         std::ptr::null_mut()
     )
@@ -2292,7 +2297,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableNativ
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_registerTableNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTableDropColumnsNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2300,8 +2305,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_registerTableNativ
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.register_table(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.alter_table_drop_columns(req))
         }),
         std::ptr::null_mut()
     )
@@ -2309,22 +2314,24 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_registerTableNativ
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_tableExistsNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listTableTagsNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
     request_json: JString,
-) {
-    ok_or_throw_without_return!(
+) -> jstring {
+    ok_or_throw_with_return!(
         env,
-        call_rest_namespace_void_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.table_exists(req))
-        })
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.list_table_tags(req))
+        }),
+        std::ptr::null_mut()
     )
+    .into_raw()
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropTableNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_getTableTagVersionNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2332,8 +2339,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropTableNative(
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.drop_table(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.get_table_tag_version(req))
         }),
         std::ptr::null_mut()
     )
@@ -2341,7 +2348,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropTableNative(
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_deregisterTableNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableTagNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2349,8 +2356,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_deregisterTableNat
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.deregister_table(req))
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.create_table_tag(req))
         }),
         std::ptr::null_mut()
     )
@@ -2358,121 +2365,163 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_deregisterTableNat
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_countTableRowsNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_deleteTableTagNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
     request_json: JString,
-) -> jlong {
+) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_count_method(&mut env, handle, request_json),
-        0
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.delete_table_tag(req))
+        }),
+        std::ptr::null_mut()
     )
+    .into_raw()
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_updateTableTagNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
     request_json: JString,
-    request_data: JByteArray,
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_with_data_method(
-            &mut env,
-            handle,
-            request_json,
-            request_data,
-            |ns, req, data| { RT.block_on(ns.inner.create_table(req, data)) }
-        ),
+        call_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.update_table_tag(req))
+        }),
         std::ptr::null_mut()
     )
     .into_raw()
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_declareTableNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_retrieveOpsMetricsNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
-    request_json: JString,
-) -> jstring {
+) -> jobject {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.declare_table(req))
-        }),
+        {
+            let namespace_client = unsafe { &*(handle as *const BlockingDirectoryNamespace) };
+            let metrics = namespace_client.inner.retrieve_ops_metrics();
+            rust_hashmap_to_java_map(&mut env, &metrics)
+        },
         std::ptr::null_mut()
     )
     .into_raw()
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_renameTableNative(
+pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_resetOpsMetricsNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
-    request_json: JString,
-) -> jstring {
+) {
+    ok_or_throw_without_return!(env, {
+        let namespace_client = unsafe { &*(handle as *const BlockingDirectoryNamespace) };
+        namespace_client.inner.reset_ops_metrics();
+        Ok::<(), Error>(())
+    });
+}
+
+// ============================================================================
+// RestNamespace JNI Functions
+// ============================================================================
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_createNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    properties_map: JObject,
+) -> jlong {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.rename_table(req))
-        }),
-        std::ptr::null_mut()
+        create_rest_namespace_internal(&mut env, properties_map, None),
+        0
     )
-    .into_raw()
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_insertIntoTableNative(
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_createNativeWithProvider(
     mut env: JNIEnv,
     _obj: JObject,
-    handle: jlong,
-    request_json: JString,
-    request_data: JByteArray,
-) -> jstring {
+    properties_map: JObject,
+    context_provider: JObject,
+) -> jlong {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_with_data_method(
-            &mut env,
-            handle,
-            request_json,
-            request_data,
-            |ns, req, data| { RT.block_on(ns.inner.insert_into_table(req, data)) }
-        ),
-        std::ptr::null_mut()
+        create_rest_namespace_internal(&mut env, properties_map, Some(context_provider)),
+        0
     )
-    .into_raw()
+}
+
+fn create_rest_namespace_internal(
+    env: &mut JNIEnv,
+    properties_map: JObject,
+    context_provider: Option<JObject>,
+) -> Result<jlong> {
+    // Convert Java HashMap to Rust HashMap
+    let jmap = JMap::from_env(env, &properties_map)?;
+    let properties = to_rust_map(env, &jmap)?;
+
+    // Build RestNamespace using builder
+    let mut builder = RestNamespaceBuilder::from_properties(properties).map_err(|e| {
+        Error::runtime_error(format!("Failed to create RestNamespaceBuilder: {}", e))
+    })?;
+
+    // Add context provider if provided
+    if let Some(provider_obj) = context_provider
+        && !provider_obj.is_null()
+    {
+        let java_provider = JavaDynamicContextProvider::new(env, &provider_obj)?;
+        builder = builder.context_provider(Arc::new(java_provider));
+    }
+
+    let namespace = builder.build();
+
+    let blocking_namespace = BlockingRestNamespace {
+        inner: Arc::new(namespace),
+    };
+    let handle = Box::into_raw(Box::new(blocking_namespace)) as jlong;
+    Ok(handle)
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_mergeInsertIntoTableNative(
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_releaseNative(
+    _env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+) {
+    if handle != 0 {
+        unsafe {
+            let _ = Box::from_raw(handle as *mut BlockingRestNamespace);
+        }
+    }
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_namespaceIdNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
-    request_json: JString,
-    request_data: JByteArray,
 ) -> jstring {
+    let namespace = unsafe { &*(handle as *const BlockingRestNamespace) };
+    let namespace_id = namespace.inner.namespace_id();
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_with_data_method(
-            &mut env,
-            handle,
-            request_json,
-            request_data,
-            |ns, req, data| { RT.block_on(ns.inner.merge_insert_into_table(req, data)) }
-        ),
+        env.new_string(namespace_id).map_err(Error::from),
         std::ptr::null_mut()
     )
     .into_raw()
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_updateTableNative(
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_listNamespacesNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2480,8 +2529,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_updateTableNative(
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.update_table(req))
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.list_namespaces(req))
         }),
         std::ptr::null_mut()
     )
@@ -2489,7 +2538,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_updateTableNative(
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_deleteFromTableNative(
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeNamespaceNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2497,8 +2546,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_deleteFromTableNat
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.delete_from_table(req))
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.describe_namespace(req))
         }),
         std::ptr::null_mut()
     )
@@ -2506,22 +2555,24 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_deleteFromTableNat
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_queryTableNative(
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_createNamespaceNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
     request_json: JString,
-) -> jbyteArray {
+) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_query_method(&mut env, handle, request_json),
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.create_namespace(req))
+        }),
         std::ptr::null_mut()
     )
     .into_raw()
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableIndexNative(
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropNamespaceNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2529,8 +2580,309 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableIndexNa
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.create_table_index(req))
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.drop_namespace(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_namespaceExistsNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) {
+    ok_or_throw_without_return!(
+        env,
+        call_rest_namespace_void_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.namespace_exists(req))
+        })
+    )
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTablesNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.list_tables(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.describe_table(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_registerTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.register_table(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_tableExistsNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) {
+    ok_or_throw_without_return!(
+        env,
+        call_rest_namespace_void_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.table_exists(req))
+        })
+    )
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.drop_table(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_deregisterTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.deregister_table(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_countTableRowsNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jlong {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_count_method(&mut env, handle, request_json),
+        0
+    )
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+    request_data: JByteArray,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_with_data_method(
+            &mut env,
+            handle,
+            request_json,
+            request_data,
+            |namespace_client, req, data| {
+                RT.block_on(namespace_client.inner.create_table(req, data))
+            }
+        ),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_declareTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.declare_table(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_renameTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.rename_table(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_insertIntoTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+    request_data: JByteArray,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_with_data_method(
+            &mut env,
+            handle,
+            request_json,
+            request_data,
+            |namespace_client, req, data| {
+                RT.block_on(namespace_client.inner.insert_into_table(req, data))
+            }
+        ),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_mergeInsertIntoTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+    request_data: JByteArray,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_with_data_method(
+            &mut env,
+            handle,
+            request_json,
+            request_data,
+            |namespace_client, req, data| {
+                RT.block_on(namespace_client.inner.merge_insert_into_table(req, data))
+            }
+        ),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_updateTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.update_table(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_deleteFromTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.delete_from_table(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_queryTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jbyteArray {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_query_method(&mut env, handle, request_json),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableIndexNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.create_table_index(req))
         }),
         std::ptr::null_mut()
     )
@@ -2546,8 +2898,59 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTableIndicesNa
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.list_table_indices(req))
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.list_table_indices(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableIndexStatsNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.describe_table_index_stats(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTransactionNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.describe_transaction(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTransactionNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.alter_transaction(req))
         }),
         std::ptr::null_mut()
     )
@@ -2555,7 +2958,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTableIndicesNa
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableIndexStatsNative(
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTableVersionsNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2563,8 +2966,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableIndex
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.describe_table_index_stats(req))
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.list_table_versions(req))
         }),
         std::ptr::null_mut()
     )
@@ -2572,7 +2975,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableIndex
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTransactionNative(
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableVersionNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2580,8 +2983,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTransactio
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.describe_transaction(req))
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.create_table_version(req))
         }),
         std::ptr::null_mut()
     )
@@ -2589,7 +2992,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTransactio
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTransactionNative(
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableVersionNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2597,8 +3000,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTransactionNa
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.alter_transaction(req))
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.describe_table_version(req))
         }),
         std::ptr::null_mut()
     )
@@ -2606,7 +3009,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTransactionNa
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTableVersionsNative(
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_batchDeleteTableVersionsNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2614,8 +3017,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTableVersionsN
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.list_table_versions(req))
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.batch_delete_table_versions(req))
         }),
         std::ptr::null_mut()
     )
@@ -2623,7 +3026,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTableVersionsN
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableVersionNative(
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableScalarIndexNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2631,8 +3034,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableVersion
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.create_table_version(req))
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.create_table_scalar_index(req))
         }),
         std::ptr::null_mut()
     )
@@ -2640,7 +3043,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableVersion
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableVersionNative(
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropTableIndexNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2648,8 +3051,8 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableVersi
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.describe_table_version(req))
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.drop_table_index(req))
         }),
         std::ptr::null_mut()
     )
@@ -2657,7 +3060,221 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableVersi
 }
 
 #[unsafe(no_mangle)]
-pub extern "system" fn Java_org_lance_namespace_RestNamespace_batchDeleteTableVersionsNative(
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_listAllTablesNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.list_all_tables(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_restoreTableNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.restore_table(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_updateTableSchemaMetadataNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.update_table_schema_metadata(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_getTableStatsNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.get_table_stats(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_explainTableQueryPlanNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_string_method(
+            &mut env,
+            handle,
+            request_json,
+            |namespace_client, req| {
+                RT.block_on(namespace_client.inner.explain_table_query_plan(req))
+            }
+        ),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_analyzeTableQueryPlanNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_string_method(
+            &mut env,
+            handle,
+            request_json,
+            |namespace_client, req| {
+                RT.block_on(namespace_client.inner.analyze_table_query_plan(req))
+            }
+        ),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTableAddColumnsNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.alter_table_add_columns(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTableAlterColumnsNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.alter_table_alter_columns(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTableDropColumnsNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.alter_table_drop_columns(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTableTagsNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.list_table_tags(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_getTableTagVersionNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.get_table_tag_version(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableTagNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.create_table_tag(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_deleteTableTagNative(
     mut env: JNIEnv,
     _obj: JObject,
     handle: jlong,
@@ -2665,14 +3282,62 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_batchDeleteTableVe
 ) -> jstring {
     ok_or_throw_with_return!(
         env,
-        call_rest_namespace_method(&mut env, handle, request_json, |ns, req| {
-            RT.block_on(ns.inner.batch_delete_table_versions(req))
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.delete_table_tag(req))
         }),
         std::ptr::null_mut()
     )
     .into_raw()
 }
 
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_updateTableTagNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+    request_json: JString,
+) -> jstring {
+    ok_or_throw_with_return!(
+        env,
+        call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| {
+            RT.block_on(namespace_client.inner.update_table_tag(req))
+        }),
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_retrieveOpsMetricsNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+) -> jobject {
+    ok_or_throw_with_return!(
+        env,
+        {
+            let namespace_client = unsafe { &*(handle as *const BlockingRestNamespace) };
+            let metrics = namespace_client.inner.retrieve_ops_metrics();
+            rust_hashmap_to_java_map(&mut env, &metrics)
+        },
+        std::ptr::null_mut()
+    )
+    .into_raw()
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_namespace_RestNamespace_resetOpsMetricsNative(
+    mut env: JNIEnv,
+    _obj: JObject,
+    handle: jlong,
+) {
+    ok_or_throw_without_return!(env, {
+        let namespace_client = unsafe { &*(handle as *const BlockingRestNamespace) };
+        namespace_client.inner.reset_ops_metrics();
+        Ok::<(), Error>(())
+    });
+}
+
 // ============================================================================
 // Helper Functions
 // ============================================================================
@@ -2694,8 +3359,7 @@ where
     let request: Req = serde_json::from_str(&request_str)
         .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?;
 
-    let response = f(namespace, request)
-        .map_err(|e| Error::runtime_error(format!("Namespace operation failed: {}", e)))?;
+    let response = f(namespace, request).map_err(Error::from)?;
 
     let response_json = serde_json::to_string(&response)
         .map_err(|e| Error::runtime_error(format!("Failed to serialize response: {}", e)))?;
@@ -2719,12 +3383,32 @@ where
     let request: Req = serde_json::from_str(&request_str)
         .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?;
 
-    f(namespace, request)
-        .map_err(|e| Error::runtime_error(format!("Namespace operation failed: {}", e)))?;
+    f(namespace, request).map_err(Error::from)?;
 
     Ok(())
 }
 
+/// Helper function for methods that return String directly (not JSON-serialized) (DirectoryNamespace)
+fn call_namespace_string_method<'local, Req, F>(
+    env: &mut JNIEnv<'local>,
+    handle: jlong,
+    request_json: JString,
+    f: F,
+) -> Result<JString<'local>>
+where
+    Req: for<'de> Deserialize<'de>,
+    F: FnOnce(&BlockingDirectoryNamespace, Req) -> lance_core::Result<String>,
+{
+    let namespace = unsafe { &*(handle as *const BlockingDirectoryNamespace) };
+    let request_str: String = env.get_string(&request_json)?.into();
+    let request: Req = serde_json::from_str(&request_str)
+        .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?;
+
+    let response = f(namespace, request).map_err(Error::from)?;
+
+    env.new_string(response).map_err(Into::into)
+}
+
 /// Helper function for count methods (DirectoryNamespace)
 fn call_namespace_count_method(
     env: &mut JNIEnv,
@@ -2738,7 +3422,7 @@ fn call_namespace_count_method(
 
     let count = RT
         .block_on(namespace.inner.count_table_rows(request))
-        .map_err(|e| Error::runtime_error(format!("Count table rows failed: {}", e)))?;
+        .map_err(Error::from)?;
 
     Ok(count)
 }
@@ -2764,8 +3448,7 @@ where
     let data_vec = env.convert_byte_array(request_data)?;
     let data = bytes::Bytes::from(data_vec);
 
-    let response = f(namespace, request, data)
-        .map_err(|e| Error::runtime_error(format!("Namespace operation failed: {}", e)))?;
+    let response = f(namespace, request, data).map_err(Error::from)?;
 
     let response_json = serde_json::to_string(&response)
         .map_err(|e| Error::runtime_error(format!("Failed to serialize response: {}", e)))?;
@@ -2786,7 +3469,7 @@ fn call_namespace_query_method<'local>(
 
     let result_bytes = RT
         .block_on(namespace.inner.query_table(request))
-        .map_err(|e| Error::runtime_error(format!("Query table failed: {}", e)))?;
+        .map_err(Error::from)?;
 
     let byte_array = env.byte_array_from_slice(&result_bytes)?;
     Ok(byte_array)
@@ -2809,8 +3492,7 @@ where
     let request: Req = serde_json::from_str(&request_str)
         .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?;
 
-    let response = f(namespace, request)
-        .map_err(|e| Error::runtime_error(format!("Namespace operation failed: {}", e)))?;
+    let response = f(namespace, request).map_err(Error::from)?;
 
     let response_json = serde_json::to_string(&response)
         .map_err(|e| Error::runtime_error(format!("Failed to serialize response: {}", e)))?;
@@ -2834,12 +3516,32 @@ where
     let request: Req = serde_json::from_str(&request_str)
         .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?;
 
-    f(namespace, request)
-        .map_err(|e| Error::runtime_error(format!("Namespace operation failed: {}", e)))?;
+    f(namespace, request).map_err(Error::from)?;
 
     Ok(())
 }
 
+/// Helper function for methods that return String directly (not JSON-serialized) (RestNamespace)
+fn call_rest_namespace_string_method<'local, Req, F>(
+    env: &mut JNIEnv<'local>,
+    handle: jlong,
+    request_json: JString,
+    f: F,
+) -> Result<JString<'local>>
+where
+    Req: for<'de> Deserialize<'de>,
+    F: FnOnce(&BlockingRestNamespace, Req) -> lance_core::Result<String>,
+{
+    let namespace = unsafe { &*(handle as *const BlockingRestNamespace) };
+    let request_str: String = env.get_string(&request_json)?.into();
+    let request: Req = serde_json::from_str(&request_str)
+        .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?;
+
+    let response = f(namespace, request).map_err(Error::from)?;
+
+    env.new_string(response).map_err(Into::into)
+}
+
 /// Helper function for count methods (RestNamespace)
 fn call_rest_namespace_count_method(
     env: &mut JNIEnv,
@@ -2853,7 +3555,7 @@ fn call_rest_namespace_count_method(
 
     let count = RT
         .block_on(namespace.inner.count_table_rows(request))
-        .map_err(|e| Error::runtime_error(format!("Count table rows failed: {}", e)))?;
+        .map_err(Error::from)?;
 
     Ok(count)
 }
@@ -2879,8 +3581,7 @@ where
     let data_vec = env.convert_byte_array(request_data)?;
     let data = bytes::Bytes::from(data_vec);
 
-    let response = f(namespace, request, data)
-        .map_err(|e| Error::runtime_error(format!("Namespace operation failed: {}", e)))?;
+    let response = f(namespace, request, data).map_err(Error::from)?;
 
     let response_json = serde_json::to_string(&response)
         .map_err(|e| Error::runtime_error(format!("Failed to serialize response: {}", e)))?;
@@ -2901,7 +3602,7 @@ fn call_rest_namespace_query_method<'local>(
 
     let result_bytes = RT
         .block_on(namespace.inner.query_table(request))
-        .map_err(|e| Error::runtime_error(format!("Query table failed: {}", e)))?;
+        .map_err(Error::from)?;
 
     let byte_array = env.byte_array_from_slice(&result_bytes)?;
     Ok(byte_array)
@@ -2921,27 +3622,27 @@ pub struct BlockingRestAdapter {
 pub extern "system" fn Java_org_lance_namespace_RestAdapter_createNative(
     mut env: JNIEnv,
     _obj: JObject,
-    namespace_impl: JString,
+    namespace_client_impl: JString,
     properties_map: JObject,
     host: JString,
     port: JObject,
 ) -> jlong {
     ok_or_throw_with_return!(
         env,
-        create_rest_adapter_internal(&mut env, namespace_impl, properties_map, host, port),
+        create_rest_adapter_internal(&mut env, namespace_client_impl, properties_map, host, port),
         0
     )
 }
 
 fn create_rest_adapter_internal(
     env: &mut JNIEnv,
-    namespace_impl: JString,
+    namespace_client_impl: JString,
     properties_map: JObject,
     host: JString,
     port: JObject,
 ) -> Result<jlong> {
-    // Get namespace implementation type
-    let impl_str: String = env.get_string(&namespace_impl)?.into();
+    // Get namespace client implementation type
+    let impl_str: String = env.get_string(&namespace_client_impl)?.into();
 
     // Convert Java HashMap to Rust HashMap
     let jmap = JMap::from_env(env, &properties_map)?;
diff --git a/java/lance-jni/src/optimize.rs b/java/lance-jni/src/optimize.rs
index 9797812ef75..0ce92baeec8 100644
--- a/java/lance-jni/src/optimize.rs
+++ b/java/lance-jni/src/optimize.rs
@@ -45,6 +45,7 @@ pub extern "system" fn Java_org_lance_compaction_Compaction_nativePlanCompaction
     defer_index_remap: JObject,               // Optional<Boolean>
     compaction_mode: JObject,                 // Optional<String>
     binary_copy_read_batch_bytes: JObject,    // Optional<Long>
+    max_source_fragments: JObject,            // Optional<Long>
 ) -> JObject<'local> {
     ok_or_throw_with_return!(
         env,
@@ -60,7 +61,8 @@ pub extern "system" fn Java_org_lance_compaction_Compaction_nativePlanCompaction
             batch_size,
             defer_index_remap,
             compaction_mode,
-            binary_copy_read_batch_bytes
+            binary_copy_read_batch_bytes,
+            max_source_fragments
         ),
         JObject::null()
     )
@@ -80,7 +82,13 @@ fn inner_plan_compaction<'local>(
     defer_index_remap: JObject,               // Optional<Boolean>
     compaction_mode: JObject,                 // Optional<String>
     binary_copy_read_batch_bytes: JObject,    // Optional<Long>
+    max_source_fragments: JObject,            // Optional<Long>
 ) -> Result<JObject<'local>> {
+    let config = {
+        let dataset =
+            unsafe { env.get_rust_field::<_, _, BlockingDataset>(&java_dataset, NATIVE_DATASET) }?;
+        dataset.inner.manifest.config.clone()
+    };
     let compaction_options = build_compaction_options(
         env,
         &target_rows_per_fragment,
@@ -93,6 +101,8 @@ fn inner_plan_compaction<'local>(
         &defer_index_remap,
         &compaction_mode,
         &binary_copy_read_batch_bytes,
+        &max_source_fragments,
+        &config,
     )?;
 
     let plan = {
@@ -119,6 +129,7 @@ pub extern "system" fn Java_org_lance_compaction_Compaction_nativeCommitCompacti
     defer_index_remap: JObject,               // Optional<Boolean>
     compaction_mode: JObject,                 // Optional<String>
     binary_copy_read_batch_bytes: JObject,    // Optional<Long>
+    max_source_fragments: JObject,            // Optional<Long>
 ) -> JObject<'local> {
     ok_or_throw_with_return!(
         env,
@@ -136,6 +147,7 @@ pub extern "system" fn Java_org_lance_compaction_Compaction_nativeCommitCompacti
             defer_index_remap,
             compaction_mode,
             binary_copy_read_batch_bytes,
+            max_source_fragments,
         ),
         JObject::null()
     )
@@ -156,7 +168,13 @@ fn inner_commit_compaction<'local>(
     defer_index_remap: JObject,               // Optional<Boolean>
     compaction_mode: JObject,                 // Optional<String>
     binary_copy_read_batch_bytes: JObject,    // Optional<Long>
+    max_source_fragments: JObject,            // Optional<Long>
 ) -> Result<JObject<'local>> {
+    let config = {
+        let dataset =
+            unsafe { env.get_rust_field::<_, _, BlockingDataset>(&java_dataset, NATIVE_DATASET) }?;
+        dataset.inner.manifest.config.clone()
+    };
     let compaction_options = build_compaction_options(
         env,
         &target_rows_per_fragment,
@@ -169,6 +187,8 @@ fn inner_commit_compaction<'local>(
         &defer_index_remap,
         &compaction_mode,
         &binary_copy_read_batch_bytes,
+        &max_source_fragments,
+        &config,
     )?;
     let completed_tasks = import_vec_to_rust(env, &rewrite_results, |env, rewrite_result| {
         rewrite_result.extract_object(env)
@@ -204,6 +224,7 @@ pub extern "system" fn Java_org_lance_compaction_CompactionTask_nativeExecute<'l
     defer_index_remap: JObject,               // Optional<Boolean>
     compaction_mode: JObject,                 // Optional<String>
     binary_copy_read_batch_bytes: JObject,    // Optional<Long>
+    max_source_fragments: JObject,            // Optional<Long>
 ) -> JObject<'local> {
     ok_or_throw_with_return!(
         env,
@@ -221,7 +242,8 @@ pub extern "system" fn Java_org_lance_compaction_CompactionTask_nativeExecute<'l
             batch_size,
             defer_index_remap,
             compaction_mode,
-            binary_copy_read_batch_bytes
+            binary_copy_read_batch_bytes,
+            max_source_fragments
         ),
         JObject::null()
     )
@@ -243,8 +265,14 @@ fn inner_execute_task<'local>(
     defer_index_remap: JObject,               // Optional<Boolean>
     compaction_mode: JObject,                 // Optional<String>
     binary_copy_read_batch_bytes: JObject,    // Optional<Long>
+    max_source_fragments: JObject,            // Optional<Long>
 ) -> Result<JObject<'local>> {
     let task_data: TaskData = task_data.extract_object(env)?;
+    let config = {
+        let dataset =
+            unsafe { env.get_rust_field::<_, _, BlockingDataset>(&java_dataset, NATIVE_DATASET) }?;
+        dataset.inner.manifest.config.clone()
+    };
     let compaction_options = build_compaction_options(
         env,
         &target_rows_per_fragment,
@@ -257,6 +285,8 @@ fn inner_execute_task<'local>(
         &defer_index_remap,
         &compaction_mode,
         &binary_copy_read_batch_bytes,
+        &max_source_fragments,
+        &config,
     )?;
     let compaction_task = CompactionTask {
         task: task_data,
@@ -282,7 +312,7 @@ const REWRITE_RESULT_CLASS: &str = "org/lance/compaction/RewriteResult";
 const REWRITE_RESULT_CONSTRUCTOR_SIG: &str =
     "(Lorg/lance/compaction/CompactionMetrics;Ljava/util/List;Ljava/util/List;J[B)V";
 const COMPACTION_OPTIONS_CLASS: &str = "org/lance/compaction/CompactionOptions";
-const COMPACTION_OPTIONS_CONSTRUCTOR_SIG: &str = "(Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;)V";
+const COMPACTION_OPTIONS_CONSTRUCTOR_SIG: &str = "(Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;)V";
 
 impl IntoJava for &TaskData {
     fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> Result<JObject<'a>> {
@@ -344,6 +374,9 @@ impl IntoJava for &CompactionOptions {
         let binary_copy_read_batch_bytes =
             to_java_long_obj(env, self.binary_copy_read_batch_bytes.map(|v| v as i64))?;
         let binary_copy_read_batch_bytes_opt = to_java_optional(env, binary_copy_read_batch_bytes)?;
+        let max_source_fragments =
+            to_java_long_obj(env, self.max_source_fragments.map(|v| v as i64))?;
+        let max_source_fragments_opt = to_java_optional(env, max_source_fragments)?;
 
         Ok(env.new_object(
             COMPACTION_OPTIONS_CLASS,
@@ -359,6 +392,7 @@ impl IntoJava for &CompactionOptions {
                 JValueGen::Object(&defer_index_remap_opt),
                 JValueGen::Object(&compaction_mode_opt),
                 JValueGen::Object(&binary_copy_read_batch_bytes_opt),
+                JValueGen::Object(&max_source_fragments_opt),
             ],
         )?)
     }
diff --git a/java/lance-jni/src/session.rs b/java/lance-jni/src/session.rs
index 48f0f423478..17b59c12ecb 100644
--- a/java/lance-jni/src/session.rs
+++ b/java/lance-jni/src/session.rs
@@ -11,7 +11,6 @@ use lance::session::Session as LanceSession;
 use lance_io::object_store::ObjectStoreRegistry;
 
 use crate::error::{Error, Result};
-use crate::ok_or_throw_with_return;
 
 /// Creates a new Session and returns a handle to it.
 ///
diff --git a/java/lance-jni/src/storage_options.rs b/java/lance-jni/src/storage_options.rs
index a5e05555a81..7ec607c26e7 100644
--- a/java/lance-jni/src/storage_options.rs
+++ b/java/lance-jni/src/storage_options.rs
@@ -1,180 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::collections::HashMap;
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use jni::JNIEnv;
-use jni::objects::{JMap, JObject, JString};
-use lance_io::object_store::StorageOptionsProvider;
-
-use crate::error::Result;
-
-/// Java-implemented storage options provider
-///
-/// This wraps a Java object that implements the StorageOptionsProvider interface
-/// and forwards get_storage_options() calls to the Java implementation.
-pub struct JavaStorageOptionsProvider {
-    /// GlobalRef to the Java StorageOptionsProvider object
-    java_provider: jni::objects::GlobalRef,
-    /// JavaVM for making JNI calls
-    jvm: Arc<jni::JavaVM>,
-}
-
-impl std::fmt::Debug for JavaStorageOptionsProvider {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.provider_id())
-    }
-}
-
-impl std::fmt::Display for JavaStorageOptionsProvider {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.provider_id())
-    }
-}
-
-impl JavaStorageOptionsProvider {
-    pub fn new(env: &mut JNIEnv, java_provider: JObject) -> Result<Self> {
-        // Create a global reference to the Java object so it persists
-        let java_provider = env.new_global_ref(java_provider)?;
-
-        // Get the JavaVM for later JNI calls
-        let jvm = Arc::new(env.get_java_vm()?);
-
-        Ok(Self { java_provider, jvm })
-    }
-}
-
-#[async_trait]
-impl StorageOptionsProvider for JavaStorageOptionsProvider {
-    async fn fetch_storage_options(&self) -> lance_core::Result<Option<HashMap<String, String>>> {
-        // Spawn blocking task to call Java method
-        let java_provider = self.java_provider.clone();
-        let jvm = self.jvm.clone();
-
-        tokio::task::spawn_blocking(move || {
-            // Attach current thread to JVM
-            let mut env = jvm.attach_current_thread().map_err(|e| {
-                lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to attach to JVM: {}",
-                    e
-                ))))
-            })?;
-
-            // Call fetchStorageOptions() method on Java object
-            // Returns Map<String, String> with all storage options including optional EXPIRES_AT_MILLIS_KEY
-            // Or null if no storage options are available
-            let result = env
-                .call_method(
-                    &java_provider,
-                    "fetchStorageOptions",
-                    "()Ljava/util/Map;",
-                    &[],
-                )
-                .map_err(|e| {
-                    lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                        "Failed to call fetchStorageOptions: {}",
-                        e
-                    ))))
-                })?;
-
-            let result_obj = result.l().map_err(|e| {
-                lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                    "fetchStorageOptions result is not an object: {}",
-                    e
-                ))))
-            })?;
-
-            // Check if result is null
-            if result_obj.is_null() {
-                return Ok(None);
-            }
-
-            // Convert Java Map to Rust HashMap
-            let storage_options_map = JMap::from_env(&mut env, &result_obj).map_err(|e| {
-                lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                    "fetchStorageOptions result is not a Map: {}",
-                    e
-                ))))
-            })?;
-
-            let mut storage_options = HashMap::new();
-            let mut iter = storage_options_map.iter(&mut env).map_err(|e| {
-                lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to iterate storage options: {}",
-                    e
-                ))))
-            })?;
-
-            while let Some((key, value)) = iter.next(&mut env).map_err(|e| {
-                lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to get next storage option entry: {}",
-                    e
-                ))))
-            })? {
-                let key_str: String = env
-                    .get_string(&JString::from(key))
-                    .map_err(|e| {
-                        lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                            "storage option key is not a string: {}",
-                            e
-                        ))))
-                    })?
-                    .into();
-
-                let value_str: String = env
-                    .get_string(&JString::from(value))
-                    .map_err(|e| {
-                        lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                            "storage option value is not a string: {}",
-                            e
-                        ))))
-                    })?
-                    .into();
-
-                storage_options.insert(key_str, value_str);
-            }
-
-            Ok(Some(storage_options))
-        })
-        .await
-        .map_err(|e| {
-            lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to spawn blocking task: {}",
-                e
-            ))))
-        })?
-    }
-
-    fn provider_id(&self) -> String {
-        // Call providerId() method on the Java object
-        // This should always succeed since StorageOptionsProvider.providerId() has a default implementation
-        let mut env = self
-            .jvm
-            .attach_current_thread()
-            .expect("Failed to attach to JVM");
-
-        let result = env
-            .call_method(
-                &self.java_provider,
-                "providerId",
-                "()Ljava/lang/String;",
-                &[],
-            )
-            .expect("Failed to call providerId() on Java StorageOptionsProvider");
-
-        let result_obj = result.l().expect("providerId() did not return an object");
-
-        if result_obj.is_null() {
-            panic!("providerId() returned null");
-        }
-
-        let jstring = JString::from(result_obj);
-        let java_string = env
-            .get_string(&jstring)
-            .expect("Failed to convert Java string to Rust string");
-
-        java_string.into()
-    }
-}
+// This module is intentionally empty.
+// Storage options providers are created directly in Rust from namespace clients
+// using LanceNamespaceStorageOptionsProvider.
diff --git a/java/lance-jni/src/task_tracker.rs b/java/lance-jni/src/task_tracker.rs
new file mode 100644
index 00000000000..bc9d9b0519f
--- /dev/null
+++ b/java/lance-jni/src/task_tracker.rs
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use jni::objects::GlobalRef;
+use std::collections::HashMap;
+use std::sync::{Arc, LazyLock};
+use tokio::sync::RwLock;
+
+pub type TaskId = u64;
+
+/// Information about an in-flight async task
+pub struct TaskInfo {
+    #[allow(dead_code)] // Used for cleanup when task is cancelled
+    pub scanner_global_ref: GlobalRef,
+    pub cancel_handle: tokio::task::JoinHandle<()>,
+}
+
+/// Thread-safe task registry for managing async scan operations
+pub struct TaskTracker {
+    tasks: Arc<RwLock<HashMap<TaskId, TaskInfo>>>,
+}
+
+impl TaskTracker {
+    pub fn new() -> Self {
+        Self {
+            tasks: Arc::new(RwLock::new(HashMap::new())),
+        }
+    }
+
+    /// Register a new task
+    pub async fn register(&self, task_id: TaskId, info: TaskInfo) {
+        let mut tasks = self.tasks.write().await;
+        tasks.insert(task_id, info);
+    }
+
+    /// Update the cancel handle for a task (used in two-phase registration)
+    /// Returns true if task was found and updated, false if task already completed
+    pub async fn update_handle(
+        &self,
+        task_id: TaskId,
+        cancel_handle: tokio::task::JoinHandle<()>,
+    ) -> bool {
+        let mut tasks = self.tasks.write().await;
+        if let Some(task_info) = tasks.get_mut(&task_id) {
+            // Abort the old placeholder handle and replace with real handle
+            task_info.cancel_handle.abort();
+            task_info.cancel_handle = cancel_handle;
+            true
+        } else {
+            // Task already completed before we could update - abort the handle
+            cancel_handle.abort();
+            false
+        }
+    }
+
+    /// Mark a task as complete and return its info
+    pub async fn complete(&self, task_id: TaskId) -> Option<TaskInfo> {
+        let mut tasks = self.tasks.write().await;
+        tasks.remove(&task_id)
+    }
+
+    /// Cancel a task by ID
+    pub async fn cancel(&self, task_id: TaskId) {
+        let info = {
+            let mut tasks = self.tasks.write().await;
+            tasks.remove(&task_id)
+        };
+
+        if let Some(info) = info {
+            info.cancel_handle.abort();
+        }
+    }
+
+    // TODO: Implement timeout-based cleanup for defense-in-depth
+    //
+    // While TaskCleanupGuard (RAII pattern) ensures cleanup in normal and panic cases,
+    // a background cleanup task provides additional safety against edge cases:
+    //
+    // Proposed implementation:
+    // ```
+    // pub async fn cleanup_stale_tasks(&self, max_age: Duration) {
+    //     let mut tasks = self.tasks.write().await;
+    //     let now = Instant::now();
+    //     tasks.retain(|task_id, info| {
+    //         let is_finished = info.cancel_handle.is_finished();
+    //         let is_stale = info.created_at.elapsed() > max_age;
+    //
+    //         if is_finished || is_stale {
+    //             log::warn!("Cleaning up stale/finished task {}", task_id);
+    //             false // remove from HashMap
+    //         } else {
+    //             true // keep in HashMap
+    //         }
+    //     });
+    // }
+    //
+    // // In JNI_OnLoad or module initialization:
+    // RT.spawn(async {
+    //     loop {
+    //         tokio::time::sleep(Duration::from_secs(60)).await;
+    //         TASK_TRACKER.cleanup_stale_tasks(Duration::from_secs(300)).await;
+    //     }
+    // });
+    // ```
+    //
+    // This would require adding `created_at: Instant` field to TaskInfo.
+}
+
+/// Global task tracker instance
+pub static TASK_TRACKER: LazyLock<TaskTracker> = LazyLock::new(TaskTracker::new);
diff --git a/java/lance-jni/src/transaction.rs b/java/lance-jni/src/transaction.rs
index 68d5425a389..d46140020f0 100644
--- a/java/lance-jni/src/transaction.rs
+++ b/java/lance-jni/src/transaction.rs
@@ -6,7 +6,6 @@ use crate::JNIEnvExt;
 use crate::RT;
 use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET, extract_namespace_info};
 use crate::error::Result;
-use crate::storage_options::JavaStorageOptionsProvider;
 use crate::traits::{
     FromJObjectWithEnv, FromJString, IntoJava, JLance, export_vec, import_vec_from_method,
 };
@@ -28,7 +27,7 @@ use lance::table::format::{Fragment, IndexMetadata};
 use lance_core::datatypes::Field;
 use lance_core::datatypes::Schema as LanceSchema;
 use lance_file::version::LanceFileVersion;
-use lance_io::object_store::StorageOptionsProvider;
+use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsProvider};
 use lance_table::io::commit::CommitHandler;
 use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;
 use prost::Message;
@@ -153,6 +152,7 @@ impl FromJObjectWithEnv<RewrittenIndex> for JObject<'_> {
                 value: new_index_details_value,
             },
             new_index_version: new_index_version as u32,
+            new_index_files: None,
         })
     }
 }
@@ -214,6 +214,7 @@ impl FromJObjectWithEnv<IndexMetadata> for JObject<'_> {
             index_version,
             created_at,
             base_id,
+            files: None,
         })
     }
 }
@@ -621,6 +622,9 @@ pub extern "system" fn Java_org_lance_CommitBuilder_nativeCommitToDataset<'local
     storage_format_obj: JObject,
     max_retries: jint,
     skip_auto_cleanup: jboolean,
+    namespace_obj: JObject,
+    table_id_obj: JObject,
+    namespace_client_managed_versioning: jboolean,
 ) -> JObject<'local> {
     ok_or_throw!(
         env,
@@ -635,6 +639,9 @@ pub extern "system" fn Java_org_lance_CommitBuilder_nativeCommitToDataset<'local
             storage_format_obj,
             max_retries as u32,
             skip_auto_cleanup != 0,
+            namespace_obj,
+            table_id_obj,
+            namespace_client_managed_versioning != 0,
         )
     )
 }
@@ -651,6 +658,9 @@ fn inner_commit_to_dataset<'local>(
     storage_format_obj: JObject,
     max_retries: u32,
     skip_auto_cleanup: bool,
+    namespace_obj: JObject,
+    table_id_obj: JObject,
+    namespace_client_managed_versioning: bool,
 ) -> Result<JObject<'local>> {
     let write_param = if write_params_obj.is_null() {
         HashMap::new()
@@ -743,6 +753,19 @@ fn inner_commit_to_dataset<'local>(
         Some(&mut java_blocking_ds),
     )?;
 
+    // Set namespace commit handler only if namespace_client_managed_versioning is true
+    let namespace_info = extract_namespace_info(env, &namespace_obj, &table_id_obj)?;
+    let commit_handler = if namespace_client_managed_versioning {
+        namespace_info.map(|(ns, tid)| {
+            let external_store = LanceNamespaceExternalManifestStore::new(ns, tid);
+            Arc::new(ExternalManifestCommitHandler {
+                external_manifest_store: Arc::new(external_store),
+            }) as Arc<dyn CommitHandler>
+        })
+    } else {
+        None
+    };
+
     let new_blocking_ds = {
         let mut dataset_guard =
             unsafe { env.get_rust_field::<_, _, BlockingDataset>(&java_dataset, NATIVE_DATASET) }?;
@@ -755,6 +778,7 @@ fn inner_commit_to_dataset<'local>(
             storage_format,
             max_retries,
             skip_auto_cleanup,
+            commit_handler,
         )?
     };
     new_blocking_ds.into_java(env)
@@ -1107,6 +1131,9 @@ fn convert_to_rust_operation(
                     to_rust_map(env, &config_upsert_values)
                 },
             )?;
+            // Pass None for dataset so that the new schema is not validated
+            // against the old schema. Overwrite replaces the entire dataset,
+            // so fields with the same name but different types are allowed.
             let schema = convert_schema_from_operation(
                 env,
                 java_operation,
@@ -1115,7 +1142,7 @@ fn convert_to_rust_operation(
                         "BufferAllocator is required for Overwrite operations".to_string(),
                     )
                 })?,
-                dataset,
+                None,
                 read_version,
             )?;
             Operation::Overwrite {
@@ -1345,7 +1372,6 @@ pub extern "system" fn Java_org_lance_CommitBuilder_nativeCommitToUri<'local>(
     java_transaction: JObject,
     detached_jbool: jboolean,
     enable_v2_manifest_paths: jboolean,
-    storage_options_provider_obj: JObject,
     namespace_obj: JObject,
     table_id_obj: JObject,
     allocator_obj: JObject,
@@ -1354,6 +1380,7 @@ pub extern "system" fn Java_org_lance_CommitBuilder_nativeCommitToUri<'local>(
     storage_format_obj: JObject,
     max_retries: jint,
     skip_auto_cleanup: jboolean,
+    namespace_client_managed_versioning: jboolean,
 ) -> JObject<'local> {
     ok_or_throw!(
         env,
@@ -1363,7 +1390,6 @@ pub extern "system" fn Java_org_lance_CommitBuilder_nativeCommitToUri<'local>(
             java_transaction,
             detached_jbool != 0,
             enable_v2_manifest_paths != 0,
-            storage_options_provider_obj,
             namespace_obj,
             table_id_obj,
             allocator_obj,
@@ -1372,6 +1398,7 @@ pub extern "system" fn Java_org_lance_CommitBuilder_nativeCommitToUri<'local>(
             storage_format_obj,
             max_retries as u32,
             skip_auto_cleanup != 0,
+            namespace_client_managed_versioning != 0,
         )
     )
 }
@@ -1383,7 +1410,6 @@ fn inner_commit_to_uri<'local>(
     java_transaction: JObject,
     detached: bool,
     enable_v2_manifest_paths: bool,
-    storage_options_provider_obj: JObject,
     namespace_obj: JObject,
     table_id_obj: JObject,
     allocator_obj: JObject,
@@ -1392,6 +1418,7 @@ fn inner_commit_to_uri<'local>(
     storage_format_obj: JObject,
     max_retries: u32,
     skip_auto_cleanup: bool,
+    namespace_client_managed_versioning: bool,
 ) -> Result<JObject<'local>> {
     let uri_str: String = uri.extract(env)?;
 
@@ -1421,13 +1448,17 @@ fn inner_commit_to_uri<'local>(
         Some(parse_storage_format(&format_str)?)
     };
 
-    // Build storage options accessor
-    let storage_options_provider: Option<JavaStorageOptionsProvider> = env
-        .get_optional(&storage_options_provider_obj, |env, provider_obj| {
-            JavaStorageOptionsProvider::new(env, provider_obj)
-        })?;
-    let storage_options_provider =
-        storage_options_provider.map(|p| Arc::new(p) as Arc<dyn StorageOptionsProvider>);
+    // Extract namespace info and create storage options provider if namespace is provided
+    let namespace_info = extract_namespace_info(env, &namespace_obj, &table_id_obj)?;
+    let storage_options_provider: Option<Arc<dyn StorageOptionsProvider>> =
+        if let Some((ref ns, ref tid)) = namespace_info {
+            Some(Arc::new(LanceNamespaceStorageOptionsProvider::new(
+                ns.clone(),
+                tid.clone(),
+            )))
+        } else {
+            None
+        };
 
     // Keep a copy of initial options for opening the read dataset.
     let initial_storage_options = write_param.clone();
@@ -1450,9 +1481,8 @@ fn inner_commit_to_uri<'local>(
         ..Default::default()
     };
 
-    let namespace_info = extract_namespace_info(env, &namespace_obj, &table_id_obj)?;
     let (open_namespace, open_table_id) = match &namespace_info {
-        Some((ns, tid)) => (Some(ns.clone()), Some(tid.clone())),
+        Some((namespace_client, tid)) => (Some(namespace_client.clone()), Some(tid.clone())),
         None => (None, None),
     };
 
@@ -1470,6 +1500,7 @@ fn inner_commit_to_uri<'local>(
         None,
         open_namespace,
         open_table_id,
+        namespace_client_managed_versioning,
     )
     .ok();
 
@@ -1501,9 +1532,9 @@ fn inner_commit_to_uri<'local>(
         builder = builder.with_skip_auto_cleanup(true);
     }
 
-    // Set namespace commit handler if provided
-    if let Some((ns, tid)) = namespace_info {
-        let external_store = LanceNamespaceExternalManifestStore::new(ns, tid);
+    // Set namespace commit handler only if namespace_client_managed_versioning is true
+    if namespace_client_managed_versioning && let Some((namespace_client, tid)) = namespace_info {
+        let external_store = LanceNamespaceExternalManifestStore::new(namespace_client, tid);
         let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
             external_manifest_store: Arc::new(external_store),
         });
diff --git a/java/lance-jni/src/utils.rs b/java/lance-jni/src/utils.rs
index 2b342c99ef7..35c1e0ffae9 100644
--- a/java/lance-jni/src/utils.rs
+++ b/java/lance-jni/src/utils.rs
@@ -23,11 +23,9 @@ use lance_linalg::distance::DistanceType;
 
 use crate::error::{Error, Result};
 use crate::ffi::JNIEnvExt;
-use crate::storage_options::JavaStorageOptionsProvider;
 
 use crate::traits::FromJObjectWithEnv;
 use lance_index::vector::Query;
-use lance_io::object_store::StorageOptionsProvider;
 use std::collections::HashMap;
 use std::str::FromStr;
 
@@ -51,9 +49,10 @@ pub fn extract_write_params(
     data_storage_version: &JObject,
     enable_v2_manifest_paths: Option<&JObject>,
     storage_options_obj: &JObject,
-    storage_options_provider_obj: &JObject, // Optional<StorageOptionsProvider>
-    initial_bases: &JObject,                // Optional<BasePath>
-    target_bases: &JObject,                 // Optional<String>
+    initial_bases: &JObject,                     // Optional<BasePath>
+    target_bases: &JObject,                      // Optional<String>
+    allow_external_blob_outside_bases: &JObject, // Optional<Boolean>
+    blob_pack_file_size_threshold: &JObject,     // Optional<Long>
 ) -> Result<WriteParams> {
     let mut write_params = WriteParams::default();
 
@@ -90,13 +89,6 @@ pub fn extract_write_params(
     let storage_options: HashMap<String, String> =
         extract_storage_options(env, storage_options_obj)?;
 
-    // Extract storage options provider if present
-    let storage_options_provider: Option<Arc<dyn StorageOptionsProvider>> = env
-        .get_optional(storage_options_provider_obj, |env, provider_obj| {
-            JavaStorageOptionsProvider::new(env, provider_obj)
-        })?
-        .map(|p| Arc::new(p) as Arc<dyn StorageOptionsProvider>);
-
     if let Some(initial_bases) =
         env.get_list_opt(initial_bases, |env, elem| elem.extract_object(env))?
     {
@@ -107,18 +99,20 @@ pub fn extract_write_params(
         write_params.target_base_names_or_paths = Some(names);
     }
 
-    // Create storage options accessor from storage_options and provider
-    let accessor = match (storage_options.is_empty(), storage_options_provider) {
-        (false, Some(provider)) => Some(Arc::new(
-            lance::io::StorageOptionsAccessor::with_initial_and_provider(storage_options, provider),
-        )),
-        (false, None) => Some(Arc::new(
+    if let Some(allow) = env.get_boolean_opt(allow_external_blob_outside_bases)? {
+        write_params.allow_external_blob_outside_bases = allow;
+    }
+    if let Some(max_bytes) = env.get_long_opt(blob_pack_file_size_threshold)? {
+        write_params.blob_pack_file_size_threshold = Some(max_bytes as usize);
+    }
+
+    // Create storage options accessor from static storage_options
+    let accessor = if storage_options.is_empty() {
+        None
+    } else {
+        Some(Arc::new(
             lance::io::StorageOptionsAccessor::with_static_options(storage_options),
-        )),
-        (true, Some(provider)) => Some(Arc::new(lance::io::StorageOptionsAccessor::with_provider(
-            provider,
-        ))),
-        (true, None) => None,
+        ))
     };
 
     write_params.store_params = Some(ObjectStoreParams {
@@ -141,8 +135,10 @@ pub fn build_compaction_options(
     defer_index_remap: &JObject,               // Optional<Boolean>
     compaction_mode: &JObject,                 // Optional<String>
     binary_copy_read_batch_bytes: &JObject,    // Optional<Long>
+    max_source_fragments: &JObject,            // Optional<Long>
+    config: &std::collections::HashMap<String, String>,
 ) -> Result<CompactionOptions> {
-    let mut compaction_options = CompactionOptions::default();
+    let mut compaction_options = CompactionOptions::from_dataset_config(config)?;
 
     if let Some(target_rows_per_fragment_val) = env.get_long_opt(target_rows_per_fragment)? {
         compaction_options.target_rows_per_fragment = target_rows_per_fragment_val as usize;
@@ -180,6 +176,9 @@ pub fn build_compaction_options(
         compaction_options.binary_copy_read_batch_bytes =
             Some(binary_copy_read_batch_bytes_val as usize);
     }
+    if let Some(max_source_fragments_val) = env.get_long_opt(max_source_fragments)? {
+        compaction_options.max_source_fragments = Some(max_source_fragments_val as usize);
+    }
 
     Ok(compaction_options)
 }
@@ -575,3 +574,85 @@ pub fn to_java_float_obj<'local>(
         None => Ok(JObject::null()),
     }
 }
+
+pub fn to_java_double_obj<'local>(
+    env: &mut JNIEnv<'local>,
+    value: Option<f64>,
+) -> Result<JObject<'local>> {
+    match value {
+        Some(v) => Ok(env.new_object("java/lang/Double", "(D)V", &[JValue::Double(v)])?),
+        None => Ok(JObject::null()),
+    }
+}
+
+pub fn to_java_string_obj<'local>(
+    env: &mut JNIEnv<'local>,
+    value: Option<&str>,
+) -> Result<JObject<'local>> {
+    match value {
+        Some(v) => {
+            let jstr = env.new_string(v)?;
+            Ok(jstr.into())
+        }
+        None => Ok(JObject::null()),
+    }
+}
+
+/// Convert a DataFusion ScalarValue to a Java Comparable object.
+///
+/// Maps numeric types to their boxed Java equivalents (Long, Double)
+/// and string types to Java String. Null ScalarValues produce JObject::null().
+///
+/// This is useful for exposing index statistics (e.g., zonemap min/max)
+/// to Java clients in a type-safe, Comparable-compatible way.
+pub fn scalar_value_to_java<'a>(
+    env: &mut JNIEnv<'a>,
+    value: &datafusion_common::ScalarValue,
+) -> Result<JObject<'a>> {
+    use datafusion_common::ScalarValue;
+
+    match value {
+        ScalarValue::Null => Ok(JObject::null()),
+
+        ScalarValue::Boolean(v) => to_java_boolean_obj(env, *v),
+
+        // Integer types → Java Long
+        ScalarValue::Int8(v) => to_java_long_obj(env, v.map(|x| x as i64)),
+        ScalarValue::Int16(v) => to_java_long_obj(env, v.map(|x| x as i64)),
+        ScalarValue::Int32(v) => to_java_long_obj(env, v.map(|x| x as i64)),
+        ScalarValue::Int64(v) => to_java_long_obj(env, *v),
+        ScalarValue::UInt8(v) => to_java_long_obj(env, v.map(|x| x as i64)),
+        ScalarValue::UInt16(v) => to_java_long_obj(env, v.map(|x| x as i64)),
+        ScalarValue::UInt32(v) => to_java_long_obj(env, v.map(|x| x as i64)),
+        // UInt64 may overflow i64, but for min/max stats this is acceptable
+        ScalarValue::UInt64(v) => to_java_long_obj(env, v.map(|x| x as i64)),
+
+        // Float types → Java Double
+        ScalarValue::Float16(v) => to_java_double_obj(env, v.map(|x| f64::from(x.to_f32()))),
+        ScalarValue::Float32(v) => to_java_double_obj(env, v.map(|x| x as f64)),
+        ScalarValue::Float64(v) => to_java_double_obj(env, *v),
+
+        // String types → Java String
+        ScalarValue::Utf8(v) => to_java_string_obj(env, v.as_deref()),
+        ScalarValue::LargeUtf8(v) => to_java_string_obj(env, v.as_deref()),
+
+        // Date types → Java Long
+        ScalarValue::Date32(v) => to_java_long_obj(env, v.map(|x| x as i64)),
+        ScalarValue::Date64(v) => to_java_long_obj(env, *v),
+
+        // Timestamp types → Java Long
+        ScalarValue::TimestampSecond(v, _) => to_java_long_obj(env, *v),
+        ScalarValue::TimestampMillisecond(v, _) => to_java_long_obj(env, *v),
+        ScalarValue::TimestampMicrosecond(v, _) => to_java_long_obj(env, *v),
+        ScalarValue::TimestampNanosecond(v, _) => to_java_long_obj(env, *v),
+
+        // For any unsupported type, return null (conservative: caller will skip)
+        _ => {
+            log::warn!(
+                "Unsupported ScalarValue type for Java conversion: {:?}",
+                value.data_type()
+            );
+            Ok(JObject::null())
+        }
+    }
+}
diff --git a/java/lance-jni/src/vector_trainer.rs b/java/lance-jni/src/vector_trainer.rs
index 92b6afa084b..9ea164d3586 100755
--- a/java/lance-jni/src/vector_trainer.rs
+++ b/java/lance-jni/src/vector_trainer.rs
@@ -117,6 +117,7 @@ fn inner_train_ivf_centroids<'local>(
             dim,
             metric_type,
             &ivf_params,
+            None,
             Arc::new(NoopIndexBuildProgress),
         ))?;
 
diff --git a/java/pom.xml b/java/pom.xml
index 09faa47f958..23b44326f96 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -7,7 +7,7 @@
     <groupId>org.lance</groupId>
     <artifactId>lance-core</artifactId>
     <name>Lance Core</name>
-    <version>4.0.0-beta.7</version>
+    <version>6.0.0-beta.1</version>
     <packaging>jar</packaging>
 
     <description>Lance Format Java API</description>
@@ -109,12 +109,12 @@
         <dependency>
             <groupId>org.lance</groupId>
             <artifactId>lance-namespace-core</artifactId>
-            <version>0.5.2</version>
+            <version>0.6.1</version>
         </dependency>
         <dependency>
             <groupId>org.lance</groupId>
             <artifactId>lance-namespace-apache-client</artifactId>
-            <version>0.5.2</version>
+            <version>0.6.1</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
diff --git a/java/src/main/java/org/lance/BlobFile.java b/java/src/main/java/org/lance/BlobFile.java
index 03c8408ab57..27af01fe73c 100755
--- a/java/src/main/java/org/lance/BlobFile.java
+++ b/java/src/main/java/org/lance/BlobFile.java
@@ -59,6 +59,13 @@ public byte[] readUpTo(int len) throws IOException {
     return nativeReadUpTo(len);
   }
 
+  /** Read a blob-local range without changing the current cursor. */
+  public byte[] readRange(long offset, int len) throws IOException {
+    if (offset < 0) throw new IllegalArgumentException("offset must be non-negative");
+    if (len < 0) throw new IllegalArgumentException("len must be non-negative");
+    return nativeReadRange(offset, len);
+  }
+
   /** Seek to a new cursor position. */
   public void seek(long newCursor) throws IOException {
     if (newCursor < 0) throw new IllegalArgumentException("newCursor must be non-negative");
diff --git a/java/src/main/java/org/lance/Branch.java b/java/src/main/java/org/lance/Branch.java
index f2a1f5b21ea..07f79a08f6a 100755
--- a/java/src/main/java/org/lance/Branch.java
+++ b/java/src/main/java/org/lance/Branch.java
@@ -14,27 +14,78 @@
 package org.lance;
 
 import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableList;
 
+import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 
 /**
  * Branch metadata aligned with Rust's BranchContents. name is the branch name, parentBranch may be
- * null (indicating main), parentVersion is the version on which the branch was created, createAt is
- * the unix timestamp (seconds), and manifestSize is the size of the referenced manifest file in
- * bytes.
+ * null (indicating main), branchIdentifier is the lineage chain {@code [(version, uuid), ...]},
+ * parentVersion is the version on which the branch was created, createAt is the unix timestamp
+ * (seconds), and manifestSize is the size of the referenced manifest file in bytes.
  */
 public class Branch {
+  /** A single lineage hop in the Rust BranchIdentifier.version_mapping vector. */
+  public static class BranchVersionMapping {
+    private final long version;
+    private final String uuid;
+
+    public BranchVersionMapping(long version, String uuid) {
+      this.version = version;
+      this.uuid = Objects.requireNonNull(uuid);
+    }
+
+    public long getVersion() {
+      return version;
+    }
+
+    public String getUuid() {
+      return uuid;
+    }
+
+    @Override
+    public String toString() {
+      return MoreObjects.toStringHelper(this).add("version", version).add("uuid", uuid).toString();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) return true;
+      if (o == null || getClass() != o.getClass()) return false;
+      BranchVersionMapping that = (BranchVersionMapping) o;
+      return version == that.version && Objects.equals(uuid, that.uuid);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(version, uuid);
+    }
+  }
+
   private final String name;
   private final Optional<String> parentBranch;
+  private final ImmutableList<BranchVersionMapping> branchIdentifier;
   private final long parentVersion;
   private final long createAt;
   private final int manifestSize;
 
   public Branch(
       String name, String parentBranch, long parentVersion, long createAt, int manifestSize) {
+    this(name, parentBranch, ImmutableList.of(), parentVersion, createAt, manifestSize);
+  }
+
+  public Branch(
+      String name,
+      String parentBranch,
+      List<BranchVersionMapping> branchIdentifier,
+      long parentVersion,
+      long createAt,
+      int manifestSize) {
     this.name = name;
     this.parentBranch = Optional.ofNullable(parentBranch);
+    this.branchIdentifier = ImmutableList.copyOf(Objects.requireNonNull(branchIdentifier));
     this.parentVersion = parentVersion;
     this.createAt = createAt;
     this.manifestSize = manifestSize;
@@ -48,6 +99,10 @@ public Optional<String> getParentBranch() {
     return parentBranch;
   }
 
+  public ImmutableList<BranchVersionMapping> getBranchIdentifier() {
+    return branchIdentifier;
+  }
+
   public long getParentVersion() {
     return parentVersion;
   }
@@ -65,6 +120,7 @@ public String toString() {
     return MoreObjects.toStringHelper(this)
         .add("name", name)
         .add("parentBranch", parentBranch)
+        .add("branchIdentifier", branchIdentifier)
         .add("parentVersion", parentVersion)
         .add("createAt", createAt)
         .add("manifestSize", manifestSize)
@@ -80,11 +136,13 @@ public boolean equals(Object o) {
         && createAt == branch.createAt
         && manifestSize == branch.manifestSize
         && Objects.equals(name, branch.name)
-        && Objects.equals(parentBranch, branch.parentBranch);
+        && Objects.equals(parentBranch, branch.parentBranch)
+        && Objects.equals(branchIdentifier, branch.branchIdentifier);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(name, parentBranch, parentVersion, createAt, manifestSize);
+    return Objects.hash(
+        name, parentBranch, branchIdentifier, parentVersion, createAt, manifestSize);
   }
 }
diff --git a/java/src/main/java/org/lance/CommitBuilder.java b/java/src/main/java/org/lance/CommitBuilder.java
index 0b2752aae89..fa083c9e93c 100644
--- a/java/src/main/java/org/lance/CommitBuilder.java
+++ b/java/src/main/java/org/lance/CommitBuilder.java
@@ -13,7 +13,6 @@
  */
 package org.lance;
 
-import org.lance.io.StorageOptionsProvider;
 import org.lance.namespace.LanceNamespace;
 
 import org.apache.arrow.memory.BufferAllocator;
@@ -65,9 +64,9 @@ public class CommitBuilder {
   private final BufferAllocator allocator;
 
   private Map<String, String> writeParams;
-  private StorageOptionsProvider storageOptionsProvider;
-  private LanceNamespace namespace;
+  private LanceNamespace namespaceClient;
   private List<String> tableId;
+  private boolean namespaceClientManagedVersioning = false;
   private boolean enableV2ManifestPaths = true;
   private boolean detached = false;
   private Boolean useStableRowIds;
@@ -113,35 +112,43 @@ public CommitBuilder writeParams(Map<String, String> writeParams) {
   }
 
   /**
-   * Set the storage options provider for credential refresh during URI-based commits.
+   * Set the namespace client for managed versioning. When set, commits are routed through the
+   * namespace client's {@code createTableVersion} API instead of writing directly to the object
+   * store. This is supported for both dataset-based and URI-based commits.
    *
-   * @param provider the storage options provider
+   * @param namespaceClient the LanceNamespace client instance
    * @return this builder instance
    */
-  public CommitBuilder storageOptionsProvider(StorageOptionsProvider provider) {
-    this.storageOptionsProvider = provider;
+  public CommitBuilder namespaceClient(LanceNamespace namespaceClient) {
+    this.namespaceClient = namespaceClient;
     return this;
   }
 
   /**
-   * Set the namespace for managed versioning during URI-based commits.
+   * Set the table ID for namespace client-based commit handling.
    *
-   * @param namespace the LanceNamespace instance
+   * <p>Must be provided together with `namespaceClient`.
+   *
+   * @param tableId the table identifier (e.g., ["workspace", "table_name"])
    * @return this builder instance
    */
-  public CommitBuilder namespace(LanceNamespace namespace) {
-    this.namespace = namespace;
+  public CommitBuilder tableId(List<String> tableId) {
+    this.tableId = tableId;
     return this;
   }
 
   /**
-   * Set the table ID for namespace-based commit handling.
+   * Set whether namespace manages versioning.
    *
-   * @param tableId the table identifier (e.g., ["workspace", "table_name"])
+   * <p>When true and namespaceClient/tableId are set, commits are routed through the namespace
+   * client's create_table_version API. This is typically set based on the managed_versioning field
+   * from describe_table or declare_table responses.
+   *
+   * @param namespaceClientManagedVersioning whether namespace manages versioning
    * @return this builder instance
    */
-  public CommitBuilder tableId(List<String> tableId) {
-    this.tableId = tableId;
+  public CommitBuilder namespaceClientManagedVersioning(boolean namespaceClientManagedVersioning) {
+    this.namespaceClientManagedVersioning = namespaceClientManagedVersioning;
     return this;
   }
 
@@ -250,7 +257,10 @@ public Dataset execute(Transaction transaction) {
               useStableRowIds,
               storageFormat,
               maxRetries,
-              skipAutoCleanup);
+              skipAutoCleanup,
+              namespaceClient,
+              tableId,
+              namespaceClientManagedVersioning);
       result.setAllocator(dataset.allocator());
       return result;
     }
@@ -261,15 +271,15 @@ public Dataset execute(Transaction transaction) {
               transaction,
               detached,
               enableV2ManifestPaths,
-              storageOptionsProvider,
-              namespace,
+              namespaceClient,
               tableId,
               allocator,
               writeParams,
               useStableRowIds,
               storageFormat,
               maxRetries,
-              skipAutoCleanup);
+              skipAutoCleanup,
+              namespaceClientManagedVersioning);
       result.setAllocator(allocator);
       return result;
     }
@@ -285,14 +295,16 @@ private static native Dataset nativeCommitToDataset(
       Boolean useStableRowIds,
       String storageFormat,
       int maxRetries,
-      boolean skipAutoCleanup);
+      boolean skipAutoCleanup,
+      Object namespace,
+      Object tableId,
+      boolean namespaceClientManagedVersioning);
 
   private static native Dataset nativeCommitToUri(
       String uri,
       Transaction transaction,
       boolean detached,
       boolean enableV2ManifestPaths,
-      Object storageOptionsProvider,
       Object namespace,
       Object tableId,
       Object allocator,
@@ -300,5 +312,6 @@ private static native Dataset nativeCommitToUri(
       Boolean useStableRowIds,
       String storageFormat,
       int maxRetries,
-      boolean skipAutoCleanup);
+      boolean skipAutoCleanup,
+      boolean namespaceClientManagedVersioning);
 }
diff --git a/java/src/main/java/org/lance/Dataset.java b/java/src/main/java/org/lance/Dataset.java
index 166feebba20..42444a9aa83 100644
--- a/java/src/main/java/org/lance/Dataset.java
+++ b/java/src/main/java/org/lance/Dataset.java
@@ -24,7 +24,7 @@
 import org.lance.index.IndexParams;
 import org.lance.index.IndexType;
 import org.lance.index.OptimizeOptions;
-import org.lance.io.StorageOptionsProvider;
+import org.lance.index.scalar.ZoneStats;
 import org.lance.ipc.DataStatistics;
 import org.lance.ipc.LanceScanner;
 import org.lance.ipc.ScanOptions;
@@ -100,12 +100,12 @@ private Dataset() {}
    *     .execute();
    * }</pre>
    *
-   * <p>Example usage with namespace and empty table:
+   * <p>Example usage with namespaceClient and empty table:
    *
    * <pre>{@code
    * Dataset dataset = Dataset.write()
    *     .schema(mySchema)
-   *     .namespace(myNamespace)
+   *     .namespaceClient(myNamespaceClient)
    *     .tableId(Arrays.asList("my_table"))
    *     .mode(WriteMode.CREATE)
    *     .execute();
@@ -151,7 +151,9 @@ public static Dataset create(
               params.getEnableV2ManifestPaths(),
               params.getStorageOptions(),
               params.getInitialBases(),
-              params.getTargetBases());
+              params.getTargetBases(),
+              params.getAllowExternalBlobOutsideBases(),
+              params.getBlobPackFileSizeThreshold());
       dataset.allocator = allocator;
       return dataset;
     }
@@ -169,32 +171,20 @@ public static Dataset create(
    *     Dataset.write().allocator(allocator).stream(stream).uri(path)
    *     .mode(WriteMode.CREATE).execute()}
    */
-  @Deprecated
-  public static Dataset create(
-      BufferAllocator allocator, ArrowArrayStream stream, String path, WriteParams params) {
-    return create(allocator, stream, path, params, null);
-  }
-
   /**
-   * Create a dataset with given stream and storage options provider.
-   *
-   * <p>This method supports credential vending through the StorageOptionsProvider interface, which
-   * allows for dynamic credential refresh during long-running write operations.
+   * Create a dataset with given stream.
    *
    * @param allocator buffer allocator
    * @param stream arrow stream
    * @param path dataset uri
    * @param params write parameters
-   * @param storageOptionsProvider optional provider for dynamic storage options/credentials
    * @return Dataset
+   * @deprecated Use {@link #write()} builder instead.
    */
-  static Dataset create(
-      BufferAllocator allocator,
-      ArrowArrayStream stream,
-      String path,
-      WriteParams params,
-      StorageOptionsProvider storageOptionsProvider) {
-    return create(allocator, stream, path, params, storageOptionsProvider, null, null);
+  @Deprecated
+  public static Dataset create(
+      BufferAllocator allocator, ArrowArrayStream stream, String path, WriteParams params) {
+    return create(allocator, stream, path, params, null, null, false);
   }
 
   private static native Dataset createWithFfiSchema(
@@ -209,8 +199,30 @@ private static native Dataset createWithFfiSchema(
       Optional<Boolean> enableV2ManifestPaths,
       Map<String, String> storageOptions,
       Optional<List<BasePath>> initialBases,
-      Optional<List<String>> targetBases);
+      Optional<List<String>> targetBases,
+      Optional<Boolean> allowExternalBlobOutsideBases,
+      Optional<Long> blobPackFileSizeThreshold);
 
+  /**
+   * Creates a dataset from an FFI arrow stream.
+   *
+   * @param arrowStreamMemoryAddress memory address of the arrow stream
+   * @param path dataset uri
+   * @param maxRowsPerFile max rows per file
+   * @param maxRowsPerGroup max rows per group
+   * @param maxBytesPerFile max bytes per file
+   * @param mode write mode
+   * @param enableStableRowIds whether to enable stable row ids
+   * @param dataStorageVersion data storage version
+   * @param enableV2ManifestPaths whether to enable v2 manifest paths
+   * @param storageOptions storage options
+   * @param initialBases initial bases
+   * @param targetBases target bases
+   * @param namespaceClient optional namespace client for managed versioning and credential refresh
+   *     (can be null)
+   * @param tableId optional table identifier within the namespace client (can be null)
+   * @return Dataset
+   */
   private static native Dataset createWithFfiStream(
       long arrowStreamMemoryAddress,
       String path,
@@ -223,38 +235,30 @@ private static native Dataset createWithFfiStream(
       Optional<Boolean> enableV2ManifestPaths,
       Map<String, String> storageOptions,
       Optional<List<BasePath>> initialBases,
-      Optional<List<String>> targetBases);
-
-  private static native Dataset createWithFfiStreamAndProvider(
-      long arrowStreamMemoryAddress,
-      String path,
-      Optional<Integer> maxRowsPerFile,
-      Optional<Integer> maxRowsPerGroup,
-      Optional<Long> maxBytesPerFile,
-      Optional<String> mode,
-      Optional<Boolean> enableStableRowIds,
-      Optional<String> dataStorageVersion,
-      Optional<Boolean> enableV2ManifestPaths,
-      Map<String, String> storageOptions,
-      Optional<StorageOptionsProvider> storageOptionsProvider,
-      Optional<List<BasePath>> initialBases,
       Optional<List<String>> targetBases,
-      LanceNamespace namespace,
-      List<String> tableId);
+      Optional<Boolean> allowExternalBlobOutsideBases,
+      Optional<Long> blobPackFileSizeThreshold,
+      LanceNamespace namespaceClient,
+      List<String> tableId,
+      boolean namespaceClientManagedVersioning);
 
   /**
-   * Creates a dataset with optional namespace support for managed versioning.
+   * Creates a dataset with optional namespace client support for managed versioning.
    *
-   * <p>When a namespace is provided, the commit handler will use the namespace's
-   * create_table_version method for version tracking.
+   * <p>When namespaceClient and tableId are provided, the Rust side will automatically create a
+   * storage options provider for credential refresh. When namespaceClientManagedVersioning is true,
+   * the commit handler will use the namespace client's create_table_version method for version
+   * tracking.
    *
    * @param allocator buffer allocator
    * @param stream arrow stream
    * @param path dataset uri
    * @param params write parameters
-   * @param storageOptionsProvider optional provider for dynamic storage options/credentials
-   * @param namespace optional namespace implementation for managed versioning (can be null)
-   * @param tableId optional table identifier within the namespace (can be null)
+   * @param namespaceClient optional namespace client for managed versioning and credential refresh
+   *     (can be null)
+   * @param tableId optional table identifier within the namespace client (can be null)
+   * @param namespaceClientManagedVersioning whether namespace manages versioning (commits go
+   *     through namespace API)
    * @return Dataset
    */
   static Dataset create(
@@ -262,15 +266,15 @@ static Dataset create(
       ArrowArrayStream stream,
       String path,
       WriteParams params,
-      StorageOptionsProvider storageOptionsProvider,
-      LanceNamespace namespace,
-      List<String> tableId) {
+      LanceNamespace namespaceClient,
+      List<String> tableId,
+      boolean namespaceClientManagedVersioning) {
     Preconditions.checkNotNull(allocator);
     Preconditions.checkNotNull(stream);
     Preconditions.checkNotNull(path);
     Preconditions.checkNotNull(params);
     Dataset dataset =
-        createWithFfiStreamAndProvider(
+        createWithFfiStream(
             stream.memoryAddress(),
             path,
             params.getMaxRowsPerFile(),
@@ -281,11 +285,13 @@ static Dataset create(
             params.getDataStorageVersion(),
             params.getEnableV2ManifestPaths(),
             params.getStorageOptions(),
-            Optional.ofNullable(storageOptionsProvider),
             params.getInitialBases(),
             params.getTargetBases(),
-            namespace,
-            tableId);
+            params.getAllowExternalBlobOutsideBases(),
+            params.getBlobPackFileSizeThreshold(),
+            namespaceClient,
+            tableId,
+            namespaceClientManagedVersioning);
     dataset.allocator = allocator;
     return dataset;
   }
@@ -359,16 +365,22 @@ static Dataset open(
       String path,
       ReadOptions options,
       Session session) {
-    return open(allocator, selfManagedAllocator, path, options, session, null, null);
+    return open(allocator, selfManagedAllocator, path, options, session, null, null, false);
   }
 
   /**
-   * Open a dataset from the specified path with additional options and namespace commit handler.
+   * Open a dataset from the specified path with additional options and namespace client.
+   *
+   * <p>When namespaceClient and tableId are provided, the Rust side will automatically create a
+   * storage options provider for credential refresh.
    *
    * @param path file path
    * @param options the open options
-   * @param namespace the LanceNamespace to use for managed versioning (null if not using namespace)
-   * @param tableId table identifier (null if not using namespace)
+   * @param namespaceClient the LanceNamespace to use for managed versioning and credential refresh
+   *     (null if not using namespace client)
+   * @param tableId table identifier (null if not using namespace client)
+   * @param namespaceClientManagedVersioning whether namespace manages versioning (commits go
+   *     through namespace API)
    * @return Dataset
    */
   static Dataset open(
@@ -377,8 +389,9 @@ static Dataset open(
       String path,
       ReadOptions options,
       Session session,
-      LanceNamespace namespace,
-      List<String> tableId) {
+      LanceNamespace namespaceClient,
+      List<String> tableId,
+      boolean namespaceClientManagedVersioning) {
     Preconditions.checkNotNull(path);
     Preconditions.checkNotNull(allocator);
     Preconditions.checkNotNull(options);
@@ -398,10 +411,10 @@ static Dataset open(
             options.getMetadataCacheSizeBytes(),
             options.getStorageOptions(),
             options.getSerializedManifest(),
-            options.getStorageOptionsProvider(),
             sessionHandle,
-            namespace,
-            tableId);
+            namespaceClient,
+            tableId,
+            namespaceClientManagedVersioning);
     dataset.allocator = allocator;
     dataset.selfManagedAllocator = selfManagedAllocator;
     if (effectiveSession != null) {
@@ -421,10 +434,10 @@ private static native Dataset openNative(
       long metadataCacheSizeBytes,
       Map<String, String> storageOptions,
       Optional<ByteBuffer> serializedManifest,
-      Optional<StorageOptionsProvider> storageOptionsProvider,
       long sessionHandle,
-      LanceNamespace namespace,
-      List<String> tableId);
+      LanceNamespace namespaceClient,
+      List<String> tableId,
+      boolean namespaceClientManagedVersioning);
 
   /**
    * Creates a builder for opening a dataset.
@@ -440,11 +453,11 @@ private static native Dataset openNative(
    *     .build();
    * }</pre>
    *
-   * <p>Example usage with namespace:
+   * <p>Example usage with namespaceClient:
    *
    * <pre>{@code
    * Dataset dataset = Dataset.open()
-   *     .namespace(myNamespace)
+   *     .namespaceClient(myNamespaceClient)
    *     .tableId(Arrays.asList("my_table"))
    *     .build();
    * }</pre>
@@ -790,9 +803,14 @@ public String uri() {
    * @return the version id of the dataset
    */
   public long version() {
-    return getVersion().getId();
+    try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
+      Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed");
+      return nativeGetVersionId();
+    }
   }
 
+  private native long nativeGetVersionId();
+
   /**
    * Gets the currently checked out version of the dataset.
    *
@@ -1019,6 +1037,43 @@ public void mergeIndexMetadata(
   private native void innerMergeIndexMetadata(
       String indexUUID, int indexType, Optional<Integer> batchReadHead);
 
+  /** Merge one caller-defined group of existing uncommitted vector index segments. */
+  public Index mergeExistingIndexSegments(List<Index> segments) {
+    Preconditions.checkNotNull(segments, "segments cannot be null");
+    Preconditions.checkArgument(!segments.isEmpty(), "segments cannot be empty");
+    try (LockManager.WriteLock writeLock = lockManager.acquireWriteLock()) {
+      Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed");
+      return nativeMergeExistingIndexSegments(segments);
+    }
+  }
+
+  private native Index nativeMergeExistingIndexSegments(List<Index> segments);
+
+  /**
+   * Publish one or more existing physical index segments as a logical index.
+   *
+   * @param indexName logical index name
+   * @param column indexed column name
+   * @param segments physical segment metadata to publish
+   * @return committed manifest metadata
+   */
+  public List<Index> commitExistingIndexSegments(
+      String indexName, String column, List<Index> segments) {
+    Preconditions.checkArgument(
+        indexName != null && !indexName.isEmpty(), "indexName cannot be null or empty");
+    Preconditions.checkArgument(
+        column != null && !column.isEmpty(), "column cannot be null or empty");
+    Preconditions.checkNotNull(segments, "segments cannot be null");
+    Preconditions.checkArgument(!segments.isEmpty(), "segments cannot be empty");
+    try (LockManager.WriteLock writeLock = lockManager.acquireWriteLock()) {
+      Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed");
+      return nativeCommitExistingIndexSegments(indexName, column, segments);
+    }
+  }
+
+  private native List<Index> nativeCommitExistingIndexSegments(
+      String indexName, String column, List<Index> segments);
+
   /**
    * Count the number of rows in the dataset.
    *
@@ -1207,7 +1262,11 @@ public List<String> listIndexes() {
   /**
    * Get all indexes with full metadata.
    *
-   * @return list of Index objects with complete metadata including index type and fragment coverage
+   * <p>Each returned {@link Index} is a physical index segment from the manifest. Use {@link
+   * #describeIndices()} for the logical-index view.
+   *
+   * @return list of Index objects with complete segment metadata, including index type and fragment
+   *     coverage
    */
   public List<Index> getIndexes() {
     try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
@@ -1266,6 +1325,29 @@ public List<IndexDescription> describeIndices() {
 
   private native List<IndexDescription> nativeDescribeIndices(Optional<IndexCriteria> criteria);
 
+  /**
+   * Read zonemap statistics for a column.
+   *
+   * <p>Returns per-zone min/max/null_count statistics for the given column, if a zonemap index
+   * exists. Returns an empty list if no zonemap index exists for the column.
+   *
+   * <p>The zonemap index file is typically small (one row per zone), so this is a lightweight
+   * metadata-only operation suitable for calling on the driver during scan planning.
+   *
+   * @param columnName the column name
+   * @return list of per-zone statistics, ordered by (fragment_id, zone_start)
+   */
+  public List<ZoneStats> getZonemapStats(String columnName) {
+    Preconditions.checkArgument(
+        columnName != null && !columnName.isEmpty(), "columnName cannot be null or empty");
+    try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
+      Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed");
+      return nativeGetZonemapStats(columnName);
+    }
+  }
+
+  private native List<ZoneStats> nativeGetZonemapStats(String columnName);
+
   /**
    * Get the table config of the dataset.
    *
@@ -1280,6 +1362,23 @@ public Map<String, String> getConfig() {
 
   private native Map<String, String> nativeGetConfig();
 
+  /**
+   * Check whether the dataset uses stable row IDs.
+   *
+   * <p>Stable row IDs remain constant when rows are moved during compaction. This reads the
+   * manifest feature flag directly rather than the user-facing config map.
+   *
+   * @return true if the dataset was created with stable row IDs enabled
+   */
+  public boolean hasStableRowIds() {
+    try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
+      Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed");
+      return nativeHasStableRowIds();
+    }
+  }
+
+  private native boolean nativeHasStableRowIds();
+
   /**
    * Get the Lance file format version of this dataset.
    *
diff --git a/java/src/main/java/org/lance/Fragment.java b/java/src/main/java/org/lance/Fragment.java
index 8eb1f70053d..43091269382 100644
--- a/java/src/main/java/org/lance/Fragment.java
+++ b/java/src/main/java/org/lance/Fragment.java
@@ -15,9 +15,9 @@
 
 import org.lance.fragment.FragmentMergeResult;
 import org.lance.fragment.FragmentUpdateResult;
-import org.lance.io.StorageOptionsProvider;
 import org.lance.ipc.LanceScanner;
 import org.lance.ipc.ScanOptions;
+import org.lance.namespace.LanceNamespace;
 
 import org.apache.arrow.c.ArrowArray;
 import org.apache.arrow.c.ArrowArrayStream;
@@ -234,30 +234,32 @@ public static WriteFragmentBuilder write() {
   @Deprecated
   public static List<FragmentMetadata> create(
       String datasetUri, BufferAllocator allocator, VectorSchemaRoot root, WriteParams params) {
-    return create(datasetUri, allocator, root, params, null);
+    return create(datasetUri, allocator, root, params, null, null);
   }
 
   /**
-   * Create a fragment from the given data with optional storage options provider.
+   * Create a fragment from the given arrow stream.
    *
    * @param datasetUri the dataset uri
-   * @param allocator the buffer allocator
-   * @param root the vector schema root
+   * @param stream the arrow stream
    * @param params the write params
-   * @param storageOptionsProvider optional provider for dynamic storage options with automatic
-   *     credential refresh
    * @return the fragment metadata
-   * @deprecated Use {@link #write()} builder instead. For example: {@code Fragment.write()
-   *     .datasetUri(uri).allocator(allocator).data(root).writeParams(params)
-   *     .storageOptionsProvider(provider).execute()}
+   * @deprecated Use {@link #write()} builder instead.
    */
   @Deprecated
   public static List<FragmentMetadata> create(
+      String datasetUri, ArrowArrayStream stream, WriteParams params) {
+    return create(datasetUri, stream, params, null, null);
+  }
+
+  /** Create a fragment from the given arrow array and schema. */
+  static List<FragmentMetadata> create(
       String datasetUri,
       BufferAllocator allocator,
       VectorSchemaRoot root,
       WriteParams params,
-      StorageOptionsProvider storageOptionsProvider) {
+      LanceNamespace namespaceClient,
+      List<String> tableId) {
     Preconditions.checkNotNull(datasetUri);
     Preconditions.checkNotNull(allocator);
     Preconditions.checkNotNull(root);
@@ -276,45 +278,20 @@ public static List<FragmentMetadata> create(
           params.getEnableStableRowIds(),
           params.getDataStorageVersion(),
           params.getStorageOptions(),
-          Optional.ofNullable(storageOptionsProvider));
+          namespaceClient,
+          tableId,
+          params.getAllowExternalBlobOutsideBases(),
+          params.getBlobPackFileSizeThreshold());
     }
   }
 
-  /**
-   * Create a fragment from the given arrow stream.
-   *
-   * @param datasetUri the dataset uri
-   * @param stream the arrow stream
-   * @param params the write params
-   * @return the fragment metadata
-   * @deprecated Use {@link #write()} builder instead. For example: {@code Fragment.write()
-   *     .datasetUri(uri).data(stream).writeParams(params).execute()}
-   */
-  @Deprecated
-  public static List<FragmentMetadata> create(
-      String datasetUri, ArrowArrayStream stream, WriteParams params) {
-    return create(datasetUri, stream, params, null);
-  }
-
-  /**
-   * Create a fragment from the given arrow stream with optional storage options provider.
-   *
-   * @param datasetUri the dataset uri
-   * @param stream the arrow stream
-   * @param params the write params
-   * @param storageOptionsProvider optional provider for dynamic storage options with automatic
-   *     credential refresh
-   * @return the fragment metadata
-   * @deprecated Use {@link #write()} builder instead. For example: {@code
-   *     Fragment.write().datasetUri(uri).data(stream).writeParams(params)
-   *     .storageOptionsProvider(provider).execute()}
-   */
-  @Deprecated
-  public static List<FragmentMetadata> create(
+  /** Create a fragment from the given arrow stream. */
+  static List<FragmentMetadata> create(
       String datasetUri,
       ArrowArrayStream stream,
       WriteParams params,
-      StorageOptionsProvider storageOptionsProvider) {
+      LanceNamespace namespaceClient,
+      List<String> tableId) {
     Preconditions.checkNotNull(datasetUri);
     Preconditions.checkNotNull(stream);
     Preconditions.checkNotNull(params);
@@ -328,14 +305,13 @@ public static List<FragmentMetadata> create(
         params.getEnableStableRowIds(),
         params.getDataStorageVersion(),
         params.getStorageOptions(),
-        Optional.ofNullable(storageOptionsProvider));
+        namespaceClient,
+        tableId,
+        params.getAllowExternalBlobOutsideBases(),
+        params.getBlobPackFileSizeThreshold());
   }
 
-  /**
-   * Create a fragment from the given arrow array and schema.
-   *
-   * @return the fragment metadata
-   */
+  /** Create a fragment from the given arrow array and schema. */
   private static native List<FragmentMetadata> createWithFfiArray(
       String datasetUri,
       long arrowArrayMemoryAddress,
@@ -347,13 +323,12 @@ private static native List<FragmentMetadata> createWithFfiArray(
       Optional<Boolean> enableStableRowIds,
       Optional<String> dataStorageVersion,
       Map<String, String> storageOptions,
-      Optional<StorageOptionsProvider> storageOptionsProvider);
+      LanceNamespace namespaceClient,
+      List<String> tableId,
+      Optional<Boolean> allowExternalBlobOutsideBases,
+      Optional<Long> blobPackFileSizeThreshold);
 
-  /**
-   * Create a fragment from the given arrow stream.
-   *
-   * @return the fragment metadata
-   */
+  /** Create a fragment from the given arrow stream. */
   private static native List<FragmentMetadata> createWithFfiStream(
       String datasetUri,
       long arrowStreamMemoryAddress,
@@ -364,5 +339,8 @@ private static native List<FragmentMetadata> createWithFfiStream(
       Optional<Boolean> enableStableRowIds,
       Optional<String> dataStorageVersion,
       Map<String, String> storageOptions,
-      Optional<StorageOptionsProvider> storageOptionsProvider);
+      LanceNamespace namespaceClient,
+      List<String> tableId,
+      Optional<Boolean> allowExternalBlobOutsideBases,
+      Optional<Long> blobPackFileSizeThreshold);
 }
diff --git a/java/src/main/java/org/lance/OpenDatasetBuilder.java b/java/src/main/java/org/lance/OpenDatasetBuilder.java
index 85bc19eac6e..4c07864091f 100644
--- a/java/src/main/java/org/lance/OpenDatasetBuilder.java
+++ b/java/src/main/java/org/lance/OpenDatasetBuilder.java
@@ -14,7 +14,6 @@
 package org.lance;
 
 import org.lance.namespace.LanceNamespace;
-import org.lance.namespace.LanceNamespaceStorageOptionsProvider;
 import org.lance.namespace.model.DescribeTableRequest;
 import org.lance.namespace.model.DescribeTableResponse;
 
@@ -42,11 +41,11 @@
  *     .build();
  * }</pre>
  *
- * <p>Example usage with namespace:
+ * <p>Example usage with namespace client:
  *
  * <pre>{@code
  * Dataset dataset = Dataset.open()
- *     .namespace(myNamespace)
+ *     .namespaceClient(myNamespaceClient)
  *     .tableId(Arrays.asList("my_table"))
  *     .build();
  * }</pre>
@@ -55,7 +54,7 @@ public class OpenDatasetBuilder {
   private BufferAllocator allocator;
   private boolean selfManagedAllocator = false;
   private String uri;
-  private LanceNamespace namespace;
+  private LanceNamespace namespaceClient;
   private List<String> tableId;
   private ReadOptions options = new ReadOptions.Builder().build();
   private Session session;
@@ -79,7 +78,7 @@ public OpenDatasetBuilder allocator(BufferAllocator allocator) {
   /**
    * Sets the dataset URI.
    *
-   * <p>Either uri() or namespace()+tableId() must be specified, but not both.
+   * <p>Either uri() or namespaceClient()+tableId() must be specified, but not both.
    *
    * @param uri The dataset URI (e.g., "s3://bucket/table.lance" or "file:///path/to/table.lance")
    * @return this builder instance
@@ -90,24 +89,24 @@ public OpenDatasetBuilder uri(String uri) {
   }
 
   /**
-   * Sets the namespace.
+   * Sets the namespace client.
    *
-   * <p>Must be used together with tableId(). Either uri() or namespace()+tableId() must be
+   * <p>Must be used together with tableId(). Either uri() or namespaceClient()+tableId() must be
    * specified, but not both.
    *
-   * @param namespace The namespace implementation to fetch table info from
+   * @param namespaceClient The namespace implementation to fetch table info from
    * @return this builder instance
    */
-  public OpenDatasetBuilder namespace(LanceNamespace namespace) {
-    this.namespace = namespace;
+  public OpenDatasetBuilder namespaceClient(LanceNamespace namespaceClient) {
+    this.namespaceClient = namespaceClient;
     return this;
   }
 
   /**
    * Sets the table identifier.
    *
-   * <p>Must be used together with namespace(). Either uri() or namespace()+tableId() must be
-   * specified, but not both.
+   * <p>Must be used together with namespaceClient(). Either uri() or namespaceClient()+tableId()
+   * must be specified, but not both.
    *
    * @param tableId The table identifier (e.g., Arrays.asList("my_table"))
    * @return this builder instance
@@ -149,32 +148,33 @@ public OpenDatasetBuilder session(Session session) {
   /**
    * Opens the dataset with the configured parameters.
    *
-   * <p>If a namespace is configured, this automatically fetches the table location and storage
-   * options from the namespace via describeTable().
+   * <p>If a namespace client is configured, this automatically fetches the table location and
+   * storage options from the namespace client via describeTable().
    *
    * @return Dataset
    * @throws IllegalArgumentException if required parameters are missing or invalid
    */
   public Dataset build() {
-    // Validate that exactly one of uri or namespace+tableId is provided
+    // Validate that exactly one of uri or namespaceClient+tableId is provided
     boolean hasUri = uri != null;
-    boolean hasNamespace = namespace != null && tableId != null;
+    boolean hasNamespaceClient = namespaceClient != null && tableId != null;
 
-    if (hasUri && hasNamespace) {
+    if (hasUri && hasNamespaceClient) {
       throw new IllegalArgumentException(
-          "Cannot specify both uri and namespace+tableId. Use one or the other.");
+          "Cannot specify both uri and namespaceClient+tableId. Use one or the other.");
     }
-    if (!hasUri && !hasNamespace) {
-      if (namespace != null) {
+    if (!hasUri && !hasNamespaceClient) {
+      if (namespaceClient != null) {
         throw new IllegalArgumentException(
-            "namespace is set but tableId is missing. Both namespace and tableId must be"
-                + " provided together.");
+            "namespaceClient is set but tableId is missing. Both namespaceClient and tableId must"
+                + " be provided together.");
       } else if (tableId != null) {
         throw new IllegalArgumentException(
-            "tableId is set but namespace is missing. Both namespace and tableId must be"
-                + " provided together.");
+            "tableId is set but namespaceClient is missing. Both namespaceClient and tableId must"
+                + " be provided together.");
       } else {
-        throw new IllegalArgumentException("Either uri or namespace+tableId must be provided.");
+        throw new IllegalArgumentException(
+            "Either uri or namespaceClient+tableId must be provided.");
       }
     }
 
@@ -186,31 +186,31 @@ public Dataset build() {
       selfManagedAllocator = true;
     }
 
-    // Handle namespace-based opening
-    if (hasNamespace) {
-      return buildFromNamespace();
+    // Handle namespace client-based opening
+    if (hasNamespaceClient) {
+      return buildFromNamespaceClient();
     }
 
     // Handle URI-based opening
     return Dataset.open(allocator, selfManagedAllocator, uri, options, session);
   }
 
-  private Dataset buildFromNamespace() {
+  private Dataset buildFromNamespaceClient() {
     // Call describe_table to get location and storage options
     DescribeTableRequest request = new DescribeTableRequest();
     request.setId(tableId);
     // Only set version if present
     options.getVersion().ifPresent(v -> request.setVersion(Long.valueOf(v)));
 
-    DescribeTableResponse response = namespace.describeTable(request);
+    DescribeTableResponse response = namespaceClient.describeTable(request);
 
     String location = response.getLocation();
     if (location == null || location.isEmpty()) {
-      throw new IllegalArgumentException("Namespace did not return a table location");
+      throw new IllegalArgumentException("Namespace client did not return a table location");
     }
 
-    // Check if namespace manages versioning (commits go through namespace API)
-    Boolean managedVersioning = response.getManagedVersioning();
+    // Check if namespace client manages versioning (commits go through namespace client API)
+    Boolean namespaceClientManagedVersioning = response.getManagedVersioning();
 
     Map<String, String> namespaceStorageOptions = response.getStorageOptions();
 
@@ -219,12 +219,6 @@ private Dataset buildFromNamespace() {
             .setIndexCacheSizeBytes(options.getIndexCacheSizeBytes())
             .setMetadataCacheSizeBytes(options.getMetadataCacheSizeBytes());
 
-    if (namespaceStorageOptions != null && !namespaceStorageOptions.isEmpty()) {
-      LanceNamespaceStorageOptionsProvider storageOptionsProvider =
-          new LanceNamespaceStorageOptionsProvider(namespace, tableId);
-      optionsBuilder.setStorageOptionsProvider(storageOptionsProvider);
-    }
-
     options.getVersion().ifPresent(optionsBuilder::setVersion);
     options.getBlockSize().ifPresent(optionsBuilder::setBlockSize);
     options.getSerializedManifest().ifPresent(optionsBuilder::setSerializedManifest);
@@ -235,19 +229,19 @@ private Dataset buildFromNamespace() {
     }
     optionsBuilder.setStorageOptions(storageOptions);
 
-    // If managed_versioning is true, pass namespace for commit handler setup
-    if (Boolean.TRUE.equals(managedVersioning)) {
-      return Dataset.open(
-          allocator,
-          selfManagedAllocator,
-          location,
-          optionsBuilder.build(),
-          session,
-          namespace,
-          tableId);
-    }
-
-    // Open dataset with regular open method (no namespace commit handler)
-    return Dataset.open(allocator, selfManagedAllocator, location, optionsBuilder.build(), session);
+    // Pass namespaceClient, tableId, and namespaceClientManagedVersioning to Rust
+    // The Rust side will:
+    // - Create a storage options provider when namespaceClient/tableId are non-null for credential
+    // refresh
+    // - Create an external manifest commit handler when namespaceClientManagedVersioning is true
+    return Dataset.open(
+        allocator,
+        selfManagedAllocator,
+        location,
+        optionsBuilder.build(),
+        session,
+        namespaceClient,
+        tableId,
+        Boolean.TRUE.equals(namespaceClientManagedVersioning));
   }
 }
diff --git a/java/src/main/java/org/lance/ReadOptions.java b/java/src/main/java/org/lance/ReadOptions.java
index b9a244c55a5..8fd65c1e3e6 100644
--- a/java/src/main/java/org/lance/ReadOptions.java
+++ b/java/src/main/java/org/lance/ReadOptions.java
@@ -13,8 +13,6 @@
  */
 package org.lance;
 
-import org.lance.io.StorageOptionsProvider;
-
 import com.google.common.base.MoreObjects;
 
 import java.nio.ByteBuffer;
@@ -31,7 +29,6 @@ public class ReadOptions {
   private final long metadataCacheSizeBytes;
   private final Optional<ByteBuffer> serializedManifest;
   private final Map<String, String> storageOptions;
-  private final Optional<StorageOptionsProvider> storageOptionsProvider;
   private final Optional<Session> session;
 
   private ReadOptions(Builder builder) {
@@ -41,7 +38,6 @@ private ReadOptions(Builder builder) {
     this.metadataCacheSizeBytes = builder.metadataCacheSizeBytes;
     this.storageOptions = builder.storageOptions;
     this.serializedManifest = builder.serializedManifest;
-    this.storageOptionsProvider = builder.storageOptionsProvider;
     this.session = builder.session;
   }
 
@@ -69,10 +65,6 @@ public Optional<ByteBuffer> getSerializedManifest() {
     return serializedManifest;
   }
 
-  public Optional<StorageOptionsProvider> getStorageOptionsProvider() {
-    return storageOptionsProvider;
-  }
-
   /**
    * Get the session to use for opening the dataset.
    *
@@ -104,7 +96,6 @@ public static class Builder {
     private long metadataCacheSizeBytes = 1024L * 1024 * 1024; // Default to 1 GiB like Rust
     private Map<String, String> storageOptions = new HashMap<>();
     private Optional<ByteBuffer> serializedManifest = Optional.empty();
-    private Optional<StorageOptionsProvider> storageOptionsProvider = Optional.empty();
     private Optional<Session> session = Optional.empty();
 
     /**
@@ -211,21 +202,6 @@ public Builder setSerializedManifest(ByteBuffer serializedManifest) {
       return this;
     }
 
-    /**
-     * Set a custom storage options provider for automatic storage options refresh.
-     *
-     * <p>The storage options provider will be called automatically before storage options expire,
-     * enabling long-running operations on cloud storage without interruption. This is currently
-     * only used for refreshing AWS temporary access credentials.
-     *
-     * @param storageOptionsProvider the storage options provider implementation
-     * @return this builder
-     */
-    public Builder setStorageOptionsProvider(StorageOptionsProvider storageOptionsProvider) {
-      this.storageOptionsProvider = Optional.of(storageOptionsProvider);
-      return this;
-    }
-
     /**
      * Set a session to share caches between multiple datasets.
      *
diff --git a/java/src/main/java/org/lance/WriteDatasetBuilder.java b/java/src/main/java/org/lance/WriteDatasetBuilder.java
index a95561acb2b..3c06b43fd54 100644
--- a/java/src/main/java/org/lance/WriteDatasetBuilder.java
+++ b/java/src/main/java/org/lance/WriteDatasetBuilder.java
@@ -13,9 +13,7 @@
  */
 package org.lance;
 
-import org.lance.io.StorageOptionsProvider;
 import org.lance.namespace.LanceNamespace;
-import org.lance.namespace.LanceNamespaceStorageOptionsProvider;
 import org.lance.namespace.model.DeclareTableRequest;
 import org.lance.namespace.model.DeclareTableResponse;
 import org.lance.namespace.model.DescribeTableRequest;
@@ -51,12 +49,12 @@
  *     .execute();
  * }</pre>
  *
- * <p>Example usage with namespace:
+ * <p>Example usage with namespace client:
  *
  * <pre>{@code
  * Dataset dataset = Dataset.write(allocator)
  *     .reader(myReader)
- *     .namespace(myNamespace)
+ *     .namespaceClient(myNamespaceClient)
  *     .tableId(Arrays.asList("my_table"))
  *     .mode(WriteMode.CREATE)
  *     .execute();
@@ -67,7 +65,7 @@ public class WriteDatasetBuilder {
   private ArrowReader reader;
   private ArrowArrayStream stream;
   private String uri;
-  private LanceNamespace namespace;
+  private LanceNamespace namespaceClient;
   private List<String> tableId;
   private WriteParams.WriteMode mode = WriteParams.WriteMode.CREATE;
   private Schema schema;
@@ -80,6 +78,8 @@ public class WriteDatasetBuilder {
   private Optional<String> dataStorageVersion = Optional.empty();
   private Optional<List<BasePath>> initialBases = Optional.empty();
   private Optional<List<String>> targetBases = Optional.empty();
+  private Optional<Boolean> allowExternalBlobOutsideBases = Optional.empty();
+  private Optional<Long> blobPackFileSizeThreshold = Optional.empty();
   private Session session;
 
   /** Creates a new builder instance. Package-private, use Dataset.write() instead. */
@@ -132,7 +132,7 @@ public WriteDatasetBuilder stream(ArrowArrayStream stream) {
   /**
    * Sets the dataset URI.
    *
-   * <p>Either uri() or namespace()+tableId() must be specified, but not both.
+   * <p>Either uri() or namespaceClient()+tableId() must be specified, but not both.
    *
    * @param uri The dataset URI (e.g., "s3://bucket/table.lance" or "file:///path/to/table.lance")
    * @return this builder instance
@@ -143,24 +143,24 @@ public WriteDatasetBuilder uri(String uri) {
   }
 
   /**
-   * Sets the namespace.
+   * Sets the namespace client.
    *
-   * <p>Must be used together with tableId(). Either uri() or namespace()+tableId() must be
+   * <p>Must be used together with tableId(). Either uri() or namespaceClient()+tableId() must be
    * specified, but not both.
    *
-   * @param namespace The namespace implementation to use for table operations
+   * @param namespaceClient The namespace implementation to use for table operations
    * @return this builder instance
    */
-  public WriteDatasetBuilder namespace(LanceNamespace namespace) {
-    this.namespace = namespace;
+  public WriteDatasetBuilder namespaceClient(LanceNamespace namespaceClient) {
+    this.namespaceClient = namespaceClient;
     return this;
   }
 
   /**
    * Sets the table identifier.
    *
-   * <p>Must be used together with namespace(). Either uri() or namespace()+tableId() must be
-   * specified, but not both.
+   * <p>Must be used together with namespaceClient(). Either uri() or namespaceClient()+tableId()
+   * must be specified, but not both.
    *
    * @param tableId The table identifier (e.g., Arrays.asList("my_table"))
    * @return this builder instance
@@ -207,10 +207,11 @@ public WriteDatasetBuilder storageOptions(Map<String, String> storageOptions) {
   }
 
   /**
-   * Sets whether to ignore storage options from the namespace's describeTable() or declareTable().
+   * Sets whether to ignore storage options from the namespace client's describeTable() or
+   * declareTable().
    *
-   * @param ignoreNamespaceStorageOptions If true, storage options returned from namespace will be
-   *     ignored
+   * @param ignoreNamespaceStorageOptions If true, storage options returned from namespace client
+   *     will be ignored
    * @return this builder instance
    */
   public WriteDatasetBuilder ignoreNamespaceStorageOptions(boolean ignoreNamespaceStorageOptions) {
@@ -283,6 +284,30 @@ public WriteDatasetBuilder targetBases(List<String> targetBases) {
     return this;
   }
 
+  /**
+   * Sets whether to allow external blob URIs outside registered base paths.
+   *
+   * @param allowExternalBlobOutsideBases Whether to allow external blob URIs outside bases
+   * @return this builder instance
+   */
+  public WriteDatasetBuilder allowExternalBlobOutsideBases(boolean allowExternalBlobOutsideBases) {
+    this.allowExternalBlobOutsideBases = Optional.of(allowExternalBlobOutsideBases);
+    return this;
+  }
+
+  /**
+   * Sets the maximum size in bytes for blob v2 pack (.blob) sidecar files.
+   *
+   * <p>When a pack file reaches this size, a new one is started. If not set, defaults to 1 GiB.
+   *
+   * @param blobPackFileSizeThreshold maximum pack file size in bytes
+   * @return this builder instance
+   */
+  public WriteDatasetBuilder blobPackFileSizeThreshold(long blobPackFileSizeThreshold) {
+    this.blobPackFileSizeThreshold = Optional.of(blobPackFileSizeThreshold);
+    return this;
+  }
+
   /**
    * Sets the session to share caches with other datasets.
    *
@@ -301,8 +326,9 @@ public WriteDatasetBuilder session(Session session) {
   /**
    * Executes the write operation and returns the created dataset.
    *
-   * <p>If a namespace is configured via namespace()+tableId(), this automatically handles table
-   * creation or retrieval through the namespace API with credential vending support.
+   * <p>If a namespace client is configured via namespaceClient()+tableId(), this automatically
+   * handles table creation or retrieval through the namespace client API with credential vending
+   * support.
    *
    * @return Dataset
    * @throws IllegalArgumentException if required parameters are missing or invalid
@@ -313,23 +339,24 @@ public Dataset execute() {
       allocator = new RootAllocator(Long.MAX_VALUE);
     }
 
-    // Validate that exactly one of uri or namespace is provided
+    // Validate that exactly one of uri or namespaceClient is provided
     boolean hasUri = uri != null;
-    boolean hasNamespace = namespace != null && tableId != null;
+    boolean hasNamespaceClient = namespaceClient != null && tableId != null;
 
-    if (hasUri && hasNamespace) {
+    if (hasUri && hasNamespaceClient) {
       throw new IllegalArgumentException(
-          "Cannot specify both uri() and namespace()+tableId(). Use one or the other.");
+          "Cannot specify both uri() and namespaceClient()+tableId(). Use one or the other.");
     }
-    if (!hasUri && !hasNamespace) {
-      if (namespace != null) {
+    if (!hasUri && !hasNamespaceClient) {
+      if (namespaceClient != null) {
         throw new IllegalArgumentException(
-            "namespace() is set but tableId() is missing. Both must be provided together.");
+            "namespaceClient() is set but tableId() is missing. Both must be provided together.");
       } else if (tableId != null) {
         throw new IllegalArgumentException(
-            "tableId() is set but namespace() is missing. Both must be provided together.");
+            "tableId() is set but namespaceClient() is missing. Both must be provided together.");
       } else {
-        throw new IllegalArgumentException("Either uri() or namespace()+tableId() must be called.");
+        throw new IllegalArgumentException(
+            "Either uri() or namespaceClient()+tableId() must be called.");
       }
     }
 
@@ -349,51 +376,53 @@ public Dataset execute() {
               + "Use only one of: reader(), stream(), or schema().");
     }
 
-    // Handle namespace-based writing
-    if (hasNamespace) {
-      return executeWithNamespace();
+    // Handle namespace client-based writing
+    if (hasNamespaceClient) {
+      return executeWithNamespaceClient();
     }
 
     // Handle URI-based writing
     return executeWithUri();
   }
 
-  private Dataset executeWithNamespace() {
+  private Dataset executeWithNamespaceClient() {
     String tableUri;
     Map<String, String> namespaceStorageOptions = null;
-    boolean managedVersioning = false;
+    boolean namespaceClientManagedVersioning = false;
 
-    // Mode-specific namespace operations
+    // Mode-specific namespace client operations
     if (mode == WriteParams.WriteMode.CREATE) {
       DeclareTableRequest declareRequest = new DeclareTableRequest();
       declareRequest.setId(tableId);
-      DeclareTableResponse declareResponse = namespace.declareTable(declareRequest);
+      DeclareTableResponse declareResponse = namespaceClient.declareTable(declareRequest);
 
       tableUri = declareResponse.getLocation();
       if (tableUri == null || tableUri.isEmpty()) {
-        throw new IllegalArgumentException("Namespace did not return a table location");
+        throw new IllegalArgumentException("Namespace client did not return a table location");
       }
 
-      managedVersioning = Boolean.TRUE.equals(declareResponse.getManagedVersioning());
+      namespaceClientManagedVersioning =
+          Boolean.TRUE.equals(declareResponse.getManagedVersioning());
       namespaceStorageOptions =
           ignoreNamespaceStorageOptions ? null : declareResponse.getStorageOptions();
     } else {
-      // For APPEND/OVERWRITE modes, call namespace.describeTable()
+      // For APPEND/OVERWRITE modes, call namespaceClient.describeTable()
       DescribeTableRequest request = new DescribeTableRequest();
       request.setId(tableId);
 
-      DescribeTableResponse response = namespace.describeTable(request);
+      DescribeTableResponse response = namespaceClient.describeTable(request);
 
       tableUri = response.getLocation();
       if (tableUri == null || tableUri.isEmpty()) {
-        throw new IllegalArgumentException("Namespace did not return a table location");
+        throw new IllegalArgumentException("Namespace client did not return a table location");
       }
 
       namespaceStorageOptions = ignoreNamespaceStorageOptions ? null : response.getStorageOptions();
-      managedVersioning = Boolean.TRUE.equals(response.getManagedVersioning());
+      namespaceClientManagedVersioning = Boolean.TRUE.equals(response.getManagedVersioning());
     }
 
-    // Merge storage options (namespace options + user options, with namespace taking precedence)
+    // Merge storage options (namespace client options + user options, with namespace client taking
+    // precedence)
     Map<String, String> mergedStorageOptions = new HashMap<>(storageOptions);
     if (namespaceStorageOptions != null && !namespaceStorageOptions.isEmpty()) {
       mergedStorageOptions.putAll(namespaceStorageOptions);
@@ -411,21 +440,26 @@ private Dataset executeWithNamespace() {
 
     initialBases.ifPresent(paramsBuilder::withInitialBases);
     targetBases.ifPresent(paramsBuilder::withTargetBases);
+    allowExternalBlobOutsideBases.ifPresent(paramsBuilder::withAllowExternalBlobOutsideBases);
+    blobPackFileSizeThreshold.ifPresent(paramsBuilder::withBlobPackFileSizeThreshold);
 
     WriteParams params = paramsBuilder.build();
 
-    // Create storage options provider for credential refresh during long-running writes
-    StorageOptionsProvider storageOptionsProvider =
-        ignoreNamespaceStorageOptions
-            ? null
-            : new LanceNamespaceStorageOptionsProvider(namespace, tableId);
-
-    // Only use namespace for commit handling if managedVersioning is enabled
-    if (managedVersioning) {
-      return createDatasetWithStreamAndNamespace(
-          tableUri, params, storageOptionsProvider, namespace, tableId);
+    // Pass namespaceClient, tableId, and namespaceClientManagedVersioning to JNI
+    // Rust will automatically create a storage options provider when namespaceClient/tableId
+    // are non-null for credential refresh, and will create an external manifest commit handler
+    // when namespaceClientManagedVersioning is true
+    if (namespaceClientManagedVersioning) {
+      return createDatasetWithStreamAndNamespaceClient(
+          tableUri, params, namespaceClient, tableId, true);
     } else {
-      return createDatasetWithStream(tableUri, params, storageOptionsProvider);
+      // Even without managed versioning, pass namespaceClient for credential refresh
+      // when namespace client vends credentials (storage options was non-null)
+      if (!ignoreNamespaceStorageOptions && namespaceStorageOptions != null) {
+        return createDatasetWithStreamAndNamespaceClient(
+            tableUri, params, namespaceClient, tableId, false);
+      }
+      return createDatasetWithStream(tableUri, params);
     }
   }
 
@@ -440,24 +474,25 @@ private Dataset executeWithUri() {
     dataStorageVersion.ifPresent(paramsBuilder::withDataStorageVersion);
     initialBases.ifPresent(paramsBuilder::withInitialBases);
     targetBases.ifPresent(paramsBuilder::withTargetBases);
+    allowExternalBlobOutsideBases.ifPresent(paramsBuilder::withAllowExternalBlobOutsideBases);
+    blobPackFileSizeThreshold.ifPresent(paramsBuilder::withBlobPackFileSizeThreshold);
 
     WriteParams params = paramsBuilder.build();
 
-    return createDatasetWithStream(uri, params, null);
+    return createDatasetWithStream(uri, params);
   }
 
-  private Dataset createDatasetWithStream(
-      String path, WriteParams params, StorageOptionsProvider storageOptionsProvider) {
+  private Dataset createDatasetWithStream(String path, WriteParams params) {
     // If stream is directly provided, use it
     if (stream != null) {
-      return Dataset.create(allocator, stream, path, params, storageOptionsProvider);
+      return Dataset.create(allocator, stream, path, params);
     }
 
     // If reader is provided, convert to stream
     if (reader != null) {
       try (ArrowArrayStream tempStream = ArrowArrayStream.allocateNew(allocator)) {
         Data.exportArrayStream(allocator, reader, tempStream);
-        return Dataset.create(allocator, tempStream, path, params, storageOptionsProvider);
+        return Dataset.create(allocator, tempStream, path, params);
       }
     }
 
@@ -469,16 +504,22 @@ private Dataset createDatasetWithStream(
     throw new IllegalStateException("No data source provided");
   }
 
-  private Dataset createDatasetWithStreamAndNamespace(
+  private Dataset createDatasetWithStreamAndNamespaceClient(
       String path,
       WriteParams params,
-      StorageOptionsProvider storageOptionsProvider,
-      LanceNamespace namespace,
-      List<String> tableId) {
+      LanceNamespace namespaceClient,
+      List<String> tableId,
+      boolean namespaceClientManagedVersioning) {
     // If stream is directly provided, use it
     if (stream != null) {
       return Dataset.create(
-          allocator, stream, path, params, storageOptionsProvider, namespace, tableId);
+          allocator,
+          stream,
+          path,
+          params,
+          namespaceClient,
+          tableId,
+          namespaceClientManagedVersioning);
     }
 
     // If reader is provided, convert to stream
@@ -486,12 +527,18 @@ private Dataset createDatasetWithStreamAndNamespace(
       try (ArrowArrayStream tempStream = ArrowArrayStream.allocateNew(allocator)) {
         Data.exportArrayStream(allocator, reader, tempStream);
         return Dataset.create(
-            allocator, tempStream, path, params, storageOptionsProvider, namespace, tableId);
+            allocator,
+            tempStream,
+            path,
+            params,
+            namespaceClient,
+            tableId,
+            namespaceClientManagedVersioning);
       }
     }
 
     // If only schema is provided (empty table), use Dataset.create with schema
-    // Note: Schema-only creation doesn't support namespace-based commit handling
+    // Note: Schema-only creation doesn't support namespace client-based commit handling
     if (schema != null) {
       return Dataset.create(allocator, path, schema, params);
     }
diff --git a/java/src/main/java/org/lance/WriteFragmentBuilder.java b/java/src/main/java/org/lance/WriteFragmentBuilder.java
index 42ccf2d8dd3..693b5b6bc87 100644
--- a/java/src/main/java/org/lance/WriteFragmentBuilder.java
+++ b/java/src/main/java/org/lance/WriteFragmentBuilder.java
@@ -13,7 +13,7 @@
  */
 package org.lance;
 
-import org.lance.io.StorageOptionsProvider;
+import org.lance.namespace.LanceNamespace;
 
 import org.apache.arrow.c.ArrowArrayStream;
 import org.apache.arrow.memory.BufferAllocator;
@@ -47,7 +47,8 @@ public class WriteFragmentBuilder {
   private ArrowArrayStream arrowArrayStream;
   private WriteParams writeParams;
   private WriteParams.Builder writeParamsBuilder;
-  private StorageOptionsProvider storageOptionsProvider;
+  private LanceNamespace namespaceClient;
+  private List<String> tableId;
 
   WriteFragmentBuilder() {}
 
@@ -123,13 +124,30 @@ public WriteFragmentBuilder storageOptions(Map<String, String> storageOptions) {
   }
 
   /**
-   * Set the storage options provider for dynamic credential refresh.
+   * Set the namespace client for automatic credential refresh.
    *
-   * @param provider the storage options provider
+   * <p>When provided with `tableId`, a storage options provider will be created automatically to
+   * refresh credentials via the namespace client. Must be provided together with `tableId`. The
+   * caller should provide initial/merged storage options via the `storageOptions` method.
+   *
+   * @param namespaceClient the LanceNamespace client instance
+   * @return this builder
+   */
+  public WriteFragmentBuilder namespaceClient(LanceNamespace namespaceClient) {
+    this.namespaceClient = namespaceClient;
+    return this;
+  }
+
+  /**
+   * Set the table ID for namespace client-based credential refresh.
+   *
+   * <p>Must be provided together with `namespaceClient`.
+   *
+   * @param tableId the table identifier (e.g., ["workspace", "table_name"])
    * @return this builder
    */
-  public WriteFragmentBuilder storageOptionsProvider(StorageOptionsProvider provider) {
-    this.storageOptionsProvider = provider;
+  public WriteFragmentBuilder tableId(List<String> tableId) {
+    this.tableId = tableId;
     return this;
   }
 
@@ -213,15 +231,17 @@ public WriteFragmentBuilder dataStorageVersion(String version) {
   public List<FragmentMetadata> execute() {
     validate();
 
-    // Build the write params if builder was used
+    // Build the write params
     WriteParams finalWriteParams = buildWriteParams();
 
+    // Pass namespaceClient and tableId to JNI - Rust will automatically create a
+    // storage options provider when these are non-null for credential refresh
     if (vectorSchemaRoot != null) {
       return Fragment.create(
-          datasetUri, allocator, vectorSchemaRoot, finalWriteParams, storageOptionsProvider);
+          datasetUri, allocator, vectorSchemaRoot, finalWriteParams, namespaceClient, tableId);
     } else {
       return Fragment.create(
-          datasetUri, arrowArrayStream, finalWriteParams, storageOptionsProvider);
+          datasetUri, arrowArrayStream, finalWriteParams, namespaceClient, tableId);
     }
   }
 
@@ -255,5 +275,9 @@ private void validate() {
     Preconditions.checkState(
         writeParams == null || writeParamsBuilder == null,
         "Cannot use both writeParams() and individual parameter methods");
+    Preconditions.checkState(
+        (namespaceClient == null && tableId == null)
+            || (namespaceClient != null && tableId != null),
+        "Both 'namespaceClient' and 'tableId' must be provided together");
   }
 }
diff --git a/java/src/main/java/org/lance/WriteParams.java b/java/src/main/java/org/lance/WriteParams.java
index 8a04a5d5b96..970007f442f 100644
--- a/java/src/main/java/org/lance/WriteParams.java
+++ b/java/src/main/java/org/lance/WriteParams.java
@@ -40,6 +40,8 @@ public enum WriteMode {
   private Map<String, String> storageOptions = new HashMap<>();
   private final Optional<List<BasePath>> initialBases;
   private final Optional<List<String>> targetBases;
+  private final Optional<Boolean> allowExternalBlobOutsideBases;
+  private final Optional<Long> blobPackFileSizeThreshold;
 
   private WriteParams(
       Optional<Integer> maxRowsPerFile,
@@ -51,7 +53,9 @@ private WriteParams(
       Optional<Boolean> enableV2ManifestPaths,
       Map<String, String> storageOptions,
       Optional<List<BasePath>> initialBases,
-      Optional<List<String>> targetBases) {
+      Optional<List<String>> targetBases,
+      Optional<Boolean> allowExternalBlobOutsideBases,
+      Optional<Long> blobPackFileSizeThreshold) {
     this.maxRowsPerFile = maxRowsPerFile;
     this.maxRowsPerGroup = maxRowsPerGroup;
     this.maxBytesPerFile = maxBytesPerFile;
@@ -62,6 +66,8 @@ private WriteParams(
     this.storageOptions = storageOptions;
     this.initialBases = initialBases;
     this.targetBases = targetBases;
+    this.allowExternalBlobOutsideBases = allowExternalBlobOutsideBases;
+    this.blobPackFileSizeThreshold = blobPackFileSizeThreshold;
   }
 
   public Optional<Integer> getMaxRowsPerFile() {
@@ -109,6 +115,29 @@ public Optional<List<String>> getTargetBases() {
     return targetBases;
   }
 
+  /**
+   * Get whether external blob URIs outside registered bases are allowed.
+   *
+   * <p>When true, blob v2 columns can reference external URIs that are not under any registered
+   * base path. The URI is stored as an absolute external reference with base_id=0.
+   *
+   * @return Optional containing the setting, or empty if not set
+   */
+  public Optional<Boolean> getAllowExternalBlobOutsideBases() {
+    return allowExternalBlobOutsideBases;
+  }
+
+  /**
+   * Get the maximum size in bytes for blob v2 pack (.blob) sidecar files.
+   *
+   * <p>When a pack file reaches this size, a new one is started. If not set, defaults to 1 GiB.
+   *
+   * @return Optional containing the max pack file size in bytes, or empty if not set
+   */
+  public Optional<Long> getBlobPackFileSizeThreshold() {
+    return blobPackFileSizeThreshold;
+  }
+
   @Override
   public String toString() {
     return MoreObjects.toStringHelper(this)
@@ -132,6 +161,8 @@ public static class Builder {
     private Map<String, String> storageOptions = new HashMap<>();
     private Optional<List<BasePath>> initialBases = Optional.empty();
     private Optional<List<String>> targetBases = Optional.empty();
+    private Optional<Boolean> allowExternalBlobOutsideBases = Optional.empty();
+    private Optional<Long> blobPackFileSizeThreshold = Optional.empty();
 
     public Builder withMaxRowsPerFile(int maxRowsPerFile) {
       this.maxRowsPerFile = Optional.of(maxRowsPerFile);
@@ -183,6 +214,34 @@ public Builder withTargetBases(List<String> targetBases) {
       return this;
     }
 
+    /**
+     * Allow external blob URIs outside registered bases.
+     *
+     * <p>When true, blob v2 columns can reference external URIs (e.g. pointing to blob files in
+     * another Lance dataset) that are not under any registered base path. The URI is stored as an
+     * absolute external reference with base_id=0.
+     *
+     * @param allow true to allow external blob URIs outside bases
+     * @return this builder
+     */
+    public Builder withAllowExternalBlobOutsideBases(boolean allow) {
+      this.allowExternalBlobOutsideBases = Optional.of(allow);
+      return this;
+    }
+
+    /**
+     * Set the maximum size in bytes for blob v2 pack (.blob) sidecar files.
+     *
+     * <p>When a pack file reaches this size, a new one is started. If not set, defaults to 1 GiB.
+     *
+     * @param maxBytes maximum pack file size in bytes
+     * @return this builder
+     */
+    public Builder withBlobPackFileSizeThreshold(long maxBytes) {
+      this.blobPackFileSizeThreshold = Optional.of(maxBytes);
+      return this;
+    }
+
     public WriteParams build() {
       return new WriteParams(
           maxRowsPerFile,
@@ -194,7 +253,9 @@ public WriteParams build() {
           enableV2ManifestPaths,
           storageOptions,
           initialBases,
-          targetBases);
+          targetBases,
+          allowExternalBlobOutsideBases,
+          blobPackFileSizeThreshold);
     }
   }
 }
diff --git a/java/src/main/java/org/lance/compaction/Compaction.java b/java/src/main/java/org/lance/compaction/Compaction.java
index 4845f2502ed..0ce7050900c 100644
--- a/java/src/main/java/org/lance/compaction/Compaction.java
+++ b/java/src/main/java/org/lance/compaction/Compaction.java
@@ -43,7 +43,8 @@ public static CompactionPlan planCompaction(
         compactionOptions.getBatchSize(),
         compactionOptions.getDeferIndexRemap(),
         compactionOptions.getCompactionMode(),
-        compactionOptions.getBinaryCopyReadBatchBytes());
+        compactionOptions.getBinaryCopyReadBatchBytes(),
+        compactionOptions.getMaxSourceFragments());
   }
 
   public static CompactionMetrics commitCompaction(
@@ -63,7 +64,8 @@ public static CompactionMetrics commitCompaction(
         compactionOptions.getBatchSize(),
         compactionOptions.getDeferIndexRemap(),
         compactionOptions.getCompactionMode(),
-        compactionOptions.getBinaryCopyReadBatchBytes());
+        compactionOptions.getBinaryCopyReadBatchBytes(),
+        compactionOptions.getMaxSourceFragments());
   }
 
   public static native CompactionMetrics nativeCommitCompaction(
@@ -78,7 +80,8 @@ public static native CompactionMetrics nativeCommitCompaction(
       Optional<Long> batchSize,
       Optional<Boolean> deferIndexRemap,
       Optional<String> compactionMode,
-      Optional<Long> binaryCopyReadBatchBytes);
+      Optional<Long> binaryCopyReadBatchBytes,
+      Optional<Long> maxSourceFragments);
 
   private static native CompactionPlan nativePlanCompaction(
       Dataset dataset,
@@ -91,5 +94,6 @@ private static native CompactionPlan nativePlanCompaction(
       Optional<Long> batchSize,
       Optional<Boolean> deferIndexRemap,
       Optional<String> compactionMode,
-      Optional<Long> binaryCopyReadBatchBytes);
+      Optional<Long> binaryCopyReadBatchBytes,
+      Optional<Long> maxSourceFragments);
 }
diff --git a/java/src/main/java/org/lance/compaction/CompactionOptions.java b/java/src/main/java/org/lance/compaction/CompactionOptions.java
index d2d483f4970..7c3d65ffc3f 100644
--- a/java/src/main/java/org/lance/compaction/CompactionOptions.java
+++ b/java/src/main/java/org/lance/compaction/CompactionOptions.java
@@ -39,6 +39,7 @@ public class CompactionOptions implements Serializable {
   private Optional<Boolean> deferIndexRemap;
   private Optional<CompactionMode> compactionMode;
   private Optional<Long> binaryCopyReadBatchBytes;
+  private Optional<Long> maxSourceFragments;
 
   private CompactionOptions(
       Optional<Long> targetRowsPerFragment,
@@ -50,7 +51,8 @@ private CompactionOptions(
       Optional<Long> batchSize,
       Optional<Boolean> deferIndexRemap,
       Optional<CompactionMode> compactionMode,
-      Optional<Long> binaryCopyReadBatchBytes) {
+      Optional<Long> binaryCopyReadBatchBytes,
+      Optional<Long> maxSourceFragments) {
     this.targetRowsPerFragment = targetRowsPerFragment;
     this.maxRowsPerGroup = maxRowsPerGroup;
     this.maxBytesPerFile = maxBytesPerFile;
@@ -61,6 +63,7 @@ private CompactionOptions(
     this.deferIndexRemap = deferIndexRemap;
     this.compactionMode = compactionMode;
     this.binaryCopyReadBatchBytes = binaryCopyReadBatchBytes;
+    this.maxSourceFragments = maxSourceFragments;
   }
 
   public Optional<Boolean> getDeferIndexRemap() {
@@ -76,6 +79,10 @@ public Optional<Long> getBinaryCopyReadBatchBytes() {
     return binaryCopyReadBatchBytes;
   }
 
+  public Optional<Long> getMaxSourceFragments() {
+    return maxSourceFragments;
+  }
+
   public Optional<Boolean> getMaterializeDeletions() {
     return materializeDeletions;
   }
@@ -121,6 +128,7 @@ public String toString() {
         .add("deferIndexRemap", deferIndexRemap.orElse(null))
         .add("compactionMode", compactionMode.orElse(null))
         .add("binaryCopyReadBatchBytes", binaryCopyReadBatchBytes.orElse(null))
+        .add("maxSourceFragments", maxSourceFragments.orElse(null))
         .toString();
   }
 
@@ -135,6 +143,7 @@ private void writeObject(ObjectOutputStream output) throws IOException {
     output.writeObject(deferIndexRemap.orElse(null));
     output.writeObject(compactionMode.map(CompactionMode::getValue).orElse(null));
     output.writeObject(binaryCopyReadBatchBytes.orElse(null));
+    output.writeObject(maxSourceFragments.orElse(null));
   }
 
   private void readObject(ObjectInputStream input) throws IOException, ClassNotFoundException {
@@ -157,6 +166,7 @@ private void readObject(ObjectInputStream input) throws IOException, ClassNotFou
       }
     }
     this.binaryCopyReadBatchBytes = Optional.ofNullable((Long) input.readObject());
+    this.maxSourceFragments = Optional.ofNullable((Long) input.readObject());
   }
 
   /** Builder for CompactionOptions. */
@@ -171,6 +181,7 @@ public static class Builder {
     private Optional<Boolean> deferIndexRemap = Optional.empty();
     private Optional<CompactionMode> compactionMode = Optional.empty();
     private Optional<Long> binaryCopyReadBatchBytes = Optional.empty();
+    private Optional<Long> maxSourceFragments = Optional.empty();
 
     private Builder() {}
 
@@ -224,6 +235,16 @@ public Builder withBinaryCopyReadBatchBytes(long binaryCopyReadBatchBytes) {
       return this;
     }
 
+    /**
+     * Maximum number of source fragments to compact in a single run. Tasks are included until
+     * adding the next task would exceed this limit, allowing for incremental compaction. Fragments
+     * are processed oldest first.
+     */
+    public Builder withMaxSourceFragments(long maxSourceFragments) {
+      this.maxSourceFragments = Optional.of(maxSourceFragments);
+      return this;
+    }
+
     public CompactionOptions build() {
       return new CompactionOptions(
           targetRowsPerFragment,
@@ -235,7 +256,8 @@ public CompactionOptions build() {
           batchSize,
           deferIndexRemap,
           compactionMode,
-          binaryCopyReadBatchBytes);
+          binaryCopyReadBatchBytes,
+          maxSourceFragments);
     }
   }
 }
diff --git a/java/src/main/java/org/lance/compaction/CompactionTask.java b/java/src/main/java/org/lance/compaction/CompactionTask.java
index 4d3d7b3b751..89ec364e980 100644
--- a/java/src/main/java/org/lance/compaction/CompactionTask.java
+++ b/java/src/main/java/org/lance/compaction/CompactionTask.java
@@ -55,7 +55,8 @@ public RewriteResult execute(Dataset dataset) {
         compactionOptions.getBatchSize(),
         compactionOptions.getDeferIndexRemap(),
         compactionOptions.getCompactionMode(),
-        compactionOptions.getBinaryCopyReadBatchBytes());
+        compactionOptions.getBinaryCopyReadBatchBytes(),
+        compactionOptions.getMaxSourceFragments());
   }
 
   private native RewriteResult nativeExecute(
@@ -71,7 +72,8 @@ private native RewriteResult nativeExecute(
       Optional<Long> batchSize,
       Optional<Boolean> deferIndexRemap,
       Optional<String> compactionMode,
-      Optional<Long> binaryCopyReadBatchBytes);
+      Optional<Long> binaryCopyReadBatchBytes,
+      Optional<Long> maxSourceFragments);
 
   public CompactionOptions getCompactionOptions() {
     return compactionOptions;
diff --git a/java/src/main/java/org/lance/index/IndexDescription.java b/java/src/main/java/org/lance/index/IndexDescription.java
index d17782eb531..1b5e5a3a8f8 100755
--- a/java/src/main/java/org/lance/index/IndexDescription.java
+++ b/java/src/main/java/org/lance/index/IndexDescription.java
@@ -83,6 +83,15 @@ public List<Index> getMetadata() {
     return metadata;
   }
 
+  /**
+   * Physical index segments for this logical index.
+   *
+   * <p>This is an alias for {@link #getMetadata()} with a less ambiguous name.
+   */
+  public List<Index> getSegments() {
+    return metadata;
+  }
+
   /**
    * JSON representation of index-specific details.
    *
diff --git a/java/src/main/java/org/lance/index/scalar/ZoneStats.java b/java/src/main/java/org/lance/index/scalar/ZoneStats.java
new file mode 100644
index 00000000000..b97042ed226
--- /dev/null
+++ b/java/src/main/java/org/lance/index/scalar/ZoneStats.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance.index.scalar;
+
+import java.io.Serializable;
+
+/**
+ * Per-zone statistics from a zonemap index.
+ *
+ * <p>Each zone covers a contiguous range of row offsets within a single fragment. The min/max
+ * values are represented as {@link Comparable} objects (Long, Double, String, etc.) matching the
+ * column's data type.
+ *
+ * <p>A zone never spans fragment boundaries: if a fragment has more rows than the zone size,
+ * multiple zones are created within that fragment.
+ *
+ * <p>This class is populated from the Rust {@code ZoneMapStatistics} via JNI. See {@code
+ * lance-index/src/scalar/zonemap.rs} for the on-disk format.
+ */
+public class ZoneStats implements Serializable {
+  private static final long serialVersionUID = 1L;
+
+  private final int fragmentId;
+  private final long zoneStart;
+  private final long zoneLength;
+  private final Comparable<?> min;
+  private final Comparable<?> max;
+  private final long nullCount;
+
+  /**
+   * Constructs a new ZoneStats instance.
+   *
+   * @param fragmentId the fragment this zone belongs to
+   * @param zoneStart the starting row offset within the fragment
+   * @param zoneLength the span of row offsets (last_offset - first_offset + 1); may differ from
+   *     physical row count if rows have been deleted
+   * @param min the minimum value in the zone, or null if all values are null
+   * @param max the maximum value in the zone, or null if all values are null
+   * @param nullCount the number of null values in the zone
+   */
+  public ZoneStats(
+      int fragmentId,
+      long zoneStart,
+      long zoneLength,
+      Comparable<?> min,
+      Comparable<?> max,
+      long nullCount) {
+    this.fragmentId = fragmentId;
+    this.zoneStart = zoneStart;
+    this.zoneLength = zoneLength;
+    this.min = min;
+    this.max = max;
+    this.nullCount = nullCount;
+  }
+
+  /** Returns the fragment ID this zone belongs to. */
+  public int getFragmentId() {
+    return fragmentId;
+  }
+
+  /** Returns the starting row offset within the fragment. */
+  public long getZoneStart() {
+    return zoneStart;
+  }
+
+  /**
+   * Returns the span of row offsets covered by this zone.
+   *
+   * <p>This is (last_row_offset - first_row_offset + 1), not the count of physical rows. Deletions
+   * may create gaps within the span.
+   */
+  public long getZoneLength() {
+    return zoneLength;
+  }
+
+  /**
+   * Returns the minimum value in the zone.
+   *
+   * @return the min value, or null if all values in the zone are null
+   */
+  public Comparable<?> getMin() {
+    return min;
+  }
+
+  /**
+   * Returns the maximum value in the zone.
+   *
+   * @return the max value, or null if all values in the zone are null
+   */
+  public Comparable<?> getMax() {
+    return max;
+  }
+
+  /** Returns the number of null values in the zone. */
+  public long getNullCount() {
+    return nullCount;
+  }
+
+  @Override
+  public String toString() {
+    return String.format(
+        "ZoneStats{fragmentId=%d, zoneStart=%d, zoneLength=%d," + " min=%s, max=%s, nullCount=%d}",
+        fragmentId, zoneStart, zoneLength, min, max, nullCount);
+  }
+}
diff --git a/java/src/main/java/org/lance/io/StorageOptionsProvider.java b/java/src/main/java/org/lance/io/StorageOptionsProvider.java
deleted file mode 100644
index 3876ca1fb06..00000000000
--- a/java/src/main/java/org/lance/io/StorageOptionsProvider.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.lance.io;
-
-import java.util.Map;
-
-/**
- * Interface for providing cloud storage options to Lance datasets.
- *
- * <p>Storage options providers enable automatic refresh for long-running operations on cloud
- * storage (S3, Azure, GCS). This is currently only used for refreshing AWS temporary access
- * credentials. Implement this interface to integrate with custom credential management systems such
- * as AWS STS, GCP STS, or proprietary credential services.
- *
- * <p>The provider is called automatically before storage options expire, ensuring uninterrupted
- * access during long-running queries, training jobs, or data processing.
- *
- * <h2>Example Implementation</h2>
- *
- * <pre>{@code
- * public class MyStorageOptionsProvider implements StorageOptionsProvider {
- *   public Map<String, String> fetchStorageOptions() {
- *     // Fetch from your credential service
- *     Map<String, String> credentials = new HashMap<>();
- *     credentials.put("aws_access_key_id", "ASIA...");
- *     credentials.put("aws_secret_access_key", "secret");
- *     credentials.put("aws_session_token", "token");
- *
- *     long expiresAtMillis = System.currentTimeMillis() + 3600000L;
- *     credentials.put("expires_at_millis", String.valueOf(expiresAtMillis));
- *
- *     return credentials;
- *   }
- * }
- *
- * // Use with dataset
- * StorageOptionsProvider vendor = new MyStorageOptionsProvider();
- * Dataset dataset = Dataset.open(
- *     "s3://bucket/table.lance",
- *     new ReadOptions.Builder()
- *         .setStorageOptionsProvider(vendor)
- *         .build()
- * );
- * }</pre>
- *
- * <h2>Error Handling</h2>
- *
- * <p>If fetchStorageOptions() throws an exception, operations requiring credentials will fail.
- * Implementations should handle recoverable errors internally (e.g., retry token refresh) and only
- * throw exceptions for unrecoverable errors.
- */
-public interface StorageOptionsProvider {
-
-  /**
-   * Fetch fresh storage credentials.
-   *
-   * <p>This method is called automatically before each request and before existing credentials
-   * expire. It must return credentials in the format described below.
-   *
-   * @return Map of string key-value pairs containing cloud storage credentials and expiration time.
-   *     Required key:
-   *     <ul>
-   *       <li>"expires_at_millis" (String): Unix timestamp in milliseconds (as string) when
-   *           credentials expire. Lance will automatically call fetchStorageOptions() again before
-   *           this time.
-   *     </ul>
-   *     Plus provider-specific credential keys:
-   *     <ul>
-   *       <li>AWS S3: "aws_access_key_id", "aws_secret_access_key", "aws_session_token" (optional)
-   *       <li>Azure Blob Storage: "account_name", "account_key" or "sas_token"
-   *       <li>Google Cloud Storage: "service_account_key" or "token"
-   *     </ul>
-   *
-   * @throws RuntimeException if unable to fetch credentials
-   */
-  Map<String, String> fetchStorageOptions();
-
-  /**
-   * Return a human-readable unique identifier for this provider instance.
-   *
-   * <p>This is used for equality comparison and hashing in the object store registry. Two providers
-   * with the same ID will be treated as equal and share the same cached ObjectStore instance.
-   *
-   * <p>The default implementation uses the class name and toString() representation. Override this
-   * method to provide semantic equality based on configuration.
-   *
-   * @return A human-readable unique identifier string. For example: "MyProvider { endpoint:
-   *     'https://api.example.com' }"
-   */
-  default String providerId() {
-    return this.getClass().getSimpleName() + " { repr: \"" + this.toString() + "\" }";
-  }
-}
diff --git a/java/src/main/java/org/lance/ipc/AsyncScanner.java b/java/src/main/java/org/lance/ipc/AsyncScanner.java
new file mode 100644
index 00000000000..59ecdebd750
--- /dev/null
+++ b/java/src/main/java/org/lance/ipc/AsyncScanner.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance.ipc;
+
+import org.lance.Dataset;
+import org.lance.LockManager;
+
+import org.apache.arrow.c.ArrowArrayStream;
+import org.apache.arrow.c.ArrowSchema;
+import org.apache.arrow.c.Data;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.Preconditions;
+import org.apache.arrow.vector.ipc.ArrowReader;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Async scanner that provides non-blocking scan operations via CompletableFuture.
+ *
+ * <p>This scanner spawns async I/O tasks in Rust and completes Java futures when data is ready,
+ * preventing thread starvation in Java query engines like Presto/Trino.
+ */
+public class AsyncScanner implements AutoCloseable {
+  private static final AtomicLong TASK_ID_GENERATOR = new AtomicLong(1);
+  private final ConcurrentHashMap<Long, CompletableFuture<Long>> pendingTasks =
+      new ConcurrentHashMap<>();
+
+  private BufferAllocator allocator;
+  private final LockManager lockManager = new LockManager();
+  private long nativeAsyncScannerHandle;
+
+  private AsyncScanner() {}
+
+  /**
+   * Create an AsyncScanner.
+   *
+   * @param dataset the dataset to scan
+   * @param options scan options
+   * @param allocator allocator
+   * @return an AsyncScanner
+   */
+  public static AsyncScanner create(
+      Dataset dataset, ScanOptions options, BufferAllocator allocator) {
+    Preconditions.checkNotNull(dataset);
+    Preconditions.checkNotNull(options);
+    Preconditions.checkNotNull(allocator);
+    AsyncScanner scanner =
+        createAsyncScanner(
+            dataset,
+            options.getFragmentIds(),
+            options.getColumns(),
+            options.getSubstraitFilter(),
+            options.getFilter(),
+            options.getBatchSize(),
+            options.getLimit(),
+            options.getOffset(),
+            options.getNearest(),
+            options.getFullTextQuery(),
+            options.isPrefilter(),
+            options.isWithRowId(),
+            options.isWithRowAddress(),
+            options.getBatchReadahead(),
+            options.getColumnOrderings(),
+            options.isUseScalarIndex(),
+            options.getSubstraitAggregate());
+    scanner.allocator = allocator;
+    return scanner;
+  }
+
+  static native AsyncScanner createAsyncScanner(
+      Dataset dataset,
+      Optional<List<Integer>> fragmentIds,
+      Optional<List<String>> columns,
+      Optional<ByteBuffer> substraitFilter,
+      Optional<String> filter,
+      Optional<Long> batchSize,
+      Optional<Long> limit,
+      Optional<Long> offset,
+      Optional<Query> query,
+      Optional<FullTextQuery> fullTextQuery,
+      boolean prefilter,
+      boolean withRowId,
+      boolean withRowAddress,
+      int batchReadahead,
+      Optional<List<ColumnOrdering>> columnOrderings,
+      boolean useScalarIndex,
+      Optional<ByteBuffer> substraitAggregate);
+
+  /**
+   * Asynchronously scan batches and return a CompletableFuture.
+   *
+   * @return a CompletableFuture that will be completed with an ArrowReader when data is ready
+   */
+  public CompletableFuture<ArrowReader> scanBatchesAsync() {
+    try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
+      if (nativeAsyncScannerHandle == 0) {
+        CompletableFuture<ArrowReader> future = new CompletableFuture<>();
+        future.completeExceptionally(new IllegalStateException("Scanner is closed"));
+        return future;
+      }
+
+      long taskId = TASK_ID_GENERATOR.getAndIncrement();
+      CompletableFuture<Long> streamPtrFuture = new CompletableFuture<>();
+      pendingTasks.put(taskId, streamPtrFuture);
+
+      // Start async scan in Rust
+      nativeStartScan(taskId);
+
+      // Transform stream pointer to ArrowReader
+      return streamPtrFuture.handle(
+          (streamPtr, error) -> {
+            pendingTasks.remove(taskId);
+
+            if (error != null) {
+              throw new RuntimeException("Scan failed", error);
+            }
+
+            if (streamPtr < 0) {
+              throw new RuntimeException("Native scan error");
+            }
+
+            try {
+              ArrowArrayStream stream = ArrowArrayStream.wrap(streamPtr);
+              return Data.importArrayStream(allocator, stream);
+            } catch (Exception e) {
+              throw new RuntimeException(e);
+            }
+          });
+    }
+  }
+
+  /** Called by Rust dispatcher thread via JNI to complete a task successfully. */
+  private void completeTask(long taskId, long resultPtr) {
+    CompletableFuture<Long> future = pendingTasks.get(taskId);
+    if (future != null) {
+      future.complete(resultPtr);
+    }
+  }
+
+  /** Called by Rust dispatcher thread via JNI to fail a task with an error. */
+  private void failTask(long taskId, String errorMessage) {
+    CompletableFuture<Long> future = pendingTasks.get(taskId);
+    if (future != null) {
+      future.completeExceptionally(new RuntimeException(errorMessage));
+    }
+  }
+
+  private native void nativeStartScan(long taskId);
+
+  /**
+   * Get schema (synchronous operation).
+   *
+   * @return the schema
+   */
+  public Schema schema() {
+    try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
+      Preconditions.checkArgument(nativeAsyncScannerHandle != 0, "Scanner is closed");
+      try (ArrowSchema ffiSchema = ArrowSchema.allocateNew(allocator)) {
+        importFfiSchema(ffiSchema.memoryAddress());
+        return Data.importSchema(allocator, ffiSchema, null);
+      }
+    }
+  }
+
+  private native void importFfiSchema(long arrowSchemaMemoryAddress);
+
+  /**
+   * Closes this scanner and releases any system resources associated with it. If the scanner is
+   * already closed, then invoking this method has no effect.
+   */
+  @Override
+  public void close() throws Exception {
+    try (LockManager.WriteLock writeLock = lockManager.acquireWriteLock()) {
+      if (nativeAsyncScannerHandle != 0) {
+        // Cancel all pending tasks
+        for (Long taskId : pendingTasks.keySet()) {
+          nativeCancelTask(taskId);
+        }
+        pendingTasks.clear();
+
+        releaseNativeScanner();
+        nativeAsyncScannerHandle = 0;
+      }
+    }
+  }
+
+  private native void nativeCancelTask(long taskId);
+
+  /** Native method to release the async scanner resources. */
+  private native void releaseNativeScanner();
+}
diff --git a/java/src/main/java/org/lance/namespace/DirectoryNamespace.java b/java/src/main/java/org/lance/namespace/DirectoryNamespace.java
index b811a57920b..7bd5f9c7f9f 100644
--- a/java/src/main/java/org/lance/namespace/DirectoryNamespace.java
+++ b/java/src/main/java/org/lance/namespace/DirectoryNamespace.java
@@ -14,7 +14,87 @@
 package org.lance.namespace;
 
 import org.lance.JniLoader;
-import org.lance.namespace.model.*;
+import org.lance.namespace.model.AlterTableAddColumnsRequest;
+import org.lance.namespace.model.AlterTableAddColumnsResponse;
+import org.lance.namespace.model.AlterTableAlterColumnsRequest;
+import org.lance.namespace.model.AlterTableAlterColumnsResponse;
+import org.lance.namespace.model.AlterTableDropColumnsRequest;
+import org.lance.namespace.model.AlterTableDropColumnsResponse;
+import org.lance.namespace.model.AlterTransactionRequest;
+import org.lance.namespace.model.AlterTransactionResponse;
+import org.lance.namespace.model.AnalyzeTableQueryPlanRequest;
+import org.lance.namespace.model.BatchDeleteTableVersionsRequest;
+import org.lance.namespace.model.BatchDeleteTableVersionsResponse;
+import org.lance.namespace.model.CountTableRowsRequest;
+import org.lance.namespace.model.CreateNamespaceRequest;
+import org.lance.namespace.model.CreateNamespaceResponse;
+import org.lance.namespace.model.CreateTableIndexRequest;
+import org.lance.namespace.model.CreateTableIndexResponse;
+import org.lance.namespace.model.CreateTableRequest;
+import org.lance.namespace.model.CreateTableResponse;
+import org.lance.namespace.model.CreateTableScalarIndexResponse;
+import org.lance.namespace.model.CreateTableTagRequest;
+import org.lance.namespace.model.CreateTableTagResponse;
+import org.lance.namespace.model.CreateTableVersionRequest;
+import org.lance.namespace.model.CreateTableVersionResponse;
+import org.lance.namespace.model.DeclareTableRequest;
+import org.lance.namespace.model.DeclareTableResponse;
+import org.lance.namespace.model.DeleteFromTableRequest;
+import org.lance.namespace.model.DeleteFromTableResponse;
+import org.lance.namespace.model.DeleteTableTagRequest;
+import org.lance.namespace.model.DeleteTableTagResponse;
+import org.lance.namespace.model.DeregisterTableRequest;
+import org.lance.namespace.model.DeregisterTableResponse;
+import org.lance.namespace.model.DescribeNamespaceRequest;
+import org.lance.namespace.model.DescribeNamespaceResponse;
+import org.lance.namespace.model.DescribeTableIndexStatsRequest;
+import org.lance.namespace.model.DescribeTableIndexStatsResponse;
+import org.lance.namespace.model.DescribeTableRequest;
+import org.lance.namespace.model.DescribeTableResponse;
+import org.lance.namespace.model.DescribeTableVersionRequest;
+import org.lance.namespace.model.DescribeTableVersionResponse;
+import org.lance.namespace.model.DescribeTransactionRequest;
+import org.lance.namespace.model.DescribeTransactionResponse;
+import org.lance.namespace.model.DropNamespaceRequest;
+import org.lance.namespace.model.DropNamespaceResponse;
+import org.lance.namespace.model.DropTableIndexRequest;
+import org.lance.namespace.model.DropTableIndexResponse;
+import org.lance.namespace.model.DropTableRequest;
+import org.lance.namespace.model.DropTableResponse;
+import org.lance.namespace.model.ExplainTableQueryPlanRequest;
+import org.lance.namespace.model.GetTableStatsRequest;
+import org.lance.namespace.model.GetTableStatsResponse;
+import org.lance.namespace.model.GetTableTagVersionRequest;
+import org.lance.namespace.model.GetTableTagVersionResponse;
+import org.lance.namespace.model.InsertIntoTableRequest;
+import org.lance.namespace.model.InsertIntoTableResponse;
+import org.lance.namespace.model.ListNamespacesRequest;
+import org.lance.namespace.model.ListNamespacesResponse;
+import org.lance.namespace.model.ListTableIndicesRequest;
+import org.lance.namespace.model.ListTableIndicesResponse;
+import org.lance.namespace.model.ListTableTagsRequest;
+import org.lance.namespace.model.ListTableTagsResponse;
+import org.lance.namespace.model.ListTableVersionsRequest;
+import org.lance.namespace.model.ListTableVersionsResponse;
+import org.lance.namespace.model.ListTablesRequest;
+import org.lance.namespace.model.ListTablesResponse;
+import org.lance.namespace.model.MergeInsertIntoTableRequest;
+import org.lance.namespace.model.MergeInsertIntoTableResponse;
+import org.lance.namespace.model.NamespaceExistsRequest;
+import org.lance.namespace.model.QueryTableRequest;
+import org.lance.namespace.model.RegisterTableRequest;
+import org.lance.namespace.model.RegisterTableResponse;
+import org.lance.namespace.model.RenameTableRequest;
+import org.lance.namespace.model.RenameTableResponse;
+import org.lance.namespace.model.RestoreTableRequest;
+import org.lance.namespace.model.RestoreTableResponse;
+import org.lance.namespace.model.TableExistsRequest;
+import org.lance.namespace.model.UpdateTableRequest;
+import org.lance.namespace.model.UpdateTableResponse;
+import org.lance.namespace.model.UpdateTableSchemaMetadataRequest;
+import org.lance.namespace.model.UpdateTableSchemaMetadataResponse;
+import org.lance.namespace.model.UpdateTableTagRequest;
+import org.lance.namespace.model.UpdateTableTagResponse;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
@@ -90,6 +170,16 @@
  *         <li>credential_vendor.azure_duration_millis (optional): Duration in ms (default: 3600000,
  *             up to 7 days)
  *       </ul>
+ *   <li>Testing properties:
+ *       <ul>
+ *         <li>ops_metrics_enabled (optional): "true" to enable operation metrics tracking. Use
+ *             {@link #retrieveOpsMetrics()} to get call counts.
+ *         <li>vend_input_storage_options (optional): "true" to return input storage options in
+ *             describeTable() when no credential vendor is configured. Useful for testing.
+ *         <li>vend_input_storage_options_refresh_interval_millis (optional): When set with
+ *             vend_input_storage_options, adds expires_at_millis to storage options. Value is
+ *             current_time_millis + this interval.
+ *       </ul>
  * </ul>
  *
  * <p>Example usage (local filesystem):
@@ -314,6 +404,14 @@ public DeclareTableResponse declareTable(DeclareTableRequest request) {
     return fromJson(responseJson, DeclareTableResponse.class);
   }
 
+  @Override
+  public RenameTableResponse renameTable(RenameTableRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = renameTableNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, RenameTableResponse.class);
+  }
+
   @Override
   public InsertIntoTableResponse insertIntoTable(
       InsertIntoTableRequest request, byte[] requestData) {
@@ -435,6 +533,135 @@ public BatchDeleteTableVersionsResponse batchDeleteTableVersions(
     return fromJson(responseJson, BatchDeleteTableVersionsResponse.class);
   }
 
+  @Override
+  public CreateTableScalarIndexResponse createTableScalarIndex(CreateTableIndexRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = createTableScalarIndexNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, CreateTableScalarIndexResponse.class);
+  }
+
+  @Override
+  public DropTableIndexResponse dropTableIndex(DropTableIndexRequest request, String indexName) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = dropTableIndexNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, DropTableIndexResponse.class);
+  }
+
+  @Override
+  public ListTablesResponse listAllTables(ListTablesRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = listAllTablesNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, ListTablesResponse.class);
+  }
+
+  @Override
+  public RestoreTableResponse restoreTable(RestoreTableRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = restoreTableNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, RestoreTableResponse.class);
+  }
+
+  @Override
+  public UpdateTableSchemaMetadataResponse updateTableSchemaMetadata(
+      UpdateTableSchemaMetadataRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson =
+        updateTableSchemaMetadataNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, UpdateTableSchemaMetadataResponse.class);
+  }
+
+  @Override
+  public GetTableStatsResponse getTableStats(GetTableStatsRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = getTableStatsNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, GetTableStatsResponse.class);
+  }
+
+  @Override
+  public String explainTableQueryPlan(ExplainTableQueryPlanRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    return explainTableQueryPlanNative(nativeDirectoryNamespaceHandle, requestJson);
+  }
+
+  @Override
+  public String analyzeTableQueryPlan(AnalyzeTableQueryPlanRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    return analyzeTableQueryPlanNative(nativeDirectoryNamespaceHandle, requestJson);
+  }
+
+  @Override
+  public AlterTableAddColumnsResponse alterTableAddColumns(AlterTableAddColumnsRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = alterTableAddColumnsNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, AlterTableAddColumnsResponse.class);
+  }
+
+  @Override
+  public AlterTableAlterColumnsResponse alterTableAlterColumns(
+      AlterTableAlterColumnsRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = alterTableAlterColumnsNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, AlterTableAlterColumnsResponse.class);
+  }
+
+  @Override
+  public AlterTableDropColumnsResponse alterTableDropColumns(AlterTableDropColumnsRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = alterTableDropColumnsNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, AlterTableDropColumnsResponse.class);
+  }
+
+  @Override
+  public ListTableTagsResponse listTableTags(ListTableTagsRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = listTableTagsNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, ListTableTagsResponse.class);
+  }
+
+  @Override
+  public GetTableTagVersionResponse getTableTagVersion(GetTableTagVersionRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = getTableTagVersionNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, GetTableTagVersionResponse.class);
+  }
+
+  @Override
+  public CreateTableTagResponse createTableTag(CreateTableTagRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = createTableTagNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, CreateTableTagResponse.class);
+  }
+
+  @Override
+  public DeleteTableTagResponse deleteTableTag(DeleteTableTagRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = deleteTableTagNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, DeleteTableTagResponse.class);
+  }
+
+  @Override
+  public UpdateTableTagResponse updateTableTag(UpdateTableTagRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = updateTableTagNative(nativeDirectoryNamespaceHandle, requestJson);
+    return fromJson(responseJson, UpdateTableTagResponse.class);
+  }
+
   @Override
   public void close() {
     if (nativeDirectoryNamespaceHandle != 0) {
@@ -452,6 +679,33 @@ public long getNativeHandle() {
     return nativeDirectoryNamespaceHandle;
   }
 
+  // Operation metrics methods
+
+  /**
+   * Retrieve operation metrics as a map.
+   *
+   * <p>Returns a map where keys are operation names (e.g., "list_tables", "describe_table") and
+   * values are the number of times each operation was called.
+   *
+   * <p>Returns an empty map if {@code ops_metrics_enabled} was false when creating the namespace.
+   *
+   * @return operation name to call count mapping
+   */
+  public Map<String, Long> retrieveOpsMetrics() {
+    ensureInitialized();
+    return retrieveOpsMetricsNative(nativeDirectoryNamespaceHandle);
+  }
+
+  /**
+   * Reset all operation metrics counters to zero.
+   *
+   * <p>Does nothing if {@code ops_metrics_enabled} was false when creating the namespace.
+   */
+  public void resetOpsMetrics() {
+    ensureInitialized();
+    resetOpsMetricsNative(nativeDirectoryNamespaceHandle);
+  }
+
   private void ensureInitialized() {
     if (nativeDirectoryNamespaceHandle == 0) {
       throw new IllegalStateException(
@@ -513,6 +767,8 @@ private native long createNativeWithProvider(
 
   private native String declareTableNative(long handle, String requestJson);
 
+  private native String renameTableNative(long handle, String requestJson);
+
   private native String insertIntoTableNative(long handle, String requestJson, byte[] requestData);
 
   private native String mergeInsertIntoTableNative(
@@ -542,6 +798,42 @@ private native String mergeInsertIntoTableNative(
 
   private native String batchDeleteTableVersionsNative(long handle, String requestJson);
 
+  private native String createTableScalarIndexNative(long handle, String requestJson);
+
+  private native String dropTableIndexNative(long handle, String requestJson);
+
+  private native String listAllTablesNative(long handle, String requestJson);
+
+  private native String restoreTableNative(long handle, String requestJson);
+
+  private native String updateTableSchemaMetadataNative(long handle, String requestJson);
+
+  private native String getTableStatsNative(long handle, String requestJson);
+
+  private native String explainTableQueryPlanNative(long handle, String requestJson);
+
+  private native String analyzeTableQueryPlanNative(long handle, String requestJson);
+
+  private native String alterTableAddColumnsNative(long handle, String requestJson);
+
+  private native String alterTableAlterColumnsNative(long handle, String requestJson);
+
+  private native String alterTableDropColumnsNative(long handle, String requestJson);
+
+  private native String listTableTagsNative(long handle, String requestJson);
+
+  private native String getTableTagVersionNative(long handle, String requestJson);
+
+  private native String createTableTagNative(long handle, String requestJson);
+
+  private native String deleteTableTagNative(long handle, String requestJson);
+
+  private native String updateTableTagNative(long handle, String requestJson);
+
+  private native Map<String, Long> retrieveOpsMetricsNative(long handle);
+
+  private native void resetOpsMetricsNative(long handle);
+
   // ==========================================================================
   // Provider loading helpers
   // ==========================================================================
diff --git a/java/src/main/java/org/lance/namespace/LanceNamespaceStorageOptionsProvider.java b/java/src/main/java/org/lance/namespace/LanceNamespaceStorageOptionsProvider.java
deleted file mode 100644
index fb65e235c36..00000000000
--- a/java/src/main/java/org/lance/namespace/LanceNamespaceStorageOptionsProvider.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.lance.namespace;
-
-import org.lance.io.StorageOptionsProvider;
-import org.lance.namespace.model.DescribeTableRequest;
-import org.lance.namespace.model.DescribeTableResponse;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * Storage options provider that fetches storage options from a LanceNamespace.
- *
- * <p>This provider automatically fetches fresh storage options by calling the namespace's
- * describeTable() method, which returns both the table location and time-limited storage options.
- * This is currently only used for refreshing AWS temporary access credentials.
- *
- * <p>This is the recommended approach for LanceDB Cloud and other namespace-based deployments, as
- * it handles storage options refresh automatically.
- *
- * <h2>Example Usage</h2>
- *
- * <pre>{@code
- * // Connect to a namespace (e.g., LanceDB Cloud)
- * LanceNamespace namespace = LanceNamespaces.connect("rest", Map.of(
- *     "url", "https://api.lancedb.com",
- *     "api_key", "your-api-key"
- * ));
- *
- * // Create storage options provider
- * LanceNamespaceStorageOptionsProvider provider = new LanceNamespaceStorageOptionsProvider(
- *     namespace,
- *     Arrays.asList("workspace", "table_name")
- * );
- *
- * // Use with dataset - storage options auto-refresh!
- * Dataset dataset = Dataset.open(
- *     "s3://bucket/table.lance",
- *     new ReadOptions.Builder()
- *         .setStorageOptionsProvider(provider)
- *         .build()
- * );
- * }</pre>
- */
-public class LanceNamespaceStorageOptionsProvider implements StorageOptionsProvider {
-
-  private final LanceNamespace namespace;
-  private final List<String> tableId;
-
-  /**
-   * Create a storage options provider that fetches storage options from a LanceNamespace.
-   *
-   * @param namespace The namespace instance to fetch storage options from
-   * @param tableId The table identifier (e.g., ["workspace", "table_name"])
-   */
-  public LanceNamespaceStorageOptionsProvider(LanceNamespace namespace, List<String> tableId) {
-    this.namespace = namespace;
-    this.tableId = tableId;
-  }
-
-  /**
-   * Fetch credentials from the namespace.
-   *
-   * <p>This calls namespace.describeTable() to get the latest credentials and optionally their
-   * expiration time.
-   *
-   * @return Flat map of string key-value pairs containing credentials. May optionally include
-   *     expires_at_millis. If expires_at_millis is not provided, credentials are treated as
-   *     non-expiring and will not be automatically refreshed.
-   * @throws RuntimeException if the namespace doesn't return storage credentials
-   */
-  @Override
-  public Map<String, String> fetchStorageOptions() {
-    // Create describe table request with table ID
-    DescribeTableRequest request = new DescribeTableRequest();
-    request.setId(tableId);
-
-    // Call namespace to describe the table and get credentials
-    DescribeTableResponse response = namespace.describeTable(request);
-
-    // Extract storage options - should already be a flat Map<String, String>
-    Map<String, String> storageOptions = response.getStorageOptions();
-    if (storageOptions == null || storageOptions.isEmpty()) {
-      throw new RuntimeException(
-          "Namespace did not return storage_options. "
-              + "Ensure the namespace supports credential vending.");
-    }
-
-    // Return storage_options directly - it's already a flat Map<String, String>
-    // Note: expires_at_millis is optional. If not provided, credentials are treated
-    // as non-expiring and will not be automatically refreshed.
-    return storageOptions;
-  }
-
-  /**
-   * Return a human-readable unique identifier for this provider instance.
-   *
-   * <p>This creates a semantic ID based on the namespace's ID and the table ID, enabling proper
-   * equality comparison and caching.
-   *
-   * @return A human-readable unique identifier string combining namespace and table info
-   */
-  @Override
-  public String providerId() {
-    // Call namespaceId() on the namespace (requires lance-namespace >= 0.0.20)
-    String namespaceId = namespace.namespaceId();
-    return String.format(
-        "LanceNamespaceStorageOptionsProvider { namespace: %s, table_id: %s }",
-        namespaceId, tableId);
-  }
-}
diff --git a/java/src/main/java/org/lance/namespace/RestAdapter.java b/java/src/main/java/org/lance/namespace/RestAdapter.java
index 534a7eabb9e..5d22c2962a3 100644
--- a/java/src/main/java/org/lance/namespace/RestAdapter.java
+++ b/java/src/main/java/org/lance/namespace/RestAdapter.java
@@ -58,16 +58,17 @@ public class RestAdapter implements Closeable, AutoCloseable {
   /**
    * Creates a new REST adapter with the given backend namespace.
    *
-   * @param namespaceImpl The namespace implementation type (e.g., "dir" for DirectoryNamespace)
-   * @param backendConfig Configuration properties for the backend namespace
+   * @param namespaceClientImpl The namespace client implementation type (e.g., "dir" for
+   *     DirectoryNamespace)
+   * @param backendConfig Configuration properties for the backend namespace client
    * @param host Host to bind the server to, or null for default (127.0.0.1)
    * @param port Port to bind the server to. Use 0 to let the OS assign an available port, or null
    *     for default (2333).
    */
   public RestAdapter(
-      String namespaceImpl, Map<String, String> backendConfig, String host, Integer port) {
-    if (namespaceImpl == null || namespaceImpl.isEmpty()) {
-      throw new IllegalArgumentException("namespace implementation cannot be null or empty");
+      String namespaceClientImpl, Map<String, String> backendConfig, String host, Integer port) {
+    if (namespaceClientImpl == null || namespaceClientImpl.isEmpty()) {
+      throw new IllegalArgumentException("namespace client implementation cannot be null or empty");
     }
     if (backendConfig == null) {
       throw new IllegalArgumentException("backend config cannot be null");
@@ -76,19 +77,29 @@ public RestAdapter(
       throw new IllegalArgumentException("port must be between 0 and 65535");
     }
 
-    this.nativeRestAdapterHandle = createNative(namespaceImpl, backendConfig, host, port);
+    this.nativeRestAdapterHandle = createNative(namespaceClientImpl, backendConfig, host, port);
   }
 
   /**
    * Creates a new REST adapter with default host and port.
    *
-   * @param namespaceImpl The namespace implementation type
-   * @param backendConfig Configuration properties for the backend namespace
+   * @param namespaceClientImpl The namespace client implementation type
+   * @param backendConfig Configuration properties for the backend namespace client
    */
-  public RestAdapter(String namespaceImpl, Map<String, String> backendConfig) {
-    this(namespaceImpl, backendConfig, null, null);
+  public RestAdapter(String namespaceClientImpl, Map<String, String> backendConfig) {
+    this(namespaceClientImpl, backendConfig, null, null);
   }
 
+  /**
+   * Creates a new REST adapter with the given backend namespace.
+   *
+   * @param namespaceImpl The namespace implementation type (e.g., "dir" for DirectoryNamespace)
+   * @param backendConfig Configuration properties for the backend namespace
+   * @param host Host to bind the server to, or null for default (127.0.0.1)
+   * @param port Port to bind the server to. Use 0 to let the OS assign an available port, or null
+   *     for default (2333).
+   */
+
   /**
    * Start the REST server in the background.
    *
@@ -144,7 +155,7 @@ public void close() {
 
   // Native methods
   private native long createNative(
-      String namespaceImpl, Map<String, String> backendConfig, String host, Integer port);
+      String namespaceClientImpl, Map<String, String> backendConfig, String host, Integer port);
 
   private native void start(long handle);
 
diff --git a/java/src/main/java/org/lance/namespace/RestNamespace.java b/java/src/main/java/org/lance/namespace/RestNamespace.java
index 8763512b321..fbf58bd55c6 100644
--- a/java/src/main/java/org/lance/namespace/RestNamespace.java
+++ b/java/src/main/java/org/lance/namespace/RestNamespace.java
@@ -14,7 +14,87 @@
 package org.lance.namespace;
 
 import org.lance.JniLoader;
-import org.lance.namespace.model.*;
+import org.lance.namespace.model.AlterTableAddColumnsRequest;
+import org.lance.namespace.model.AlterTableAddColumnsResponse;
+import org.lance.namespace.model.AlterTableAlterColumnsRequest;
+import org.lance.namespace.model.AlterTableAlterColumnsResponse;
+import org.lance.namespace.model.AlterTableDropColumnsRequest;
+import org.lance.namespace.model.AlterTableDropColumnsResponse;
+import org.lance.namespace.model.AlterTransactionRequest;
+import org.lance.namespace.model.AlterTransactionResponse;
+import org.lance.namespace.model.AnalyzeTableQueryPlanRequest;
+import org.lance.namespace.model.BatchDeleteTableVersionsRequest;
+import org.lance.namespace.model.BatchDeleteTableVersionsResponse;
+import org.lance.namespace.model.CountTableRowsRequest;
+import org.lance.namespace.model.CreateNamespaceRequest;
+import org.lance.namespace.model.CreateNamespaceResponse;
+import org.lance.namespace.model.CreateTableIndexRequest;
+import org.lance.namespace.model.CreateTableIndexResponse;
+import org.lance.namespace.model.CreateTableRequest;
+import org.lance.namespace.model.CreateTableResponse;
+import org.lance.namespace.model.CreateTableScalarIndexResponse;
+import org.lance.namespace.model.CreateTableTagRequest;
+import org.lance.namespace.model.CreateTableTagResponse;
+import org.lance.namespace.model.CreateTableVersionRequest;
+import org.lance.namespace.model.CreateTableVersionResponse;
+import org.lance.namespace.model.DeclareTableRequest;
+import org.lance.namespace.model.DeclareTableResponse;
+import org.lance.namespace.model.DeleteFromTableRequest;
+import org.lance.namespace.model.DeleteFromTableResponse;
+import org.lance.namespace.model.DeleteTableTagRequest;
+import org.lance.namespace.model.DeleteTableTagResponse;
+import org.lance.namespace.model.DeregisterTableRequest;
+import org.lance.namespace.model.DeregisterTableResponse;
+import org.lance.namespace.model.DescribeNamespaceRequest;
+import org.lance.namespace.model.DescribeNamespaceResponse;
+import org.lance.namespace.model.DescribeTableIndexStatsRequest;
+import org.lance.namespace.model.DescribeTableIndexStatsResponse;
+import org.lance.namespace.model.DescribeTableRequest;
+import org.lance.namespace.model.DescribeTableResponse;
+import org.lance.namespace.model.DescribeTableVersionRequest;
+import org.lance.namespace.model.DescribeTableVersionResponse;
+import org.lance.namespace.model.DescribeTransactionRequest;
+import org.lance.namespace.model.DescribeTransactionResponse;
+import org.lance.namespace.model.DropNamespaceRequest;
+import org.lance.namespace.model.DropNamespaceResponse;
+import org.lance.namespace.model.DropTableIndexRequest;
+import org.lance.namespace.model.DropTableIndexResponse;
+import org.lance.namespace.model.DropTableRequest;
+import org.lance.namespace.model.DropTableResponse;
+import org.lance.namespace.model.ExplainTableQueryPlanRequest;
+import org.lance.namespace.model.GetTableStatsRequest;
+import org.lance.namespace.model.GetTableStatsResponse;
+import org.lance.namespace.model.GetTableTagVersionRequest;
+import org.lance.namespace.model.GetTableTagVersionResponse;
+import org.lance.namespace.model.InsertIntoTableRequest;
+import org.lance.namespace.model.InsertIntoTableResponse;
+import org.lance.namespace.model.ListNamespacesRequest;
+import org.lance.namespace.model.ListNamespacesResponse;
+import org.lance.namespace.model.ListTableIndicesRequest;
+import org.lance.namespace.model.ListTableIndicesResponse;
+import org.lance.namespace.model.ListTableTagsRequest;
+import org.lance.namespace.model.ListTableTagsResponse;
+import org.lance.namespace.model.ListTableVersionsRequest;
+import org.lance.namespace.model.ListTableVersionsResponse;
+import org.lance.namespace.model.ListTablesRequest;
+import org.lance.namespace.model.ListTablesResponse;
+import org.lance.namespace.model.MergeInsertIntoTableRequest;
+import org.lance.namespace.model.MergeInsertIntoTableResponse;
+import org.lance.namespace.model.NamespaceExistsRequest;
+import org.lance.namespace.model.QueryTableRequest;
+import org.lance.namespace.model.RegisterTableRequest;
+import org.lance.namespace.model.RegisterTableResponse;
+import org.lance.namespace.model.RenameTableRequest;
+import org.lance.namespace.model.RenameTableResponse;
+import org.lance.namespace.model.RestoreTableRequest;
+import org.lance.namespace.model.RestoreTableResponse;
+import org.lance.namespace.model.TableExistsRequest;
+import org.lance.namespace.model.UpdateTableRequest;
+import org.lance.namespace.model.UpdateTableResponse;
+import org.lance.namespace.model.UpdateTableSchemaMetadataRequest;
+import org.lance.namespace.model.UpdateTableSchemaMetadataResponse;
+import org.lance.namespace.model.UpdateTableTagRequest;
+import org.lance.namespace.model.UpdateTableTagResponse;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -360,6 +440,134 @@ public BatchDeleteTableVersionsResponse batchDeleteTableVersions(
     return fromJson(responseJson, BatchDeleteTableVersionsResponse.class);
   }
 
+  @Override
+  public CreateTableScalarIndexResponse createTableScalarIndex(CreateTableIndexRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = createTableScalarIndexNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, CreateTableScalarIndexResponse.class);
+  }
+
+  @Override
+  public DropTableIndexResponse dropTableIndex(DropTableIndexRequest request, String indexName) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = dropTableIndexNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, DropTableIndexResponse.class);
+  }
+
+  @Override
+  public ListTablesResponse listAllTables(ListTablesRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = listAllTablesNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, ListTablesResponse.class);
+  }
+
+  @Override
+  public RestoreTableResponse restoreTable(RestoreTableRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = restoreTableNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, RestoreTableResponse.class);
+  }
+
+  @Override
+  public UpdateTableSchemaMetadataResponse updateTableSchemaMetadata(
+      UpdateTableSchemaMetadataRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = updateTableSchemaMetadataNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, UpdateTableSchemaMetadataResponse.class);
+  }
+
+  @Override
+  public GetTableStatsResponse getTableStats(GetTableStatsRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = getTableStatsNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, GetTableStatsResponse.class);
+  }
+
+  @Override
+  public String explainTableQueryPlan(ExplainTableQueryPlanRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    return explainTableQueryPlanNative(nativeRestNamespaceHandle, requestJson);
+  }
+
+  @Override
+  public String analyzeTableQueryPlan(AnalyzeTableQueryPlanRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    return analyzeTableQueryPlanNative(nativeRestNamespaceHandle, requestJson);
+  }
+
+  @Override
+  public AlterTableAddColumnsResponse alterTableAddColumns(AlterTableAddColumnsRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = alterTableAddColumnsNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, AlterTableAddColumnsResponse.class);
+  }
+
+  @Override
+  public AlterTableAlterColumnsResponse alterTableAlterColumns(
+      AlterTableAlterColumnsRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = alterTableAlterColumnsNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, AlterTableAlterColumnsResponse.class);
+  }
+
+  @Override
+  public AlterTableDropColumnsResponse alterTableDropColumns(AlterTableDropColumnsRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = alterTableDropColumnsNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, AlterTableDropColumnsResponse.class);
+  }
+
+  @Override
+  public ListTableTagsResponse listTableTags(ListTableTagsRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = listTableTagsNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, ListTableTagsResponse.class);
+  }
+
+  @Override
+  public GetTableTagVersionResponse getTableTagVersion(GetTableTagVersionRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = getTableTagVersionNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, GetTableTagVersionResponse.class);
+  }
+
+  @Override
+  public CreateTableTagResponse createTableTag(CreateTableTagRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = createTableTagNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, CreateTableTagResponse.class);
+  }
+
+  @Override
+  public DeleteTableTagResponse deleteTableTag(DeleteTableTagRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = deleteTableTagNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, DeleteTableTagResponse.class);
+  }
+
+  @Override
+  public UpdateTableTagResponse updateTableTag(UpdateTableTagRequest request) {
+    ensureInitialized();
+    String requestJson = toJson(request);
+    String responseJson = updateTableTagNative(nativeRestNamespaceHandle, requestJson);
+    return fromJson(responseJson, UpdateTableTagResponse.class);
+  }
+
   @Override
   public void close() {
     if (nativeRestNamespaceHandle != 0) {
@@ -377,6 +585,33 @@ public long getNativeHandle() {
     return nativeRestNamespaceHandle;
   }
 
+  // Operation metrics methods
+
+  /**
+   * Retrieve operation metrics as a map.
+   *
+   * <p>Returns a map where keys are operation names (e.g., "list_tables", "describe_table") and
+   * values are the number of times each operation was called.
+   *
+   * <p>Returns an empty map if {@code ops_metrics_enabled} was false when creating the namespace.
+   *
+   * @return operation name to call count mapping
+   */
+  public Map<String, Long> retrieveOpsMetrics() {
+    ensureInitialized();
+    return retrieveOpsMetricsNative(nativeRestNamespaceHandle);
+  }
+
+  /**
+   * Reset all operation metrics counters to zero.
+   *
+   * <p>Does nothing if {@code ops_metrics_enabled} was false when creating the namespace.
+   */
+  public void resetOpsMetrics() {
+    ensureInitialized();
+    resetOpsMetricsNative(nativeRestNamespaceHandle);
+  }
+
   private void ensureInitialized() {
     if (nativeRestNamespaceHandle == 0) {
       throw new IllegalStateException("RestNamespace not initialized. Call initialize() first.");
@@ -468,6 +703,42 @@ private native String mergeInsertIntoTableNative(
 
   private native String batchDeleteTableVersionsNative(long handle, String requestJson);
 
+  private native String createTableScalarIndexNative(long handle, String requestJson);
+
+  private native String dropTableIndexNative(long handle, String requestJson);
+
+  private native String listAllTablesNative(long handle, String requestJson);
+
+  private native String restoreTableNative(long handle, String requestJson);
+
+  private native String updateTableSchemaMetadataNative(long handle, String requestJson);
+
+  private native String getTableStatsNative(long handle, String requestJson);
+
+  private native String explainTableQueryPlanNative(long handle, String requestJson);
+
+  private native String analyzeTableQueryPlanNative(long handle, String requestJson);
+
+  private native String alterTableAddColumnsNative(long handle, String requestJson);
+
+  private native String alterTableAlterColumnsNative(long handle, String requestJson);
+
+  private native String alterTableDropColumnsNative(long handle, String requestJson);
+
+  private native String listTableTagsNative(long handle, String requestJson);
+
+  private native String getTableTagVersionNative(long handle, String requestJson);
+
+  private native String createTableTagNative(long handle, String requestJson);
+
+  private native String deleteTableTagNative(long handle, String requestJson);
+
+  private native String updateTableTagNative(long handle, String requestJson);
+
+  private native Map<String, Long> retrieveOpsMetricsNative(long handle);
+
+  private native void resetOpsMetricsNative(long handle);
+
   // ==========================================================================
   // Provider loading helpers
   // ==========================================================================
diff --git a/java/src/test/java/org/lance/AsyncScannerTest.java b/java/src/test/java/org/lance/AsyncScannerTest.java
new file mode 100644
index 00000000000..98f46887b64
--- /dev/null
+++ b/java/src/test/java/org/lance/AsyncScannerTest.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance;
+
+import org.lance.ipc.AsyncScanner;
+import org.lance.ipc.ScanOptions;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.ipc.ArrowReader;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Example tests demonstrating AsyncScanner usage with CompletableFuture-based API.
+ *
+ * <p>AsyncScanner provides non-blocking scan operations that prevent thread starvation in Java
+ * query engines like Presto/Trino.
+ */
+public class AsyncScannerTest {
+  private static Dataset dataset;
+
+  @BeforeAll
+  static void setup() {}
+
+  @AfterAll
+  static void tearDown() {
+    if (dataset != null) {
+      dataset.close();
+    }
+  }
+
+  /**
+   * Example 1: Basic async scan with CompletableFuture.
+   *
+   * <p>This shows the simplest usage - create an async scanner and wait for results.
+   */
+  @Test
+  void testBasicAsyncScan(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("async_scanner_basic").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      testDataset.createEmptyDataset().close();
+      int totalRows = 40;
+
+      try (Dataset dataset = testDataset.write(1, totalRows)) {
+        // Create AsyncScanner with same options as LanceScanner
+        ScanOptions options = new ScanOptions.Builder().batchSize(20L).build();
+
+        try (AsyncScanner scanner = AsyncScanner.create(dataset, options, allocator)) {
+          // Start async scan - returns CompletableFuture<ArrowReader>
+          CompletableFuture<ArrowReader> future = scanner.scanBatchesAsync();
+
+          // Wait for result (blocks current thread, but doesn't block Rust I/O threads)
+          ArrowReader reader = future.get(10, TimeUnit.SECONDS);
+          assertNotNull(reader);
+
+          // Read all batches
+          int rowCount = 0;
+          while (reader.loadNextBatch()) {
+            VectorSchemaRoot root = reader.getVectorSchemaRoot();
+            rowCount += root.getRowCount();
+          }
+
+          assertEquals(totalRows, rowCount, "Should read all rows");
+          reader.close();
+        }
+      }
+    }
+  }
+
+  /**
+   * Example 2: Async scan with filter.
+   *
+   * <p>Shows how to use async scanner with SQL-like filters.
+   */
+  @Test
+  void testAsyncScanWithFilter(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("async_scanner_filter").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      testDataset.createEmptyDataset().close();
+
+      try (Dataset dataset = testDataset.write(1, 40)) {
+        // Scan with filter - only rows where id < 20
+        ScanOptions options = new ScanOptions.Builder().filter("id < 20").build();
+
+        try (AsyncScanner scanner = AsyncScanner.create(dataset, options, allocator)) {
+          CompletableFuture<ArrowReader> future = scanner.scanBatchesAsync();
+
+          ArrowReader reader = future.get(10, TimeUnit.SECONDS);
+          int rowCount = 0;
+          while (reader.loadNextBatch()) {
+            VectorSchemaRoot root = reader.getVectorSchemaRoot();
+            rowCount += root.getRowCount();
+          }
+
+          assertEquals(20, rowCount, "Should read only filtered rows");
+          reader.close();
+        }
+      }
+    }
+  }
+
+  /**
+   * Example 3: Multiple concurrent async scans.
+   *
+   * <p>Shows how to run multiple scans in parallel without blocking threads. This is the key
+   * benefit for query engines like Presto/Trino.
+   */
+  @Test
+  void testConcurrentAsyncScans(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("async_scanner_concurrent").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      testDataset.createEmptyDataset().close();
+      int totalRows = 100;
+
+      try (Dataset dataset = testDataset.write(1, totalRows)) {
+        // Create 5 concurrent scans with different filters
+        List<CompletableFuture<Integer>> futures = new ArrayList<>();
+
+        for (int i = 0; i < 5; i++) {
+          final int rangeStart = i * 20;
+          final int rangeEnd = rangeStart + 20;
+          String filter = String.format("id >= %d AND id < %d", rangeStart, rangeEnd);
+
+          ScanOptions options = new ScanOptions.Builder().filter(filter).build();
+
+          AsyncScanner scanner = AsyncScanner.create(dataset, options, allocator);
+
+          // Chain async operations: scan -> read -> count rows -> cleanup
+          CompletableFuture<Integer> future =
+              scanner
+                  .scanBatchesAsync()
+                  .thenApply(
+                      reader -> {
+                        try {
+                          int count = 0;
+                          while (reader.loadNextBatch()) {
+                            count += reader.getVectorSchemaRoot().getRowCount();
+                          }
+                          reader.close();
+                          scanner.close();
+                          return count;
+                        } catch (Exception e) {
+                          throw new RuntimeException(e);
+                        }
+                      });
+
+          futures.add(future);
+        }
+
+        // Wait for all scans to complete
+        CompletableFuture<Void> allDone =
+            CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
+        allDone.get(30, TimeUnit.SECONDS);
+
+        // Verify each scan read the expected number of rows
+        for (CompletableFuture<Integer> future : futures) {
+          assertEquals(20, future.get(), "Each range should have 20 rows");
+        }
+      }
+    }
+  }
+
+  /**
+   * Example 4: Async scan with error handling.
+   *
+   * <p>Shows how to handle errors in async operations.
+   */
+  @Test
+  void testAsyncScanErrorHandling(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("async_scanner_error").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      testDataset.createEmptyDataset().close();
+
+      try (Dataset dataset = testDataset.write(1, 40)) {
+        ScanOptions options = new ScanOptions.Builder().build();
+
+        try (AsyncScanner scanner = AsyncScanner.create(dataset, options, allocator)) {
+          CompletableFuture<ArrowReader> future =
+              scanner
+                  .scanBatchesAsync()
+                  .whenComplete(
+                      (reader, error) -> {
+                        if (error != null) {
+                          // Handle error
+                          System.err.println("Scan failed: " + error.getMessage());
+                        } else {
+                          // Process successful result
+                          assertNotNull(reader);
+                        }
+                      });
+
+          ArrowReader reader = future.get(10, TimeUnit.SECONDS);
+          assertNotNull(reader);
+          reader.close();
+        }
+      }
+    }
+  }
+
+  /**
+   * Example 5: Async scan with projection (column selection).
+   *
+   * <p>Shows how to select specific columns for better performance.
+   */
+  @Test
+  void testAsyncScanWithProjection(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("async_scanner_projection").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      testDataset.createEmptyDataset().close();
+
+      try (Dataset dataset = testDataset.write(1, 40)) {
+        // Select only "id" column
+        ScanOptions options = new ScanOptions.Builder().columns(List.of("id")).build();
+
+        try (AsyncScanner scanner = AsyncScanner.create(dataset, options, allocator)) {
+          CompletableFuture<ArrowReader> future = scanner.scanBatchesAsync();
+
+          ArrowReader reader = future.get(10, TimeUnit.SECONDS);
+
+          // Verify schema has only one column
+          assertEquals(1, reader.getVectorSchemaRoot().getFieldVectors().size());
+          assertEquals("id", reader.getVectorSchemaRoot().getVector(0).getName());
+
+          reader.close();
+        }
+      }
+    }
+  }
+
+  /**
+   * Example 6: Using thenCompose for sequential async operations.
+   *
+   * <p>Shows how to chain multiple async operations together.
+   */
+  @Test
+  void testAsyncChaining(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("async_scanner_chaining").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      testDataset.createEmptyDataset().close();
+
+      try (Dataset dataset = testDataset.write(1, 40)) {
+        ScanOptions options = new ScanOptions.Builder().build();
+
+        try (AsyncScanner scanner = AsyncScanner.create(dataset, options, allocator)) {
+          // Chain operations: scan -> read first batch -> extract values
+          CompletableFuture<List<Integer>> future =
+              scanner
+                  .scanBatchesAsync()
+                  .thenApply(
+                      reader -> {
+                        try {
+                          List<Integer> values = new ArrayList<>();
+                          if (reader.loadNextBatch()) {
+                            VectorSchemaRoot root = reader.getVectorSchemaRoot();
+                            IntVector idVector = (IntVector) root.getVector("id");
+                            for (int i = 0; i < root.getRowCount(); i++) {
+                              values.add(idVector.get(i));
+                            }
+                          }
+                          reader.close();
+                          return values;
+                        } catch (Exception e) {
+                          throw new RuntimeException(e);
+                        }
+                      });
+
+          List<Integer> values = future.get(10, TimeUnit.SECONDS);
+          assertTrue(values.size() > 0, "Should read some values");
+        }
+      }
+    }
+  }
+}
diff --git a/java/src/test/java/org/lance/CustomNamespaceIntegrationTest.java b/java/src/test/java/org/lance/CustomNamespaceIntegrationTest.java
new file mode 100644
index 00000000000..a836d48bf3f
--- /dev/null
+++ b/java/src/test/java/org/lance/CustomNamespaceIntegrationTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance;
+
+import org.lance.namespace.CustomNamespace;
+import org.lance.namespace.DirectoryNamespace;
+import org.lance.namespace.LanceNamespace;
+
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+
+/**
+ * Integration tests for CustomNamespace wrapper around DirectoryNamespace.
+ *
+ * <p>This test class extends DirectoryNamespaceIntegrationTest to verify that all tests pass when
+ * using a CustomNamespace wrapper around DirectoryNamespace. This validates that the Java-Rust
+ * binding works correctly for custom namespace implementations in integration scenarios.
+ *
+ * <p>These tests require LocalStack to be running. Run with: docker compose up -d
+ *
+ * <p>Set LANCE_INTEGRATION_TEST=1 environment variable to enable these tests.
+ */
+@EnabledIfEnvironmentVariable(named = "LANCE_INTEGRATION_TEST", matches = "1")
+public class CustomNamespaceIntegrationTest extends DirectoryNamespaceIntegrationTest {
+
+  @Override
+  protected LanceNamespace wrapNamespace(DirectoryNamespace inner) {
+    return new CustomNamespace(inner);
+  }
+}
diff --git a/java/src/test/java/org/lance/DatasetTest.java b/java/src/test/java/org/lance/DatasetTest.java
index 59e0ee80e7b..2b241013427 100644
--- a/java/src/test/java/org/lance/DatasetTest.java
+++ b/java/src/test/java/org/lance/DatasetTest.java
@@ -123,12 +123,12 @@ void testCreateEmptyDataset(@TempDir Path tempDir) {
   @Test
   void testGetLanceFileFormatVersion(@TempDir Path tempDir) {
     try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
-      // Test default version (V2_0)
+      // Test default version (V2_1)
       String defaultPath = tempDir.resolve("default_version").toString();
       TestUtils.SimpleTestDataset testDataset =
           new TestUtils.SimpleTestDataset(allocator, defaultPath);
       try (Dataset dataset = testDataset.createEmptyDataset()) {
-        assertEquals(LanceConstants.FILE_FORMAT_VERSION_2_0, dataset.getLanceFileFormatVersion());
+        assertEquals(LanceConstants.FILE_FORMAT_VERSION_2_1, dataset.getLanceFileFormatVersion());
       }
 
       // Test LEGACY version
@@ -178,6 +178,7 @@ void testDatasetVersion(@TempDir Path tempDir) {
       try (Dataset dataset = testDataset.createEmptyDataset()) {
         ZonedDateTime time1 = dataset.getVersion().getDataTime();
         assertEquals(1, dataset.version());
+        assertEquals(dataset.getVersion().getId(), dataset.version());
         assertTrue(time1.isEqual(before) || time1.isAfter(before));
         assertTrue(time1.isEqual(ZonedDateTime.now()) || time1.isBefore(ZonedDateTime.now()));
         assertEquals(time1.getZone(), Clock.systemUTC().getZone());
@@ -187,8 +188,10 @@ void testDatasetVersion(@TempDir Path tempDir) {
         try (Dataset dataset2 = testDataset.write(1, 5)) {
           ZonedDateTime time2 = dataset2.getVersion().getDataTime();
           assertEquals(1, dataset.version());
+          assertEquals(dataset.getVersion().getId(), dataset.version());
           assertEquals(2, dataset.latestVersion());
           assertEquals(2, dataset2.version());
+          assertEquals(dataset2.getVersion().getId(), dataset2.version());
           assertEquals(2, dataset2.latestVersion());
           assertTrue(time2.isEqual(before) || time2.isAfter(before));
           assertTrue(time2.isEqual(time1) || time2.isAfter(time1));
@@ -198,6 +201,7 @@ void testDatasetVersion(@TempDir Path tempDir) {
           ReadOptions options1 = new ReadOptions.Builder().setVersion(1).build();
           try (Dataset datasetV1 = Dataset.open(allocator, datasetPath, options1)) {
             assertEquals(1, datasetV1.version());
+            assertEquals(datasetV1.getVersion().getId(), datasetV1.version());
             assertTrue(time1.isEqual(dataset.getVersion().getDataTime()));
             assertEquals(2, datasetV1.latestVersion());
           }
@@ -206,19 +210,23 @@ void testDatasetVersion(@TempDir Path tempDir) {
           try (Dataset dataset3 = testDataset.write(2, 3)) {
             ZonedDateTime time3 = dataset3.getVersion().getDataTime();
             assertEquals(1, dataset.version());
+            assertEquals(dataset.getVersion().getId(), dataset.version());
             assertTrue(time1.isEqual(dataset.getVersion().getDataTime()));
             assertEquals(3, dataset.latestVersion());
             assertEquals(2, dataset2.version());
+            assertEquals(dataset2.getVersion().getId(), dataset2.version());
             assertTrue(time2.isEqual(dataset2.getVersion().getDataTime()));
             assertEquals(3, dataset2.latestVersion());
             assertTrue(time3.isEqual(before) || time3.isAfter(before));
             assertEquals(3, dataset3.version());
+            assertEquals(dataset3.getVersion().getId(), dataset3.version());
             assertEquals(3, dataset3.latestVersion());
 
             // Open dataset with version 2
             ReadOptions options2 = new ReadOptions.Builder().setVersion(2).build();
             try (Dataset datasetV2 = Dataset.open(allocator, datasetPath, options2)) {
               assertEquals(2, datasetV2.version());
+              assertEquals(datasetV2.getVersion().getId(), datasetV2.version());
               assertTrue(time2.isEqual(datasetV2.getVersion().getDataTime()));
               assertEquals(3, datasetV2.latestVersion());
             }
@@ -226,6 +234,7 @@ void testDatasetVersion(@TempDir Path tempDir) {
             // Open dataset with latest version (3)
             try (Dataset datasetLatest = Dataset.open(datasetPath, allocator)) {
               assertEquals(3, datasetLatest.version());
+              assertEquals(datasetLatest.getVersion().getId(), datasetLatest.version());
               assertTrue(time3.isEqual(datasetLatest.getVersion().getDataTime()));
               assertEquals(3, datasetLatest.latestVersion());
             }
@@ -242,6 +251,7 @@ void testDatasetVersion(@TempDir Path tempDir) {
             assertArrayEquals(versions.toArray(), dataset3.listVersions().toArray());
             dataset.checkoutLatest();
             assertEquals(3, dataset.version());
+            assertEquals(dataset.getVersion().getId(), dataset.version());
             assertTrue(time3.isEqual(dataset.getVersion().getDataTime()));
             assertEquals(3, dataset.latestVersion());
 
@@ -859,7 +869,7 @@ void testCalculateDataSize(@TempDir Path tempDir) {
       dataset = testDataset.createEmptyDataset();
 
       try (Dataset dataset2 = testDataset.write(1, 5)) {
-        assertEquals(100, dataset2.calculateDataSize());
+        assertEquals(108, dataset2.calculateDataSize());
       }
     }
   }
@@ -1282,6 +1292,33 @@ void testEnableStableRowIds(@TempDir Path tempDir) throws Exception {
     }
   }
 
+  @Test
+  void testHasStableRowIds(@TempDir Path tempDir) {
+    String datasetPath = tempDir.resolve("uses_stable_row_ids").toString();
+    try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+
+      // Dataset created without stable row IDs
+      try (Dataset ds = testDataset.createEmptyDataset()) {
+        assertFalse(ds.hasStableRowIds());
+      }
+    }
+
+    String datasetPathWithRowIds = tempDir.resolve("uses_stable_row_ids_enabled").toString();
+    try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPathWithRowIds);
+
+      // Dataset created with stable row IDs
+      try (Dataset ds =
+          testDataset.createDatasetWithWriteParams(
+              new WriteParams.Builder().withEnableStableRowIds(true).build())) {
+        assertTrue(ds.hasStableRowIds());
+      }
+    }
+  }
+
   @Test
   void testCompact(@TempDir Path tempDir) {
     String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
@@ -1699,12 +1736,21 @@ void testBranches(@TempDir Path tempDir) {
                   assertEquals("branch1", branch1Meta.getName());
                   assertEquals(2, branch1Meta.getParentVersion());
                   assertFalse(branch1Meta.getParentBranch().isPresent());
+                  assertEquals(1, branch1Meta.getBranchIdentifier().size());
+                  assertEquals(2, branch1Meta.getBranchIdentifier().get(0).getVersion());
+                  assertFalse(branch1Meta.getBranchIdentifier().get(0).getUuid().isEmpty());
                   assertTrue(branch1Meta.getCreateAt() > 0);
                   assertTrue(branch1Meta.getManifestSize() > 0);
 
                   assertEquals("branch2", branch2Meta.getName());
                   assertTrue(branch2Meta.getParentBranch().isPresent());
                   assertEquals("branch1", branch2Meta.getParentBranch().get());
+                  assertEquals(2, branch2Meta.getBranchIdentifier().size());
+                  assertEquals(
+                      branch1Meta.getBranchIdentifier().get(0),
+                      branch2Meta.getBranchIdentifier().get(0));
+                  assertEquals(3, branch2Meta.getBranchIdentifier().get(1).getVersion());
+                  assertFalse(branch2Meta.getBranchIdentifier().get(1).getUuid().isEmpty());
                   assertEquals(3, branch2Meta.getParentVersion());
                   assertTrue(branch2Meta.getCreateAt() > 0);
                   assertTrue(branch2Meta.getManifestSize() > 0);
@@ -1914,14 +1960,23 @@ public void testDescribeIndicesByName(@TempDir Path tempDir) throws Exception {
         assertTrue(desc.getRowsIndexed() > 0, "rowsIndexed should be positive");
         assertNotNull(desc.getMetadata(), "Metadata list should not be null");
         assertFalse(desc.getMetadata().isEmpty(), "Metadata list should not be empty");
+        assertEquals(
+            desc.getMetadata(), desc.getSegments(), "segments alias should match metadata");
         assertNotNull(desc.getDetailsJson(), "Details JSON should not be null");
 
+        assertEquals(1, desc.getSegments().size(), "Expected exactly one physical segment");
+        assertEquals("index1", desc.getSegments().get(0).name());
+
         descriptions = dataset.describeIndices();
         assertEquals(2, descriptions.size(), "Expected exactly one matching index");
         for (IndexDescription indexDesc : descriptions) {
           assertTrue(indexDesc.getRowsIndexed() > 0, "rowsIndexed should be positive");
           assertNotNull(indexDesc.getMetadata(), "Metadata list should not be null");
           assertFalse(indexDesc.getMetadata().isEmpty(), "Metadata list should not be empty");
+          assertEquals(
+              indexDesc.getMetadata(),
+              indexDesc.getSegments(),
+              "segments alias should match metadata");
           assertNotNull(indexDesc.getDetailsJson(), "Details JSON should not be null");
         }
       }
diff --git a/java/src/test/java/org/lance/NamespaceIntegrationTest.java b/java/src/test/java/org/lance/DirectoryNamespaceIntegrationTest.java
similarity index 79%
rename from java/src/test/java/org/lance/NamespaceIntegrationTest.java
rename to java/src/test/java/org/lance/DirectoryNamespaceIntegrationTest.java
index 15a329fe451..e6728f3e244 100644
--- a/java/src/test/java/org/lance/NamespaceIntegrationTest.java
+++ b/java/src/test/java/org/lance/DirectoryNamespaceIntegrationTest.java
@@ -15,14 +15,12 @@
 
 import org.lance.namespace.DirectoryNamespace;
 import org.lance.namespace.LanceNamespace;
-import org.lance.namespace.LanceNamespaceStorageOptionsProvider;
+import org.lance.namespace.errors.LanceNamespaceException;
 import org.lance.namespace.model.CreateNamespaceRequest;
 import org.lance.namespace.model.CreateTableRequest;
 import org.lance.namespace.model.CreateTableResponse;
 import org.lance.namespace.model.DeclareTableRequest;
 import org.lance.namespace.model.DeclareTableResponse;
-import org.lance.namespace.model.DescribeTableRequest;
-import org.lance.namespace.model.DescribeTableResponse;
 import org.lance.namespace.model.DropTableRequest;
 import org.lance.namespace.model.DropTableResponse;
 import org.lance.namespace.model.TableExistsRequest;
@@ -78,15 +76,15 @@
 /**
  * Integration tests for Lance with S3 and credential refresh using StorageOptionsProvider.
  *
- * <p>This test simulates a tracking credential provider that returns incrementing credentials and
- * verifies that the credential refresh mechanism works correctly.
+ * <p>This test uses DirectoryNamespace with native ops_metrics and vend_input_storage_options
+ * features to track API calls and test credential refresh mechanisms.
  *
  * <p>These tests require LocalStack to be running. Run with: docker compose up -d
  *
  * <p>Set LANCE_INTEGRATION_TEST=1 environment variable to enable these tests.
  */
 @EnabledIfEnvironmentVariable(named = "LANCE_INTEGRATION_TEST", matches = "1")
-public class NamespaceIntegrationTest {
+public class DirectoryNamespaceIntegrationTest {
 
   private static final String ENDPOINT_URL = "http://localhost:4566";
   private static final String REGION = "us-east-1";
@@ -160,105 +158,108 @@ private static void deleteBucket() {
   }
 
   /**
-   * Tracking LanceNamespace implementation for testing.
-   *
-   * <p>This implementation wraps DirectoryNamespace and tracks API calls. It returns incrementing
-   * credentials with expiration timestamps to test the credential refresh mechanism.
+   * Result holder for namespace creation that includes both the namespace to use for operations and
+   * the inner DirectoryNamespace for metrics retrieval.
    */
-  static class TrackingNamespace implements LanceNamespace {
-    private final String bucketName;
-    private final Map<String, String> baseStorageOptions;
-    private final int credentialExpiresInSeconds;
-    private final AtomicInteger describeCallCount = new AtomicInteger(0);
-    private final AtomicInteger createCallCount = new AtomicInteger(0);
-    private final DirectoryNamespace inner;
-
-    public TrackingNamespace(
-        String bucketName, Map<String, String> storageOptions, int credentialExpiresInSeconds) {
-      this.bucketName = bucketName;
-      this.baseStorageOptions = new HashMap<>(storageOptions);
-      this.credentialExpiresInSeconds = credentialExpiresInSeconds;
-
-      // Create underlying DirectoryNamespace with storage options
-      Map<String, String> dirProps = new HashMap<>();
-      for (Map.Entry<String, String> entry : storageOptions.entrySet()) {
-        dirProps.put("storage." + entry.getKey(), entry.getValue());
-      }
+  protected static class TrackingNamespaceResult {
+    /** The namespace client to use for operations. May be wrapped. */
+    public final LanceNamespace namespaceClient;
 
-      // Set root based on bucket type
-      if (bucketName.startsWith("/") || bucketName.startsWith("file://")) {
-        dirProps.put("root", bucketName + "/namespace_root");
-      } else {
-        dirProps.put("root", "s3://" + bucketName + "/namespace_root");
-      }
+    /** The inner DirectoryNamespace for metrics retrieval. */
+    public final DirectoryNamespace innerNamespaceClient;
 
-      this.inner = new DirectoryNamespace();
-      try (BufferAllocator allocator = new RootAllocator()) {
-        this.inner.initialize(dirProps, allocator);
-      }
-    }
-
-    public int getDescribeCallCount() {
-      return describeCallCount.get();
-    }
-
-    public int getCreateCallCount() {
-      return createCallCount.get();
-    }
-
-    @Override
-    public void initialize(Map<String, String> configProperties, BufferAllocator allocator) {
-      // Already initialized in constructor
+    public TrackingNamespaceResult(
+        LanceNamespace namespaceClient, DirectoryNamespace innerNamespaceClient) {
+      this.namespaceClient = namespaceClient;
+      this.innerNamespaceClient = innerNamespaceClient;
     }
+  }
 
-    @Override
-    public String namespaceId() {
-      return "TrackingNamespace { inner: " + inner.namespaceId() + " }";
+  /**
+   * Creates a DirectoryNamespace configured for testing with ops metrics and credential vending.
+   *
+   * <p>Uses native DirectoryNamespace features:
+   *
+   * <ul>
+   *   <li>ops_metrics_enabled=true: Tracks API call counts via retrieveOpsMetrics()
+   *   <li>vend_input_storage_options=true: Returns input storage options in responses
+   *   <li>vend_input_storage_options_refresh_interval_millis: Adds expires_at_millis
+   * </ul>
+   *
+   * @param bucketName S3 bucket name or local path
+   * @param storageOptions Storage options to pass through (credentials, endpoint, etc.)
+   * @param credentialExpiresInSeconds Interval in seconds for credential expiration
+   * @return TrackingNamespaceResult containing the namespace and inner DirectoryNamespace
+   */
+  protected TrackingNamespaceResult createTrackingNamespace(
+      String bucketName, Map<String, String> storageOptions, int credentialExpiresInSeconds) {
+    Map<String, String> dirProps = new HashMap<>();
+
+    // Add refresh_offset_millis to storage options so that credentials are not
+    // considered expired immediately. Set to 1 second (1000ms) so that refresh
+    // checks work correctly with short-lived credentials in tests.
+    Map<String, String> storageOptionsWithRefresh = new HashMap<>(storageOptions);
+    storageOptionsWithRefresh.put("refresh_offset_millis", "1000");
+
+    for (Map.Entry<String, String> entry : storageOptionsWithRefresh.entrySet()) {
+      dirProps.put("storage." + entry.getKey(), entry.getValue());
     }
 
-    /**
-     * Modifies storage options to add incrementing credentials with expiration timestamp.
-     *
-     * @param storageOptions Original storage options
-     * @param count Call count to use for credential generation
-     * @return Modified storage options with new credentials
-     */
-    private Map<String, String> modifyStorageOptions(
-        Map<String, String> storageOptions, int count) {
-      Map<String, String> modified =
-          storageOptions != null ? new HashMap<>(storageOptions) : new HashMap<>();
-
-      modified.put("aws_access_key_id", "AKID_" + count);
-      modified.put("aws_secret_access_key", "SECRET_" + count);
-      modified.put("aws_session_token", "TOKEN_" + count);
-
-      long expiresAtMillis = System.currentTimeMillis() + (credentialExpiresInSeconds * 1000L);
-      modified.put("expires_at_millis", String.valueOf(expiresAtMillis));
-      // Set refresh offset to 1 second (1000ms) for short-lived credential tests
-      modified.put("refresh_offset_millis", "1000");
-
-      return modified;
+    // Set root based on bucket type
+    if (bucketName.startsWith("/") || bucketName.startsWith("file://")) {
+      dirProps.put("root", bucketName + "/namespace_root");
+    } else {
+      dirProps.put("root", "s3://" + bucketName + "/namespace_root");
     }
 
-    @Override
-    public DeclareTableResponse declareTable(DeclareTableRequest request) {
-      int count = createCallCount.incrementAndGet();
-
-      DeclareTableResponse response = inner.declareTable(request);
-      response.setStorageOptions(modifyStorageOptions(response.getStorageOptions(), count));
+    // Enable ops metrics tracking
+    dirProps.put("ops_metrics_enabled", "true");
+    // Enable storage options vending
+    dirProps.put("vend_input_storage_options", "true");
+    // Set refresh interval in milliseconds
+    dirProps.put(
+        "vend_input_storage_options_refresh_interval_millis",
+        String.valueOf(credentialExpiresInSeconds * 1000L));
 
-      return response;
+    DirectoryNamespace innerNamespaceClient = new DirectoryNamespace();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      innerNamespaceClient.initialize(dirProps, allocator);
     }
+    LanceNamespace namespaceClient = wrapNamespace(innerNamespaceClient);
+    return new TrackingNamespaceResult(namespaceClient, innerNamespaceClient);
+  }
 
-    @Override
-    public DescribeTableResponse describeTable(DescribeTableRequest request) {
-      int count = describeCallCount.incrementAndGet();
+  /**
+   * Factory method to wrap the DirectoryNamespace. Subclasses can override this to provide a custom
+   * namespace implementation.
+   *
+   * @param inner The DirectoryNamespace to wrap
+   * @return The namespace to use in tests (may be the same as inner or a wrapper)
+   */
+  protected LanceNamespace wrapNamespace(DirectoryNamespace inner) {
+    return inner;
+  }
 
-      DescribeTableResponse response = inner.describeTable(request);
-      response.setStorageOptions(modifyStorageOptions(response.getStorageOptions(), count));
+  /**
+   * Gets the number of describe_table calls made to the namespace.
+   *
+   * @param namespaceClient The DirectoryNamespace to check
+   * @return Number of describe_table calls
+   */
+  protected static int getDescribeCallCount(DirectoryNamespace namespaceClient) {
+    Map<String, Long> metrics = namespaceClient.retrieveOpsMetrics();
+    return metrics.getOrDefault("describe_table", 0L).intValue();
+  }
 
-      return response;
-    }
+  /**
+   * Gets the number of declare_table calls made to the namespace.
+   *
+   * @param namespaceClient The DirectoryNamespace to check
+   * @return Number of declare_table calls
+   */
+  protected static int getDeclareCallCount(DirectoryNamespace namespaceClient) {
+    Map<String, Long> metrics = namespaceClient.retrieveOpsMetrics();
+    return metrics.getOrDefault("declare_table", 0L).intValue();
   }
 
   @Test
@@ -273,7 +274,9 @@ void testOpenDatasetWithoutRefresh() throws Exception {
       storageOptions.put("aws_region", REGION);
 
       // Create tracking namespace with 60-second expiration (long enough to not expire during test)
-      TrackingNamespace namespace = new TrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      TrackingNamespaceResult nsResult = createTrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      LanceNamespace namespaceClient = nsResult.namespaceClient;
+      DirectoryNamespace innerNamespaceClient = nsResult.innerNamespaceClient;
       String tableName = UUID.randomUUID().toString();
 
       // Create schema and data
@@ -332,36 +335,38 @@ public VectorSchemaRoot getVectorSchemaRoot() {
               }
             };
 
-        // Create dataset through namespace
+        // Create dataset through namespace client
         try (Dataset dataset =
             Dataset.write()
                 .allocator(allocator)
                 .reader(testReader)
-                .namespace(namespace)
+                .namespaceClient(namespaceClient)
                 .tableId(Arrays.asList(tableName))
                 .mode(WriteParams.WriteMode.CREATE)
+                .storageOptions(storageOptions)
                 .execute()) {
           assertEquals(2, dataset.countRows());
         }
       }
 
       // Verify declareTable was called
-      assertEquals(1, namespace.getCreateCallCount(), "declareTable should be called once");
+      assertEquals(
+          1, getDeclareCallCount(innerNamespaceClient), "declareTable should be called once");
 
-      // Open dataset through namespace WITH refresh enabled
-      ReadOptions readOptions = new ReadOptions.Builder().build();
+      // Open dataset through namespace client WITH refresh enabled
+      ReadOptions readOptions = new ReadOptions.Builder().setStorageOptions(storageOptions).build();
 
-      int callCountBeforeOpen = namespace.getDescribeCallCount();
+      int callCountBeforeOpen = getDescribeCallCount(innerNamespaceClient);
       try (Dataset dsFromNamespace =
           Dataset.open()
               .allocator(allocator)
-              .namespace(namespace)
+              .namespaceClient(namespaceClient)
               .tableId(Arrays.asList(tableName))
               .readOptions(readOptions)
               .build()) {
         // With the fix, describeTable should only be called once during open
         // to get the table location and initial storage options
-        int callCountAfterOpen = namespace.getDescribeCallCount();
+        int callCountAfterOpen = getDescribeCallCount(innerNamespaceClient);
         assertEquals(
             1,
             callCountAfterOpen - callCountBeforeOpen,
@@ -380,7 +385,7 @@ public VectorSchemaRoot getVectorSchemaRoot() {
         assertEquals(1, versions.size());
 
         // With the fix, credentials are cached so no additional calls are made
-        int finalCallCount = namespace.getDescribeCallCount();
+        int finalCallCount = getDescribeCallCount(innerNamespaceClient);
         int totalCalls = finalCallCount - callCountBeforeOpen;
         assertEquals(
             1,
@@ -403,7 +408,9 @@ void testStorageOptionsProviderWithRefresh() throws Exception {
       storageOptions.put("aws_region", REGION);
 
       // Create tracking namespace with 5-second expiration for faster testing
-      TrackingNamespace namespace = new TrackingNamespace(BUCKET_NAME, storageOptions, 5);
+      TrackingNamespaceResult nsResult = createTrackingNamespace(BUCKET_NAME, storageOptions, 5);
+      LanceNamespace namespaceClient = nsResult.namespaceClient;
+      DirectoryNamespace innerNamespaceClient = nsResult.innerNamespaceClient;
       String tableName = UUID.randomUUID().toString();
 
       // Create schema and data
@@ -462,35 +469,37 @@ public VectorSchemaRoot getVectorSchemaRoot() {
               }
             };
 
-        // Create dataset through namespace with refresh enabled
+        // Create dataset through namespace client with refresh enabled
         try (Dataset dataset =
             Dataset.write()
                 .allocator(allocator)
                 .reader(testReader)
-                .namespace(namespace)
+                .namespaceClient(namespaceClient)
                 .tableId(Arrays.asList(tableName))
                 .mode(WriteParams.WriteMode.CREATE)
+                .storageOptions(storageOptions)
                 .execute()) {
           assertEquals(2, dataset.countRows());
         }
       }
 
       // Verify declareTable was called
-      assertEquals(1, namespace.getCreateCallCount(), "declareTable should be called once");
+      assertEquals(
+          1, getDeclareCallCount(innerNamespaceClient), "declareTable should be called once");
 
-      // Open dataset through namespace with refresh enabled
-      ReadOptions readOptions = new ReadOptions.Builder().build();
+      // Open dataset through namespace client with refresh enabled
+      ReadOptions readOptions = new ReadOptions.Builder().setStorageOptions(storageOptions).build();
 
-      int callCountBeforeOpen = namespace.getDescribeCallCount();
+      int callCountBeforeOpen = getDescribeCallCount(innerNamespaceClient);
       try (Dataset dsFromNamespace =
           Dataset.open()
               .allocator(allocator)
-              .namespace(namespace)
+              .namespaceClient(namespaceClient)
               .tableId(Arrays.asList(tableName))
               .readOptions(readOptions)
               .build()) {
         // With the fix, describeTable should only be called once during open
-        int callCountAfterOpen = namespace.getDescribeCallCount();
+        int callCountAfterOpen = getDescribeCallCount(innerNamespaceClient);
         assertEquals(
             1,
             callCountAfterOpen - callCountBeforeOpen,
@@ -501,7 +510,7 @@ public VectorSchemaRoot getVectorSchemaRoot() {
         assertEquals(2, dsFromNamespace.countRows());
 
         // Record call count after initial reads
-        int callCountAfterInitialReads = namespace.getDescribeCallCount();
+        int callCountAfterInitialReads = getDescribeCallCount(innerNamespaceClient);
         int callsAfterFirstRead = callCountAfterInitialReads - callCountBeforeOpen;
         assertEquals(
             1,
@@ -520,7 +529,7 @@ public VectorSchemaRoot getVectorSchemaRoot() {
         List<Version> versions = dsFromNamespace.listVersions();
         assertEquals(1, versions.size());
 
-        int finalCallCount = namespace.getDescribeCallCount();
+        int finalCallCount = getDescribeCallCount(innerNamespaceClient);
         int totalCallsAfterExpiration = finalCallCount - callCountBeforeOpen;
         assertEquals(
             2,
@@ -544,7 +553,9 @@ void testWriteDatasetBuilderWithNamespaceCreate() throws Exception {
       storageOptions.put("aws_region", REGION);
 
       // Create tracking namespace
-      TrackingNamespace namespace = new TrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      TrackingNamespaceResult nsResult = createTrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      LanceNamespace namespaceClient = nsResult.namespaceClient;
+      DirectoryNamespace innerNamespaceClient = nsResult.innerNamespaceClient;
       String tableName = UUID.randomUUID().toString();
 
       // Create schema and data
@@ -603,20 +614,21 @@ public VectorSchemaRoot getVectorSchemaRoot() {
               }
             };
 
-        int callCountBefore = namespace.getCreateCallCount();
+        int callCountBefore = getDeclareCallCount(innerNamespaceClient);
 
-        // Use the write builder to create a dataset through namespace
+        // Use the write builder to create a dataset through namespace client
         try (Dataset dataset =
             Dataset.write()
                 .allocator(allocator)
                 .reader(testReader)
-                .namespace(namespace)
+                .namespaceClient(namespaceClient)
                 .tableId(Arrays.asList(tableName))
                 .mode(WriteParams.WriteMode.CREATE)
+                .storageOptions(storageOptions)
                 .execute()) {
 
           // Verify declareTable was called
-          int callCountAfter = namespace.getCreateCallCount();
+          int callCountAfter = getDeclareCallCount(innerNamespaceClient);
           assertEquals(1, callCountAfter - callCountBefore, "declareTable should be called once");
 
           // Verify dataset was created successfully
@@ -641,12 +653,16 @@ void testWriteDatasetBuilderWithNamespaceCreateCallCounts() throws Exception {
       // Create tracking namespace with 60-second expiration (long enough that no refresh happens)
       // Credentials expire at T+60s. With a 1s refresh offset, refresh would happen at T+59s.
       // Since writes complete well under 59 seconds, NO credential refresh should occur.
-      TrackingNamespace namespace = new TrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      TrackingNamespaceResult nsResult = createTrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      LanceNamespace namespaceClient = nsResult.namespaceClient;
+      DirectoryNamespace innerNamespaceClient = nsResult.innerNamespaceClient;
       String tableName = UUID.randomUUID().toString();
 
       // Verify initial call counts
-      assertEquals(0, namespace.getCreateCallCount(), "declareTable should not be called yet");
-      assertEquals(0, namespace.getDescribeCallCount(), "describeTable should not be called yet");
+      assertEquals(
+          0, getDeclareCallCount(innerNamespaceClient), "declareTable should not be called yet");
+      assertEquals(
+          0, getDescribeCallCount(innerNamespaceClient), "describeTable should not be called yet");
 
       // Create schema and data
       Schema schema =
@@ -704,27 +720,30 @@ public VectorSchemaRoot getVectorSchemaRoot() {
               }
             };
 
-        // Use the write builder to create a dataset through namespace
+        // Use the write builder to create a dataset through namespace client
         // Write completes instantly, so NO describeTable call should happen for refresh.
         try (Dataset dataset =
             Dataset.write()
                 .allocator(allocator)
                 .reader(testReader)
-                .namespace(namespace)
+                .namespaceClient(namespaceClient)
                 .tableId(Arrays.asList(tableName))
                 .mode(WriteParams.WriteMode.CREATE)
+                .storageOptions(storageOptions)
                 .execute()) {
 
           // Verify declareTable was called exactly ONCE
           assertEquals(
-              1, namespace.getCreateCallCount(), "declareTable should be called exactly once");
+              1,
+              getDeclareCallCount(innerNamespaceClient),
+              "declareTable should be called exactly once");
 
           // Verify describeTable was NOT called during CREATE
           // Initial credentials come from declareTable response, and since credentials
           // don't expire during the fast write, NO refresh (describeTable) is needed
           assertEquals(
               0,
-              namespace.getDescribeCallCount(),
+              getDescribeCallCount(innerNamespaceClient),
               "describeTable should NOT be called during CREATE - "
                   + "initial credentials come from declareTable response and don't expire");
 
@@ -735,19 +754,22 @@ public VectorSchemaRoot getVectorSchemaRoot() {
       }
 
       // Verify counts after dataset is closed
-      assertEquals(1, namespace.getCreateCallCount(), "declareTable should still be 1 after close");
+      assertEquals(
+          1,
+          getDeclareCallCount(innerNamespaceClient),
+          "declareTable should still be 1 after close");
       assertEquals(
           0,
-          namespace.getDescribeCallCount(),
+          getDescribeCallCount(innerNamespaceClient),
           "describeTable should still be 0 after close (no refresh needed)");
 
-      // Now open the dataset through namespace with long-lived credentials (60s expiration)
-      ReadOptions readOptions = new ReadOptions.Builder().build();
+      // Now open the dataset through namespace client with long-lived credentials (60s expiration)
+      ReadOptions readOptions = new ReadOptions.Builder().setStorageOptions(storageOptions).build();
 
       try (Dataset dsFromNamespace =
           Dataset.open()
               .allocator(allocator)
-              .namespace(namespace)
+              .namespaceClient(namespaceClient)
               .tableId(Arrays.asList(tableName))
               .readOptions(readOptions)
               .build()) {
@@ -755,13 +777,13 @@ public VectorSchemaRoot getVectorSchemaRoot() {
         // declareTable should NOT be called during open (only during CREATE)
         assertEquals(
             1,
-            namespace.getCreateCallCount(),
+            getDeclareCallCount(innerNamespaceClient),
             "declareTable should still be 1 (not called during open)");
 
         // describeTable is called exactly ONCE during open to get table location
         assertEquals(
             1,
-            namespace.getDescribeCallCount(),
+            getDescribeCallCount(innerNamespaceClient),
             "describeTable should be called exactly once during open");
 
         // Verify we can read the data multiple times
@@ -773,13 +795,13 @@ public VectorSchemaRoot getVectorSchemaRoot() {
         // (credentials are cached and don't expire during this fast test)
         assertEquals(
             1,
-            namespace.getDescribeCallCount(),
+            getDescribeCallCount(innerNamespaceClient),
             "describeTable should still be 1 after reads (credentials cached, no refresh needed)");
       }
 
       // Final verification
-      assertEquals(1, namespace.getCreateCallCount(), "Final: declareTable = 1");
-      assertEquals(1, namespace.getDescribeCallCount(), "Final: describeTable = 1");
+      assertEquals(1, getDeclareCallCount(innerNamespaceClient), "Final: declareTable = 1");
+      assertEquals(1, getDescribeCallCount(innerNamespaceClient), "Final: describeTable = 1");
     }
   }
 
@@ -795,7 +817,9 @@ void testWriteDatasetBuilderWithNamespaceAppend() throws Exception {
       storageOptions.put("aws_region", REGION);
 
       // Create tracking namespace
-      TrackingNamespace namespace = new TrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      TrackingNamespaceResult nsResult = createTrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      LanceNamespace namespaceClient = nsResult.namespaceClient;
+      DirectoryNamespace innerNamespaceClient = nsResult.innerNamespaceClient;
       String tableName = UUID.randomUUID().toString();
 
       Schema schema =
@@ -853,22 +877,24 @@ public VectorSchemaRoot getVectorSchemaRoot() {
               }
             };
 
-        // Create initial dataset through namespace
+        // Create initial dataset through namespace client
         try (Dataset dataset =
             Dataset.write()
                 .allocator(allocator)
                 .reader(testReader)
-                .namespace(namespace)
+                .namespaceClient(namespaceClient)
                 .tableId(Arrays.asList(tableName))
                 .mode(WriteParams.WriteMode.CREATE)
+                .storageOptions(storageOptions)
                 .execute()) {
           assertEquals(2, dataset.countRows());
         }
 
-        assertEquals(1, namespace.getCreateCallCount(), "declareTable should be called once");
-        int initialDescribeCount = namespace.getDescribeCallCount();
+        assertEquals(
+            1, getDeclareCallCount(innerNamespaceClient), "declareTable should be called once");
+        int initialDescribeCount = getDescribeCallCount(innerNamespaceClient);
 
-        // Now append data using the write builder with namespace
+        // Now append data using the write builder with namespace client
         ArrowReader appendReader =
             new ArrowReader(allocator) {
               boolean firstRead = true;
@@ -901,18 +927,19 @@ public VectorSchemaRoot getVectorSchemaRoot() {
               }
             };
 
-        // Use the write builder to append to dataset through namespace
+        // Use the write builder to append to dataset through namespace client
         try (Dataset dataset =
             Dataset.write()
                 .allocator(allocator)
                 .reader(appendReader)
-                .namespace(namespace)
+                .namespaceClient(namespaceClient)
                 .tableId(Arrays.asList(tableName))
                 .mode(WriteParams.WriteMode.APPEND)
+                .storageOptions(storageOptions)
                 .execute()) {
 
           // Verify describeTable was called
-          int callCountAfter = namespace.getDescribeCallCount();
+          int callCountAfter = getDescribeCallCount(innerNamespaceClient);
           assertEquals(
               1,
               callCountAfter - initialDescribeCount,
@@ -937,7 +964,9 @@ void testWriteDatasetBuilderWithNamespaceOverwrite() throws Exception {
       storageOptions.put("aws_region", REGION);
 
       // Create tracking namespace
-      TrackingNamespace namespace = new TrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      TrackingNamespaceResult nsResult = createTrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      LanceNamespace namespaceClient = nsResult.namespaceClient;
+      DirectoryNamespace innerNamespaceClient = nsResult.innerNamespaceClient;
       String tableName = UUID.randomUUID().toString();
 
       Schema schema =
@@ -997,15 +1026,20 @@ public VectorSchemaRoot getVectorSchemaRoot() {
             Dataset.write()
                 .allocator(allocator)
                 .reader(createReader)
-                .namespace(namespace)
+                .namespaceClient(namespaceClient)
                 .tableId(Arrays.asList(tableName))
                 .mode(WriteParams.WriteMode.CREATE)
+                .storageOptions(storageOptions)
                 .execute()) {
           assertEquals(1, dataset.countRows());
         }
 
-        assertEquals(1, namespace.getCreateCallCount(), "declareTable should be called once");
-        assertEquals(0, namespace.getDescribeCallCount(), "describeTable should not be called yet");
+        assertEquals(
+            1, getDeclareCallCount(innerNamespaceClient), "declareTable should be called once");
+        assertEquals(
+            0,
+            getDescribeCallCount(innerNamespaceClient),
+            "describeTable should not be called yet");
 
         // Now overwrite with 2 rows
         aVector.allocateNew(2);
@@ -1056,14 +1090,16 @@ public VectorSchemaRoot getVectorSchemaRoot() {
             Dataset.write()
                 .allocator(allocator)
                 .reader(overwriteReader)
-                .namespace(namespace)
+                .namespaceClient(namespaceClient)
                 .tableId(Arrays.asList(tableName))
                 .mode(WriteParams.WriteMode.OVERWRITE)
+                .storageOptions(storageOptions)
                 .execute()) {
 
           // Verify describeTable was called for overwrite
-          assertEquals(1, namespace.getCreateCallCount(), "declareTable should still be 1");
-          int describeCountAfterOverwrite = namespace.getDescribeCallCount();
+          assertEquals(
+              1, getDeclareCallCount(innerNamespaceClient), "declareTable should still be 1");
+          int describeCountAfterOverwrite = getDescribeCallCount(innerNamespaceClient);
           assertEquals(
               1, describeCountAfterOverwrite, "describeTable should be called once for overwrite");
 
@@ -1073,12 +1109,13 @@ public VectorSchemaRoot getVectorSchemaRoot() {
               2, dataset.listVersions().size()); // Version 1 (create) + Version 2 (overwrite)
         }
 
-        // Verify we can open and read the dataset through namespace
+        // Verify we can open and read the dataset through namespace client
         try (Dataset ds =
             Dataset.open()
                 .allocator(allocator)
-                .namespace(namespace)
+                .namespaceClient(namespaceClient)
                 .tableId(Arrays.asList(tableName))
+                .readOptions(new ReadOptions.Builder().setStorageOptions(storageOptions).build())
                 .build()) {
           assertEquals(2, ds.countRows(), "Should have 2 rows after overwrite");
           assertEquals(2, ds.listVersions().size(), "Should have 2 versions");
@@ -1099,7 +1136,9 @@ void testDistributedWriteWithNamespace() throws Exception {
       storageOptions.put("aws_region", REGION);
 
       // Create tracking namespace
-      TrackingNamespace namespace = new TrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      TrackingNamespaceResult nsResult = createTrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      LanceNamespace namespaceClient = nsResult.namespaceClient;
+      DirectoryNamespace innerNamespaceClient = nsResult.innerNamespaceClient;
       String tableName = UUID.randomUUID().toString();
 
       Schema schema =
@@ -1111,10 +1150,12 @@ void testDistributedWriteWithNamespace() throws Exception {
       // Step 1: Declare table via namespace
       DeclareTableRequest request = new DeclareTableRequest();
       request.setId(Arrays.asList(tableName));
-      DeclareTableResponse response = namespace.declareTable(request);
+      DeclareTableResponse response = namespaceClient.declareTable(request);
 
-      assertEquals(1, namespace.getCreateCallCount(), "declareTable should be called once");
-      assertEquals(0, namespace.getDescribeCallCount(), "describeTable should not be called yet");
+      assertEquals(
+          1, getDeclareCallCount(innerNamespaceClient), "declareTable should be called once");
+      assertEquals(
+          0, getDescribeCallCount(innerNamespaceClient), "describeTable should not be called yet");
 
       String tableUri = response.getLocation();
       Map<String, String> namespaceStorageOptions = response.getStorageOptions();
@@ -1125,11 +1166,8 @@ void testDistributedWriteWithNamespace() throws Exception {
         mergedOptions.putAll(namespaceStorageOptions);
       }
 
-      // Create storage options provider
-      LanceNamespaceStorageOptionsProvider storageOptionsProvider =
-          new LanceNamespaceStorageOptionsProvider(namespace, Arrays.asList(tableName));
-
       WriteParams writeParams = new WriteParams.Builder().withStorageOptions(mergedOptions).build();
+      List<String> tableId = Arrays.asList(tableName);
 
       // Step 2: Write multiple fragments in parallel (simulated)
       List<FragmentMetadata> allFragments = new ArrayList<>();
@@ -1150,7 +1188,7 @@ void testDistributedWriteWithNamespace() throws Exception {
         root.setRowCount(2);
 
         List<FragmentMetadata> fragment1 =
-            Fragment.create(tableUri, allocator, root, writeParams, storageOptionsProvider);
+            Fragment.create(tableUri, allocator, root, writeParams, namespaceClient, tableId);
         allFragments.addAll(fragment1);
       }
 
@@ -1170,7 +1208,7 @@ void testDistributedWriteWithNamespace() throws Exception {
         root.setRowCount(2);
 
         List<FragmentMetadata> fragment2 =
-            Fragment.create(tableUri, allocator, root, writeParams, storageOptionsProvider);
+            Fragment.create(tableUri, allocator, root, writeParams, namespaceClient, tableId);
         allFragments.addAll(fragment2);
       }
 
@@ -1188,7 +1226,7 @@ void testDistributedWriteWithNamespace() throws Exception {
         root.setRowCount(1);
 
         List<FragmentMetadata> fragment3 =
-            Fragment.create(tableUri, allocator, root, writeParams, storageOptionsProvider);
+            Fragment.create(tableUri, allocator, root, writeParams, namespaceClient, tableId);
         allFragments.addAll(fragment3);
       }
 
@@ -1202,14 +1240,15 @@ void testDistributedWriteWithNamespace() throws Exception {
         assertEquals(1, dataset.listVersions().size(), "Should have 1 version after commit");
       }
 
-      // Step 4: Open dataset through namespace and verify
+      // Step 4: Open dataset through namespace client and verify
       try (Dataset dsFromNamespace =
           Dataset.open()
               .allocator(allocator)
-              .namespace(namespace)
+              .namespaceClient(namespaceClient)
               .tableId(Arrays.asList(tableName))
+              .readOptions(new ReadOptions.Builder().setStorageOptions(storageOptions).build())
               .build()) {
-        assertEquals(5, dsFromNamespace.countRows(), "Should read 5 rows through namespace");
+        assertEquals(5, dsFromNamespace.countRows(), "Should read 5 rows through namespace client");
       }
     }
   }
@@ -1226,7 +1265,9 @@ void testFragmentCreateAndCommitWithNamespace() throws Exception {
       storageOptions.put("aws_region", REGION);
 
       // Create tracking namespace with 60-second expiration
-      TrackingNamespace namespace = new TrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      TrackingNamespaceResult nsResult = createTrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      LanceNamespace namespaceClient = nsResult.namespaceClient;
+      DirectoryNamespace innerNamespaceClient = nsResult.innerNamespaceClient;
       String tableName = UUID.randomUUID().toString();
 
       Schema schema =
@@ -1238,9 +1279,10 @@ void testFragmentCreateAndCommitWithNamespace() throws Exception {
       // Declare table via namespace
       DeclareTableRequest request = new DeclareTableRequest();
       request.setId(Arrays.asList(tableName));
-      DeclareTableResponse response = namespace.declareTable(request);
+      DeclareTableResponse response = namespaceClient.declareTable(request);
 
-      assertEquals(1, namespace.getCreateCallCount(), "declareTable should be called once");
+      assertEquals(
+          1, getDeclareCallCount(innerNamespaceClient), "declareTable should be called once");
 
       String tableUri = response.getLocation();
       Map<String, String> namespaceStorageOptions = response.getStorageOptions();
@@ -1251,11 +1293,8 @@ void testFragmentCreateAndCommitWithNamespace() throws Exception {
         mergedOptions.putAll(namespaceStorageOptions);
       }
 
-      // Create storage options provider
-      LanceNamespaceStorageOptionsProvider provider =
-          new LanceNamespaceStorageOptionsProvider(namespace, Arrays.asList(tableName));
-
       WriteParams writeParams = new WriteParams.Builder().withStorageOptions(mergedOptions).build();
+      List<String> tableId = Arrays.asList(tableName);
 
       try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
         IntVector idVector = (IntVector) root.getVector("id");
@@ -1276,9 +1315,9 @@ void testFragmentCreateAndCommitWithNamespace() throws Exception {
         valueVector.setValueCount(3);
         root.setRowCount(3);
 
-        // Create fragment with StorageOptionsProvider
+        // Create fragment with namespace client
         List<FragmentMetadata> fragments1 =
-            Fragment.create(tableUri, allocator, root, writeParams, provider);
+            Fragment.create(tableUri, allocator, root, writeParams, namespaceClient, tableId);
 
         assertEquals(1, fragments1.size());
 
@@ -1291,9 +1330,9 @@ void testFragmentCreateAndCommitWithNamespace() throws Exception {
         valueVector.set(2, 600);
         root.setRowCount(3);
 
-        // Create another fragment with the same provider
+        // Create another fragment with the same namespace client
         List<FragmentMetadata> fragments2 =
-            Fragment.create(tableUri, allocator, root, writeParams, provider);
+            Fragment.create(tableUri, allocator, root, writeParams, namespaceClient, tableId);
 
         assertEquals(1, fragments2.size());
 
@@ -1315,12 +1354,13 @@ void testFragmentCreateAndCommitWithNamespace() throws Exception {
         }
       }
 
-      // Verify we can open and read the dataset through namespace
+      // Verify we can open and read the dataset through namespace client
       try (Dataset ds =
           Dataset.open()
               .allocator(allocator)
-              .namespace(namespace)
+              .namespaceClient(namespaceClient)
               .tableId(Arrays.asList(tableName))
+              .readOptions(new ReadOptions.Builder().setStorageOptions(storageOptions).build())
               .build()) {
         assertEquals(6, ds.countRows(), "Should have 6 rows total");
         assertEquals(2, ds.listVersions().size(), "Should have 2 versions");
@@ -1340,7 +1380,8 @@ void testTransactionCommitWithNamespace() throws Exception {
       storageOptions.put("aws_region", REGION);
 
       // Create tracking namespace
-      TrackingNamespace namespace = new TrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      TrackingNamespaceResult nsResult = createTrackingNamespace(BUCKET_NAME, storageOptions, 60);
+      LanceNamespace namespaceClient = nsResult.namespaceClient;
       String tableName = UUID.randomUUID().toString();
 
       Schema schema =
@@ -1352,7 +1393,7 @@ void testTransactionCommitWithNamespace() throws Exception {
       // Declare table via namespace
       DeclareTableRequest request = new DeclareTableRequest();
       request.setId(Arrays.asList(tableName));
-      DeclareTableResponse response = namespace.declareTable(request);
+      DeclareTableResponse response = namespaceClient.declareTable(request);
 
       String tableUri = response.getLocation();
       Map<String, String> namespaceStorageOptions = response.getStorageOptions();
@@ -1363,12 +1404,9 @@ void testTransactionCommitWithNamespace() throws Exception {
         mergedOptions.putAll(namespaceStorageOptions);
       }
 
-      // Create storage options provider
-      LanceNamespaceStorageOptionsProvider provider =
-          new LanceNamespaceStorageOptionsProvider(namespace, Arrays.asList(tableName));
-
       // First, write some initial data using Fragment.create and commit
       WriteParams writeParams = new WriteParams.Builder().withStorageOptions(mergedOptions).build();
+      List<String> tableId = Arrays.asList(tableName);
 
       List<FragmentMetadata> initialFragments;
       try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
@@ -1388,7 +1426,8 @@ void testTransactionCommitWithNamespace() throws Exception {
         nameVector.setValueCount(2);
         root.setRowCount(2);
 
-        initialFragments = Fragment.create(tableUri, allocator, root, writeParams, provider);
+        initialFragments =
+            Fragment.create(tableUri, allocator, root, writeParams, namespaceClient, tableId);
       }
 
       // Commit initial fragments
@@ -1400,15 +1439,17 @@ void testTransactionCommitWithNamespace() throws Exception {
         assertEquals(2, dataset.countRows());
       }
 
-      // Now test Transaction.commit with provider
-      // Open dataset with provider using mergedOptions (which has expires_at_millis)
-      ReadOptions readOptions =
-          new ReadOptions.Builder()
-              .setStorageOptions(mergedOptions)
-              .setStorageOptionsProvider(provider)
-              .build();
+      // Now test Transaction.commit with namespace client
+      // Open dataset with namespace client using mergedOptions (which has expires_at_millis)
+      ReadOptions readOptions = new ReadOptions.Builder().setStorageOptions(mergedOptions).build();
 
-      try (Dataset datasetWithProvider = Dataset.open(allocator, tableUri, readOptions)) {
+      try (Dataset datasetWithNamespaceClient =
+          Dataset.open()
+              .allocator(allocator)
+              .namespaceClient(namespaceClient)
+              .tableId(tableId)
+              .readOptions(readOptions)
+              .build()) {
         // Create more fragments to append
         List<FragmentMetadata> newFragments;
         try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
@@ -1428,30 +1469,32 @@ void testTransactionCommitWithNamespace() throws Exception {
           nameVector.setValueCount(2);
           root.setRowCount(2);
 
-          newFragments = Fragment.create(tableUri, allocator, root, writeParams, provider);
+          newFragments =
+              Fragment.create(tableUri, allocator, root, writeParams, namespaceClient, tableId);
         }
 
         // Create and commit transaction
         Append appendOp = Append.builder().fragments(newFragments).build();
         try (Transaction transaction =
             new Transaction.Builder()
-                .readVersion(datasetWithProvider.version())
+                .readVersion(datasetWithNamespaceClient.version())
                 .operation(appendOp)
                 .build()) {
           try (Dataset committedDataset =
-              new CommitBuilder(datasetWithProvider).execute(transaction)) {
+              new CommitBuilder(datasetWithNamespaceClient).execute(transaction)) {
             assertEquals(2, committedDataset.version());
             assertEquals(4, committedDataset.countRows());
           }
         }
       }
 
-      // Verify we can open and read the dataset through namespace
+      // Verify we can open and read the dataset through namespace client
       try (Dataset ds =
           Dataset.open()
               .allocator(allocator)
-              .namespace(namespace)
+              .namespaceClient(namespaceClient)
               .tableId(Arrays.asList(tableName))
+              .readOptions(new ReadOptions.Builder().setStorageOptions(storageOptions).build())
               .build()) {
         assertEquals(4, ds.countRows(), "Should have 4 rows total");
         assertEquals(2, ds.listVersions().size(), "Should have 2 versions");
@@ -1518,8 +1561,8 @@ private byte[] createTestTableData() throws Exception {
 
   @Test
   void testBasicCreateAndDropOnS3() throws Exception {
-    DirectoryNamespace namespace = new DirectoryNamespace();
-    namespace.initialize(createDirectoryNamespaceS3Config(), testAllocator);
+    DirectoryNamespace namespaceClient = new DirectoryNamespace();
+    namespaceClient.initialize(createDirectoryNamespaceS3Config(), testAllocator);
 
     try {
       String tableName = "basic_test_table";
@@ -1527,32 +1570,32 @@ void testBasicCreateAndDropOnS3() throws Exception {
       byte[] tableData = createTestTableData();
 
       CreateTableRequest createReq = new CreateTableRequest().id(tableId);
-      CreateTableResponse createResp = namespace.createTable(createReq, tableData);
+      CreateTableResponse createResp = namespaceClient.createTable(createReq, tableData);
       assertNotNull(createResp);
       assertNotNull(createResp.getLocation());
 
       DropTableRequest dropReq = new DropTableRequest().id(tableId);
-      DropTableResponse dropResp = namespace.dropTable(dropReq);
+      DropTableResponse dropResp = namespaceClient.dropTable(dropReq);
       assertNotNull(dropResp);
 
       TableExistsRequest existsReq = new TableExistsRequest().id(tableId);
-      assertThrows(RuntimeException.class, () -> namespace.tableExists(existsReq));
+      assertThrows(LanceNamespaceException.class, () -> namespaceClient.tableExists(existsReq));
     } finally {
-      namespace.close();
+      namespaceClient.close();
     }
   }
 
   @Test
   void testConcurrentCreateAndDropWithSingleInstanceOnS3() throws Exception {
-    DirectoryNamespace namespace = new DirectoryNamespace();
-    namespace.initialize(createDirectoryNamespaceS3Config(), testAllocator);
+    DirectoryNamespace namespaceClient = new DirectoryNamespace();
+    namespaceClient.initialize(createDirectoryNamespaceS3Config(), testAllocator);
 
     try {
       // Initialize namespace first - create parent namespace to ensure __manifest table
       // is created before concurrent operations
       CreateNamespaceRequest createNsReq =
           new CreateNamespaceRequest().id(Arrays.asList("test_ns"));
-      namespace.createNamespace(createNsReq);
+      namespaceClient.createNamespace(createNsReq);
 
       int numTables = 10;
       ExecutorService executor = Executors.newFixedThreadPool(numTables);
@@ -1573,10 +1616,10 @@ void testConcurrentCreateAndDropWithSingleInstanceOnS3() throws Exception {
                 byte[] tableData = createTestTableData();
 
                 CreateTableRequest createReq = new CreateTableRequest().id(tableId);
-                namespace.createTable(createReq, tableData);
+                namespaceClient.createTable(createReq, tableData);
 
                 DropTableRequest dropReq = new DropTableRequest().id(tableId);
-                namespace.dropTable(dropReq);
+                namespaceClient.dropTable(dropReq);
 
                 successCount.incrementAndGet();
               } catch (Exception e) {
@@ -1596,7 +1639,7 @@ void testConcurrentCreateAndDropWithSingleInstanceOnS3() throws Exception {
       assertEquals(numTables, successCount.get(), "All tasks should succeed");
       assertEquals(0, failCount.get(), "No tasks should fail");
     } finally {
-      namespace.close();
+      namespaceClient.close();
     }
   }
 
diff --git a/java/src/test/java/org/lance/TestVectorDataset.java b/java/src/test/java/org/lance/TestVectorDataset.java
index f05c7dc7abb..873b0fa7f66 100644
--- a/java/src/test/java/org/lance/TestVectorDataset.java
+++ b/java/src/test/java/org/lance/TestVectorDataset.java
@@ -20,7 +20,10 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.*;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
 import org.apache.arrow.vector.types.FloatingPointPrecision;
 import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -31,7 +34,13 @@
 
 import java.io.IOException;
 import java.nio.file.Path;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 
 public class TestVectorDataset implements AutoCloseable {
   public static final String vectorColumnName = "vec";
diff --git a/java/src/test/java/org/lance/index/VectorIndexTest.java b/java/src/test/java/org/lance/index/VectorIndexTest.java
index 4a5902044da..50499197b34 100755
--- a/java/src/test/java/org/lance/index/VectorIndexTest.java
+++ b/java/src/test/java/org/lance/index/VectorIndexTest.java
@@ -13,18 +13,15 @@
  */
 package org.lance.index;
 
-import org.lance.CommitBuilder;
 import org.lance.Dataset;
 import org.lance.Fragment;
 import org.lance.TestVectorDataset;
-import org.lance.Transaction;
 import org.lance.index.vector.IvfBuildParams;
 import org.lance.index.vector.PQBuildParams;
 import org.lance.index.vector.RQBuildParams;
 import org.lance.index.vector.SQBuildParams;
 import org.lance.index.vector.VectorIndexParams;
 import org.lance.index.vector.VectorTrainer;
-import org.lance.operation.CreateIndex;
 
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
@@ -32,9 +29,6 @@
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.List;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.stream.Collectors;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -75,72 +69,38 @@ public void testCreateIvfFlatIndexDistributively(@TempDir Path tempDir) throws E
         IndexParams indexParams =
             IndexParams.builder().setVectorIndexParams(vectorIndexParams).build();
 
-        UUID indexUUID = UUID.randomUUID();
-
-        // Partially create index on the first fragment
-        dataset.createIndex(
-            IndexOptions.builder(
-                    Collections.singletonList(TestVectorDataset.vectorColumnName),
-                    IndexType.IVF_FLAT,
-                    indexParams)
-                .withIndexName(TestVectorDataset.indexName)
-                .withIndexUUID(indexUUID.toString())
-                .withFragmentIds(Collections.singletonList(fragments.get(0).getId()))
-                .build());
-
-        // Partially create index on the second fragment with the same UUID
-        dataset.createIndex(
-            IndexOptions.builder(
-                    Collections.singletonList(TestVectorDataset.vectorColumnName),
-                    IndexType.IVF_FLAT,
-                    indexParams)
-                .withIndexName(TestVectorDataset.indexName)
-                .withIndexUUID(indexUUID.toString())
-                .withFragmentIds(Collections.singletonList(fragments.get(1).getId()))
-                .build());
+        Index firstSegment =
+            dataset.createIndex(
+                IndexOptions.builder(
+                        Collections.singletonList(TestVectorDataset.vectorColumnName),
+                        IndexType.IVF_FLAT,
+                        indexParams)
+                    .withIndexName(TestVectorDataset.indexName)
+                    .withFragmentIds(Collections.singletonList(fragments.get(0).getId()))
+                    .build());
+
+        Index secondSegment =
+            dataset.createIndex(
+                IndexOptions.builder(
+                        Collections.singletonList(TestVectorDataset.vectorColumnName),
+                        IndexType.IVF_FLAT,
+                        indexParams)
+                    .withIndexName(TestVectorDataset.indexName)
+                    .withFragmentIds(Collections.singletonList(fragments.get(1).getId()))
+                    .build());
 
         // The index should not be visible before metadata merge & commit
         assertFalse(
             dataset.listIndexes().contains(TestVectorDataset.indexName),
             "Partially created IVF_FLAT index should not present before commit");
 
-        // Merge index metadata for all fragment-level pieces
-        dataset.mergeIndexMetadata(indexUUID.toString(), IndexType.IVF_FLAT, Optional.empty());
-
-        int fieldId =
-            dataset.getLanceSchema().fields().stream()
-                .filter(f -> f.getName().equals(TestVectorDataset.vectorColumnName))
-                .findAny()
-                .orElseThrow(
-                    () -> new RuntimeException("Cannot find vector field for TestVectorDataset"))
-                .getId();
-
-        long datasetVersion = dataset.version();
-
-        Index index =
-            Index.builder()
-                .uuid(indexUUID)
-                .name(TestVectorDataset.indexName)
-                .fields(Collections.singletonList(fieldId))
-                .datasetVersion(datasetVersion)
-                .indexVersion(0)
-                .fragments(
-                    fragments.stream().limit(2).map(Fragment::getId).collect(Collectors.toList()))
-                .build();
-
-        CreateIndex createIndexOp =
-            CreateIndex.builder().withNewIndices(Collections.singletonList(index)).build();
-
-        try (Transaction createIndexTx =
-            new Transaction.Builder()
-                .readVersion(dataset.version())
-                .operation(createIndexOp)
-                .build()) {
-          try (Dataset newDataset = new CommitBuilder(dataset).execute(createIndexTx)) {
-            assertEquals(datasetVersion + 1, newDataset.version());
-            assertTrue(newDataset.listIndexes().contains(TestVectorDataset.indexName));
-          }
-        }
+        List<Index> committed =
+            dataset.commitExistingIndexSegments(
+                TestVectorDataset.indexName,
+                TestVectorDataset.vectorColumnName,
+                List.of(firstSegment, secondSegment));
+        assertEquals(2, committed.size());
+        assertTrue(dataset.listIndexes().contains(TestVectorDataset.indexName));
       }
     }
   }
@@ -200,68 +160,37 @@ public void testCreateIvfPqIndexDistributively(@TempDir Path tempDir) throws Exc
         IndexParams indexParams =
             IndexParams.builder().setVectorIndexParams(vectorIndexParams).build();
 
-        UUID indexUUID = UUID.randomUUID();
-
-        dataset.createIndex(
-            IndexOptions.builder(
-                    Collections.singletonList(TestVectorDataset.vectorColumnName),
-                    IndexType.IVF_PQ,
-                    indexParams)
-                .withIndexName(TestVectorDataset.indexName)
-                .withIndexUUID(indexUUID.toString())
-                .withFragmentIds(Collections.singletonList(fragments.get(0).getId()))
-                .build());
-
-        dataset.createIndex(
-            IndexOptions.builder(
-                    Collections.singletonList(TestVectorDataset.vectorColumnName),
-                    IndexType.IVF_PQ,
-                    indexParams)
-                .withIndexName(TestVectorDataset.indexName)
-                .withIndexUUID(indexUUID.toString())
-                .withFragmentIds(Collections.singletonList(fragments.get(1).getId()))
-                .build());
+        Index firstSegment =
+            dataset.createIndex(
+                IndexOptions.builder(
+                        Collections.singletonList(TestVectorDataset.vectorColumnName),
+                        IndexType.IVF_PQ,
+                        indexParams)
+                    .withIndexName(TestVectorDataset.indexName)
+                    .withFragmentIds(Collections.singletonList(fragments.get(0).getId()))
+                    .build());
+
+        Index secondSegment =
+            dataset.createIndex(
+                IndexOptions.builder(
+                        Collections.singletonList(TestVectorDataset.vectorColumnName),
+                        IndexType.IVF_PQ,
+                        indexParams)
+                    .withIndexName(TestVectorDataset.indexName)
+                    .withFragmentIds(Collections.singletonList(fragments.get(1).getId()))
+                    .build());
 
         assertFalse(
             dataset.listIndexes().contains(TestVectorDataset.indexName),
             "Partially created IVF_PQ index should not present before commit");
 
-        dataset.mergeIndexMetadata(indexUUID.toString(), IndexType.IVF_PQ, Optional.empty());
-
-        int fieldId =
-            dataset.getLanceSchema().fields().stream()
-                .filter(f -> f.getName().equals(TestVectorDataset.vectorColumnName))
-                .findAny()
-                .orElseThrow(
-                    () -> new RuntimeException("Cannot find vector field for TestVectorDataset"))
-                .getId();
-
-        long datasetVersion = dataset.version();
-
-        Index index =
-            Index.builder()
-                .uuid(indexUUID)
-                .name(TestVectorDataset.indexName)
-                .fields(Collections.singletonList(fieldId))
-                .datasetVersion(datasetVersion)
-                .indexVersion(0)
-                .fragments(
-                    fragments.stream().limit(2).map(Fragment::getId).collect(Collectors.toList()))
-                .build();
-
-        CreateIndex createIndexOp =
-            CreateIndex.builder().withNewIndices(Collections.singletonList(index)).build();
-
-        try (Transaction createIndexTx =
-            new Transaction.Builder()
-                .readVersion(dataset.version())
-                .operation(createIndexOp)
-                .build()) {
-          try (Dataset newDataset = new CommitBuilder(dataset).execute(createIndexTx)) {
-            assertEquals(datasetVersion + 1, newDataset.version());
-            assertTrue(newDataset.listIndexes().contains(TestVectorDataset.indexName));
-          }
-        }
+        List<Index> committed =
+            dataset.commitExistingIndexSegments(
+                TestVectorDataset.indexName,
+                TestVectorDataset.vectorColumnName,
+                List.of(firstSegment, secondSegment));
+        assertEquals(2, committed.size());
+        assertTrue(dataset.listIndexes().contains(TestVectorDataset.indexName));
       }
     }
   }
@@ -305,68 +234,37 @@ public void testCreateIvfSqIndexDistributively(@TempDir Path tempDir) throws Exc
         IndexParams indexParams =
             IndexParams.builder().setVectorIndexParams(vectorIndexParams).build();
 
-        UUID indexUUID = UUID.randomUUID();
-
-        dataset.createIndex(
-            IndexOptions.builder(
-                    Collections.singletonList(TestVectorDataset.vectorColumnName),
-                    IndexType.IVF_SQ,
-                    indexParams)
-                .withIndexName(TestVectorDataset.indexName)
-                .withIndexUUID(indexUUID.toString())
-                .withFragmentIds(Collections.singletonList(fragments.get(0).getId()))
-                .build());
-
-        dataset.createIndex(
-            IndexOptions.builder(
-                    Collections.singletonList(TestVectorDataset.vectorColumnName),
-                    IndexType.IVF_SQ,
-                    indexParams)
-                .withIndexName(TestVectorDataset.indexName)
-                .withIndexUUID(indexUUID.toString())
-                .withFragmentIds(Collections.singletonList(fragments.get(1).getId()))
-                .build());
+        Index firstSegment =
+            dataset.createIndex(
+                IndexOptions.builder(
+                        Collections.singletonList(TestVectorDataset.vectorColumnName),
+                        IndexType.IVF_SQ,
+                        indexParams)
+                    .withIndexName(TestVectorDataset.indexName)
+                    .withFragmentIds(Collections.singletonList(fragments.get(0).getId()))
+                    .build());
+
+        Index secondSegment =
+            dataset.createIndex(
+                IndexOptions.builder(
+                        Collections.singletonList(TestVectorDataset.vectorColumnName),
+                        IndexType.IVF_SQ,
+                        indexParams)
+                    .withIndexName(TestVectorDataset.indexName)
+                    .withFragmentIds(Collections.singletonList(fragments.get(1).getId()))
+                    .build());
 
         assertFalse(
             dataset.listIndexes().contains(TestVectorDataset.indexName),
             "Partially created IVF_SQ index should not present before commit");
 
-        dataset.mergeIndexMetadata(indexUUID.toString(), IndexType.IVF_SQ, Optional.empty());
-
-        int fieldId =
-            dataset.getLanceSchema().fields().stream()
-                .filter(f -> f.getName().equals(TestVectorDataset.vectorColumnName))
-                .findAny()
-                .orElseThrow(
-                    () -> new RuntimeException("Cannot find vector field for TestVectorDataset"))
-                .getId();
-
-        long datasetVersion = dataset.version();
-
-        Index index =
-            Index.builder()
-                .uuid(indexUUID)
-                .name(TestVectorDataset.indexName)
-                .fields(Collections.singletonList(fieldId))
-                .datasetVersion(datasetVersion)
-                .indexVersion(0)
-                .fragments(
-                    fragments.stream().limit(2).map(Fragment::getId).collect(Collectors.toList()))
-                .build();
-
-        CreateIndex createIndexOp =
-            CreateIndex.builder().withNewIndices(Collections.singletonList(index)).build();
-
-        try (Transaction createIndexTx =
-            new Transaction.Builder()
-                .readVersion(dataset.version())
-                .operation(createIndexOp)
-                .build()) {
-          try (Dataset newDataset = new CommitBuilder(dataset).execute(createIndexTx)) {
-            assertEquals(datasetVersion + 1, newDataset.version());
-            assertTrue(newDataset.listIndexes().contains(TestVectorDataset.indexName));
-          }
-        }
+        List<Index> committed =
+            dataset.commitExistingIndexSegments(
+                TestVectorDataset.indexName,
+                TestVectorDataset.vectorColumnName,
+                List.of(firstSegment, secondSegment));
+        assertEquals(2, committed.size());
+        assertTrue(dataset.listIndexes().contains(TestVectorDataset.indexName));
       }
     }
   }
diff --git a/java/src/test/java/org/lance/index/ZonemapStatsTest.java b/java/src/test/java/org/lance/index/ZonemapStatsTest.java
new file mode 100644
index 00000000000..19824cd4cfd
--- /dev/null
+++ b/java/src/test/java/org/lance/index/ZonemapStatsTest.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance.index;
+
+import org.lance.Dataset;
+import org.lance.Fragment;
+import org.lance.FragmentMetadata;
+import org.lance.FragmentOperation;
+import org.lance.WriteParams;
+import org.lance.index.scalar.ScalarIndexParams;
+import org.lance.index.scalar.ZoneStats;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/** Tests for {@link Dataset#getZonemapStats(String)} and the {@link ZoneStats} data class. */
+public class ZonemapStatsTest {
+
+  private static Schema intSchema() {
+    return new Schema(
+        Arrays.asList(
+            Field.nullable("id", new ArrowType.Int(32, true)),
+            Field.nullable("value", new ArrowType.Int(32, true))),
+        null);
+  }
+
+  /** Write a single fragment with sequential integer values. */
+  private Dataset writeIntFragment(
+      BufferAllocator allocator, String path, long version, int startValue, int rowCount) {
+    Schema schema = intSchema();
+    List<FragmentMetadata> metas;
+    try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
+      root.allocateNew();
+      IntVector idVec = (IntVector) root.getVector("id");
+      IntVector valVec = (IntVector) root.getVector("value");
+      for (int i = 0; i < rowCount; i++) {
+        idVec.setSafe(i, startValue + i);
+        valVec.setSafe(i, (startValue + i) * 10);
+      }
+      root.setRowCount(rowCount);
+      metas = Fragment.create(path, allocator, root, new WriteParams.Builder().build());
+    }
+    FragmentOperation.Append appendOp = new FragmentOperation.Append(metas);
+    return Dataset.commit(allocator, path, appendOp, Optional.of(version));
+  }
+
+  // -------------------------------------------------------
+  // ZoneStats data class tests
+  // -------------------------------------------------------
+
+  @Test
+  public void testZoneStatsGetters() {
+    ZoneStats stats = new ZoneStats(3, 100, 50, 10L, 99L, 5);
+    assertEquals(3, stats.getFragmentId());
+    assertEquals(100, stats.getZoneStart());
+    assertEquals(50, stats.getZoneLength());
+    assertEquals(10L, stats.getMin());
+    assertEquals(99L, stats.getMax());
+    assertEquals(5, stats.getNullCount());
+  }
+
+  @Test
+  public void testZoneStatsNullMinMax() {
+    ZoneStats stats = new ZoneStats(0, 0, 10, null, null, 10);
+    assertNull(stats.getMin());
+    assertNull(stats.getMax());
+    assertEquals(10, stats.getNullCount());
+  }
+
+  @Test
+  public void testZoneStatsToString() {
+    ZoneStats stats = new ZoneStats(1, 0, 100, 0L, 99L, 0);
+    String str = stats.toString();
+    assertTrue(str.contains("fragmentId=1"));
+    assertTrue(str.contains("min=0"));
+    assertTrue(str.contains("max=99"));
+  }
+
+  // -------------------------------------------------------
+  // getZonemapStats integration tests
+  // -------------------------------------------------------
+
+  @Test
+  public void testGetZonemapStatsNoIndex(@TempDir Path tempDir) {
+    String path = tempDir.resolve("no_index").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      try (Dataset dataset =
+          Dataset.create(allocator, path, intSchema(), new WriteParams.Builder().build())) {
+        List<ZoneStats> stats = dataset.getZonemapStats("id");
+        assertNotNull(stats);
+        assertTrue(stats.isEmpty());
+      }
+    }
+  }
+
+  @Test
+  public void testGetZonemapStatsNonexistentColumn(@TempDir Path tempDir) {
+    String path = tempDir.resolve("bad_col").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      try (Dataset dataset =
+          Dataset.create(allocator, path, intSchema(), new WriteParams.Builder().build())) {
+        assertThrows(IllegalArgumentException.class, () -> dataset.getZonemapStats("nonexistent"));
+      }
+    }
+  }
+
+  @Test
+  public void testGetZonemapStatsWithData(@TempDir Path tempDir) throws Exception {
+    String path = tempDir.resolve("with_data").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      try (Dataset ds =
+          Dataset.create(allocator, path, intSchema(), new WriteParams.Builder().build())) {
+        // empty
+      }
+      Dataset ds2 = writeIntFragment(allocator, path, 1, 0, 100);
+      ds2.close();
+
+      try (Dataset dataset = Dataset.open(path, allocator)) {
+        ScalarIndexParams params = ScalarIndexParams.create("zonemap", "{}");
+        IndexParams indexParams = IndexParams.builder().setScalarIndexParams(params).build();
+        dataset.createIndex(
+            Collections.singletonList("value"),
+            IndexType.ZONEMAP,
+            Optional.of("value_zm"),
+            indexParams,
+            true);
+
+        List<ZoneStats> stats = dataset.getZonemapStats("value");
+        assertNotNull(stats);
+        assertFalse(stats.isEmpty());
+
+        for (ZoneStats z : stats) {
+          assertEquals(0, z.getFragmentId());
+          assertNotNull(z.getMin());
+          assertNotNull(z.getMax());
+          assertTrue(z.getZoneLength() > 0);
+        }
+
+        ZoneStats first = stats.get(0);
+        assertTrue(
+            ((Number) first.getMin()).longValue() <= 0,
+            "First zone min should be <= 0, got: " + first.getMin());
+
+        ZoneStats last = stats.get(stats.size() - 1);
+        assertTrue(
+            ((Number) last.getMax()).longValue() >= 990,
+            "Last zone max should be >= 990, got: " + last.getMax());
+      }
+    }
+  }
+
+  @Test
+  public void testGetZonemapStatsMultiFragment(@TempDir Path tempDir) throws Exception {
+    String path = tempDir.resolve("multi_frag").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      try (Dataset ds =
+          Dataset.create(allocator, path, intSchema(), new WriteParams.Builder().build())) {
+        // empty
+      }
+      Dataset ds2 = writeIntFragment(allocator, path, 1, 0, 50);
+      ds2.close();
+      Dataset ds3 = writeIntFragment(allocator, path, 2, 50, 50);
+      ds3.close();
+
+      try (Dataset dataset = Dataset.open(path, allocator)) {
+        assertEquals(2, dataset.getFragments().size());
+
+        ScalarIndexParams params = ScalarIndexParams.create("zonemap", "{}");
+        IndexParams indexParams = IndexParams.builder().setScalarIndexParams(params).build();
+        dataset.createIndex(
+            Collections.singletonList("value"),
+            IndexType.ZONEMAP,
+            Optional.of("value_zm"),
+            indexParams,
+            true);
+
+        List<ZoneStats> stats = dataset.getZonemapStats("value");
+        assertNotNull(stats);
+        assertFalse(stats.isEmpty());
+
+        Set<Integer> fragmentIds = new HashSet<>();
+        for (ZoneStats z : stats) {
+          fragmentIds.add(z.getFragmentId());
+        }
+        assertEquals(2, fragmentIds.size(), "Expected zones from 2 fragments, got: " + fragmentIds);
+      }
+    }
+  }
+
+  @Test
+  public void testGetZonemapStatsWrongColumnReturnsEmpty(@TempDir Path tempDir) throws Exception {
+    String path = tempDir.resolve("wrong_col").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      try (Dataset ds =
+          Dataset.create(allocator, path, intSchema(), new WriteParams.Builder().build())) {
+        // empty
+      }
+      Dataset ds2 = writeIntFragment(allocator, path, 1, 0, 100);
+      ds2.close();
+
+      try (Dataset dataset = Dataset.open(path, allocator)) {
+        ScalarIndexParams params = ScalarIndexParams.create("zonemap", "{}");
+        IndexParams indexParams = IndexParams.builder().setScalarIndexParams(params).build();
+        dataset.createIndex(
+            Collections.singletonList("value"),
+            IndexType.ZONEMAP,
+            Optional.of("value_zm"),
+            indexParams,
+            true);
+
+        List<ZoneStats> stats = dataset.getZonemapStats("id");
+        assertNotNull(stats);
+        assertTrue(stats.isEmpty(), "Expected empty for non-indexed column");
+      }
+    }
+  }
+
+  @Test
+  public void testGetZonemapStatsNullArgument(@TempDir Path tempDir) {
+    String path = tempDir.resolve("null_arg").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      try (Dataset dataset =
+          Dataset.create(allocator, path, intSchema(), new WriteParams.Builder().build())) {
+        assertThrows(IllegalArgumentException.class, () -> dataset.getZonemapStats(null));
+      }
+    }
+  }
+
+  @Test
+  public void testGetZonemapStatsEmptyArgument(@TempDir Path tempDir) {
+    String path = tempDir.resolve("empty_arg").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      try (Dataset dataset =
+          Dataset.create(allocator, path, intSchema(), new WriteParams.Builder().build())) {
+        assertThrows(IllegalArgumentException.class, () -> dataset.getZonemapStats(""));
+      }
+    }
+  }
+}
diff --git a/java/src/test/java/org/lance/namespace/CustomNamespace.java b/java/src/test/java/org/lance/namespace/CustomNamespace.java
new file mode 100644
index 00000000000..47423ab25b3
--- /dev/null
+++ b/java/src/test/java/org/lance/namespace/CustomNamespace.java
@@ -0,0 +1,426 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance.namespace;
+
+import org.lance.namespace.model.AlterTableAddColumnsRequest;
+import org.lance.namespace.model.AlterTableAddColumnsResponse;
+import org.lance.namespace.model.AlterTableAlterColumnsRequest;
+import org.lance.namespace.model.AlterTableAlterColumnsResponse;
+import org.lance.namespace.model.AlterTableDropColumnsRequest;
+import org.lance.namespace.model.AlterTableDropColumnsResponse;
+import org.lance.namespace.model.AlterTransactionRequest;
+import org.lance.namespace.model.AlterTransactionResponse;
+import org.lance.namespace.model.AnalyzeTableQueryPlanRequest;
+import org.lance.namespace.model.BatchCommitTablesRequest;
+import org.lance.namespace.model.BatchCommitTablesResponse;
+import org.lance.namespace.model.BatchCreateTableVersionsRequest;
+import org.lance.namespace.model.BatchCreateTableVersionsResponse;
+import org.lance.namespace.model.BatchDeleteTableVersionsRequest;
+import org.lance.namespace.model.BatchDeleteTableVersionsResponse;
+import org.lance.namespace.model.CountTableRowsRequest;
+import org.lance.namespace.model.CreateEmptyTableRequest;
+import org.lance.namespace.model.CreateEmptyTableResponse;
+import org.lance.namespace.model.CreateNamespaceRequest;
+import org.lance.namespace.model.CreateNamespaceResponse;
+import org.lance.namespace.model.CreateTableIndexRequest;
+import org.lance.namespace.model.CreateTableIndexResponse;
+import org.lance.namespace.model.CreateTableRequest;
+import org.lance.namespace.model.CreateTableResponse;
+import org.lance.namespace.model.CreateTableScalarIndexResponse;
+import org.lance.namespace.model.CreateTableTagRequest;
+import org.lance.namespace.model.CreateTableTagResponse;
+import org.lance.namespace.model.CreateTableVersionRequest;
+import org.lance.namespace.model.CreateTableVersionResponse;
+import org.lance.namespace.model.DeclareTableRequest;
+import org.lance.namespace.model.DeclareTableResponse;
+import org.lance.namespace.model.DeleteFromTableRequest;
+import org.lance.namespace.model.DeleteFromTableResponse;
+import org.lance.namespace.model.DeleteTableTagRequest;
+import org.lance.namespace.model.DeleteTableTagResponse;
+import org.lance.namespace.model.DeregisterTableRequest;
+import org.lance.namespace.model.DeregisterTableResponse;
+import org.lance.namespace.model.DescribeNamespaceRequest;
+import org.lance.namespace.model.DescribeNamespaceResponse;
+import org.lance.namespace.model.DescribeTableIndexStatsRequest;
+import org.lance.namespace.model.DescribeTableIndexStatsResponse;
+import org.lance.namespace.model.DescribeTableRequest;
+import org.lance.namespace.model.DescribeTableResponse;
+import org.lance.namespace.model.DescribeTableVersionRequest;
+import org.lance.namespace.model.DescribeTableVersionResponse;
+import org.lance.namespace.model.DescribeTransactionRequest;
+import org.lance.namespace.model.DescribeTransactionResponse;
+import org.lance.namespace.model.DropNamespaceRequest;
+import org.lance.namespace.model.DropNamespaceResponse;
+import org.lance.namespace.model.DropTableIndexRequest;
+import org.lance.namespace.model.DropTableIndexResponse;
+import org.lance.namespace.model.DropTableRequest;
+import org.lance.namespace.model.DropTableResponse;
+import org.lance.namespace.model.ExplainTableQueryPlanRequest;
+import org.lance.namespace.model.GetTableStatsRequest;
+import org.lance.namespace.model.GetTableStatsResponse;
+import org.lance.namespace.model.GetTableTagVersionRequest;
+import org.lance.namespace.model.GetTableTagVersionResponse;
+import org.lance.namespace.model.InsertIntoTableRequest;
+import org.lance.namespace.model.InsertIntoTableResponse;
+import org.lance.namespace.model.ListNamespacesRequest;
+import org.lance.namespace.model.ListNamespacesResponse;
+import org.lance.namespace.model.ListTableIndicesRequest;
+import org.lance.namespace.model.ListTableIndicesResponse;
+import org.lance.namespace.model.ListTableTagsRequest;
+import org.lance.namespace.model.ListTableTagsResponse;
+import org.lance.namespace.model.ListTableVersionsRequest;
+import org.lance.namespace.model.ListTableVersionsResponse;
+import org.lance.namespace.model.ListTablesRequest;
+import org.lance.namespace.model.ListTablesResponse;
+import org.lance.namespace.model.MergeInsertIntoTableRequest;
+import org.lance.namespace.model.MergeInsertIntoTableResponse;
+import org.lance.namespace.model.NamespaceExistsRequest;
+import org.lance.namespace.model.QueryTableRequest;
+import org.lance.namespace.model.RegisterTableRequest;
+import org.lance.namespace.model.RegisterTableResponse;
+import org.lance.namespace.model.RenameTableRequest;
+import org.lance.namespace.model.RenameTableResponse;
+import org.lance.namespace.model.RestoreTableRequest;
+import org.lance.namespace.model.RestoreTableResponse;
+import org.lance.namespace.model.TableExistsRequest;
+import org.lance.namespace.model.UpdateTableRequest;
+import org.lance.namespace.model.UpdateTableResponse;
+import org.lance.namespace.model.UpdateTableSchemaMetadataRequest;
+import org.lance.namespace.model.UpdateTableSchemaMetadataResponse;
+import org.lance.namespace.model.UpdateTableTagRequest;
+import org.lance.namespace.model.UpdateTableTagResponse;
+
+import org.apache.arrow.memory.BufferAllocator;
+
+import java.io.Closeable;
+import java.util.Map;
+
+/**
+ * A custom namespace wrapper that delegates all operations to an inner DirectoryNamespace.
+ *
+ * <p>This class is used for testing to verify that the Java-Rust binding works correctly for custom
+ * namespace implementations that wrap DirectoryNamespace.
+ */
+public class CustomNamespace implements LanceNamespace, Closeable {
+  private final DirectoryNamespace inner;
+
+  /**
+   * Creates a CustomNamespace wrapping the given DirectoryNamespace.
+   *
+   * @param inner The DirectoryNamespace to wrap
+   */
+  public CustomNamespace(DirectoryNamespace inner) {
+    this.inner = inner;
+  }
+
+  /**
+   * Gets the wrapped DirectoryNamespace for metrics retrieval.
+   *
+   * @return The inner DirectoryNamespace
+   */
+  public DirectoryNamespace getInner() {
+    return inner;
+  }
+
+  /**
+   * Gets the native handle from the inner namespace. This is required for the JNI layer to work
+   * correctly with wrapped namespaces.
+   *
+   * @return The native handle
+   */
+  public long getNativeHandle() {
+    return inner.getNativeHandle();
+  }
+
+  @Override
+  public void initialize(Map<String, String> configProperties, BufferAllocator allocator) {
+    // Already initialized in constructor via inner namespace
+  }
+
+  @Override
+  public String namespaceId() {
+    return "CustomNamespace[" + inner.namespaceId() + "]";
+  }
+
+  // Namespace operations
+
+  @Override
+  public ListNamespacesResponse listNamespaces(ListNamespacesRequest request) {
+    return inner.listNamespaces(request);
+  }
+
+  @Override
+  public DescribeNamespaceResponse describeNamespace(DescribeNamespaceRequest request) {
+    return inner.describeNamespace(request);
+  }
+
+  @Override
+  public CreateNamespaceResponse createNamespace(CreateNamespaceRequest request) {
+    return inner.createNamespace(request);
+  }
+
+  @Override
+  public DropNamespaceResponse dropNamespace(DropNamespaceRequest request) {
+    return inner.dropNamespace(request);
+  }
+
+  @Override
+  public void namespaceExists(NamespaceExistsRequest request) {
+    inner.namespaceExists(request);
+  }
+
+  // Table operations
+
+  @Override
+  public ListTablesResponse listTables(ListTablesRequest request) {
+    return inner.listTables(request);
+  }
+
+  @Override
+  public DescribeTableResponse describeTable(DescribeTableRequest request) {
+    return inner.describeTable(request);
+  }
+
+  @Override
+  public RegisterTableResponse registerTable(RegisterTableRequest request) {
+    return inner.registerTable(request);
+  }
+
+  @Override
+  public void tableExists(TableExistsRequest request) {
+    inner.tableExists(request);
+  }
+
+  @Override
+  public DropTableResponse dropTable(DropTableRequest request) {
+    return inner.dropTable(request);
+  }
+
+  @Override
+  public DeregisterTableResponse deregisterTable(DeregisterTableRequest request) {
+    return inner.deregisterTable(request);
+  }
+
+  @Override
+  public Long countTableRows(CountTableRowsRequest request) {
+    return inner.countTableRows(request);
+  }
+
+  // Data operations
+
+  @Override
+  public CreateTableResponse createTable(CreateTableRequest request, byte[] requestData) {
+    return inner.createTable(request, requestData);
+  }
+
+  @Override
+  public DeclareTableResponse declareTable(DeclareTableRequest request) {
+    return inner.declareTable(request);
+  }
+
+  @Override
+  @SuppressWarnings("deprecation")
+  public CreateEmptyTableResponse createEmptyTable(CreateEmptyTableRequest request) {
+    return inner.createEmptyTable(request);
+  }
+
+  @Override
+  public InsertIntoTableResponse insertIntoTable(
+      InsertIntoTableRequest request, byte[] requestData) {
+    return inner.insertIntoTable(request, requestData);
+  }
+
+  @Override
+  public MergeInsertIntoTableResponse mergeInsertIntoTable(
+      MergeInsertIntoTableRequest request, byte[] requestData) {
+    return inner.mergeInsertIntoTable(request, requestData);
+  }
+
+  @Override
+  public UpdateTableResponse updateTable(UpdateTableRequest request) {
+    return inner.updateTable(request);
+  }
+
+  @Override
+  public DeleteFromTableResponse deleteFromTable(DeleteFromTableRequest request) {
+    return inner.deleteFromTable(request);
+  }
+
+  @Override
+  public byte[] queryTable(QueryTableRequest request) {
+    return inner.queryTable(request);
+  }
+
+  // Index operations
+
+  @Override
+  public CreateTableIndexResponse createTableIndex(CreateTableIndexRequest request) {
+    return inner.createTableIndex(request);
+  }
+
+  @Override
+  public CreateTableScalarIndexResponse createTableScalarIndex(CreateTableIndexRequest request) {
+    return inner.createTableScalarIndex(request);
+  }
+
+  @Override
+  public ListTableIndicesResponse listTableIndices(ListTableIndicesRequest request) {
+    return inner.listTableIndices(request);
+  }
+
+  @Override
+  public DescribeTableIndexStatsResponse describeTableIndexStats(
+      DescribeTableIndexStatsRequest request, String indexName) {
+    return inner.describeTableIndexStats(request, indexName);
+  }
+
+  @Override
+  public DropTableIndexResponse dropTableIndex(DropTableIndexRequest request, String indexName) {
+    return inner.dropTableIndex(request, indexName);
+  }
+
+  // Table version and schema operations
+
+  @Override
+  public ListTablesResponse listAllTables(ListTablesRequest request) {
+    return inner.listAllTables(request);
+  }
+
+  @Override
+  public RestoreTableResponse restoreTable(RestoreTableRequest request) {
+    return inner.restoreTable(request);
+  }
+
+  @Override
+  public RenameTableResponse renameTable(RenameTableRequest request) {
+    return inner.renameTable(request);
+  }
+
+  @Override
+  public ListTableVersionsResponse listTableVersions(ListTableVersionsRequest request) {
+    return inner.listTableVersions(request);
+  }
+
+  @Override
+  public CreateTableVersionResponse createTableVersion(CreateTableVersionRequest request) {
+    return inner.createTableVersion(request);
+  }
+
+  @Override
+  public DescribeTableVersionResponse describeTableVersion(DescribeTableVersionRequest request) {
+    return inner.describeTableVersion(request);
+  }
+
+  @Override
+  public BatchDeleteTableVersionsResponse batchDeleteTableVersions(
+      BatchDeleteTableVersionsRequest request) {
+    return inner.batchDeleteTableVersions(request);
+  }
+
+  @Override
+  public BatchCreateTableVersionsResponse batchCreateTableVersions(
+      BatchCreateTableVersionsRequest request) {
+    return inner.batchCreateTableVersions(request);
+  }
+
+  @Override
+  public BatchCommitTablesResponse batchCommitTables(BatchCommitTablesRequest request) {
+    return inner.batchCommitTables(request);
+  }
+
+  @Override
+  public UpdateTableSchemaMetadataResponse updateTableSchemaMetadata(
+      UpdateTableSchemaMetadataRequest request) {
+    return inner.updateTableSchemaMetadata(request);
+  }
+
+  @Override
+  public GetTableStatsResponse getTableStats(GetTableStatsRequest request) {
+    return inner.getTableStats(request);
+  }
+
+  // Query plan operations
+
+  @Override
+  public String explainTableQueryPlan(ExplainTableQueryPlanRequest request) {
+    return inner.explainTableQueryPlan(request);
+  }
+
+  @Override
+  public String analyzeTableQueryPlan(AnalyzeTableQueryPlanRequest request) {
+    return inner.analyzeTableQueryPlan(request);
+  }
+
+  // Column operations
+
+  @Override
+  public AlterTableAddColumnsResponse alterTableAddColumns(AlterTableAddColumnsRequest request) {
+    return inner.alterTableAddColumns(request);
+  }
+
+  @Override
+  public AlterTableAlterColumnsResponse alterTableAlterColumns(
+      AlterTableAlterColumnsRequest request) {
+    return inner.alterTableAlterColumns(request);
+  }
+
+  @Override
+  public AlterTableDropColumnsResponse alterTableDropColumns(AlterTableDropColumnsRequest request) {
+    return inner.alterTableDropColumns(request);
+  }
+
+  // Tag operations
+
+  @Override
+  public ListTableTagsResponse listTableTags(ListTableTagsRequest request) {
+    return inner.listTableTags(request);
+  }
+
+  @Override
+  public GetTableTagVersionResponse getTableTagVersion(GetTableTagVersionRequest request) {
+    return inner.getTableTagVersion(request);
+  }
+
+  @Override
+  public CreateTableTagResponse createTableTag(CreateTableTagRequest request) {
+    return inner.createTableTag(request);
+  }
+
+  @Override
+  public DeleteTableTagResponse deleteTableTag(DeleteTableTagRequest request) {
+    return inner.deleteTableTag(request);
+  }
+
+  @Override
+  public UpdateTableTagResponse updateTableTag(UpdateTableTagRequest request) {
+    return inner.updateTableTag(request);
+  }
+
+  // Transaction operations
+
+  @Override
+  public DescribeTransactionResponse describeTransaction(DescribeTransactionRequest request) {
+    return inner.describeTransaction(request);
+  }
+
+  @Override
+  public AlterTransactionResponse alterTransaction(AlterTransactionRequest request) {
+    return inner.alterTransaction(request);
+  }
+
+  @Override
+  public void close() {
+    inner.close();
+  }
+}
diff --git a/java/src/test/java/org/lance/namespace/CustomNamespaceTest.java b/java/src/test/java/org/lance/namespace/CustomNamespaceTest.java
new file mode 100644
index 00000000000..fd13747662f
--- /dev/null
+++ b/java/src/test/java/org/lance/namespace/CustomNamespaceTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance.namespace;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * Tests for CustomNamespace implementation that wraps DirectoryNamespace.
+ *
+ * <p>This test class extends DirectoryNamespaceTest to verify that all tests pass when using a
+ * CustomNamespace wrapper around DirectoryNamespace. This validates that the Java-Rust binding
+ * works correctly for custom namespace implementations.
+ */
+public class CustomNamespaceTest extends DirectoryNamespaceTest {
+
+  @Override
+  protected LanceNamespace wrapNamespace(DirectoryNamespace inner) {
+    return new CustomNamespace(inner);
+  }
+
+  @Test
+  void testCustomNamespaceId() {
+    String namespaceId = namespaceClient.namespaceId();
+    assertNotNull(namespaceId);
+    assertTrue(
+        namespaceId.startsWith("CustomNamespace["),
+        "namespaceId should start with 'CustomNamespace[', got: " + namespaceId);
+    assertTrue(
+        namespaceId.contains("DirectoryNamespace"),
+        "namespaceId should contain 'DirectoryNamespace', got: " + namespaceId);
+  }
+
+  @Test
+  void testCustomNamespaceInnerAccess() {
+    assertTrue(
+        namespaceClient instanceof CustomNamespace, "namespaceClient should be a CustomNamespace");
+    CustomNamespace customNs = (CustomNamespace) namespaceClient;
+    assertNotNull(customNs.getInner(), "inner namespace should not be null");
+    assertSame(
+        innerNamespaceClient, customNs.getInner(), "inner namespace should be the same instance");
+  }
+}
diff --git a/java/src/test/java/org/lance/namespace/DirectoryNamespaceTest.java b/java/src/test/java/org/lance/namespace/DirectoryNamespaceTest.java
index adfb2bd0022..7599fcbc6cf 100644
--- a/java/src/test/java/org/lance/namespace/DirectoryNamespaceTest.java
+++ b/java/src/test/java/org/lance/namespace/DirectoryNamespaceTest.java
@@ -13,12 +13,53 @@
  */
 package org.lance.namespace;
 
+import org.lance.CommitBuilder;
 import org.lance.Dataset;
+import org.lance.Fragment;
+import org.lance.FragmentMetadata;
 import org.lance.ReadOptions;
+import org.lance.Transaction;
 import org.lance.WriteParams;
-import org.lance.namespace.model.*;
+import org.lance.namespace.errors.ErrorCode;
+import org.lance.namespace.errors.LanceNamespaceException;
+import org.lance.namespace.model.CountTableRowsRequest;
+import org.lance.namespace.model.CreateNamespaceRequest;
+import org.lance.namespace.model.CreateNamespaceResponse;
+import org.lance.namespace.model.CreateTableIndexRequest;
+import org.lance.namespace.model.CreateTableIndexResponse;
+import org.lance.namespace.model.CreateTableRequest;
+import org.lance.namespace.model.CreateTableResponse;
+import org.lance.namespace.model.DeclareTableRequest;
+import org.lance.namespace.model.DeclareTableResponse;
+import org.lance.namespace.model.DeregisterTableRequest;
+import org.lance.namespace.model.DeregisterTableResponse;
+import org.lance.namespace.model.DescribeNamespaceRequest;
+import org.lance.namespace.model.DescribeNamespaceResponse;
+import org.lance.namespace.model.DescribeTableRequest;
+import org.lance.namespace.model.DescribeTableResponse;
 import org.lance.namespace.model.DescribeTableVersionRequest;
 import org.lance.namespace.model.DescribeTableVersionResponse;
+import org.lance.namespace.model.DropNamespaceRequest;
+import org.lance.namespace.model.DropNamespaceResponse;
+import org.lance.namespace.model.DropTableRequest;
+import org.lance.namespace.model.DropTableResponse;
+import org.lance.namespace.model.InsertIntoTableRequest;
+import org.lance.namespace.model.InsertIntoTableResponse;
+import org.lance.namespace.model.ListNamespacesRequest;
+import org.lance.namespace.model.ListNamespacesResponse;
+import org.lance.namespace.model.ListTableIndicesRequest;
+import org.lance.namespace.model.ListTableIndicesResponse;
+import org.lance.namespace.model.ListTableVersionsRequest;
+import org.lance.namespace.model.ListTableVersionsResponse;
+import org.lance.namespace.model.ListTablesRequest;
+import org.lance.namespace.model.ListTablesResponse;
+import org.lance.namespace.model.NamespaceExistsRequest;
+import org.lance.namespace.model.QueryTableRequest;
+import org.lance.namespace.model.QueryTableRequestVector;
+import org.lance.namespace.model.RegisterTableRequest;
+import org.lance.namespace.model.RegisterTableResponse;
+import org.lance.namespace.model.TableExistsRequest;
+import org.lance.operation.Append;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
@@ -55,23 +96,40 @@
 public class DirectoryNamespaceTest {
   @TempDir Path tempDir;
 
-  private BufferAllocator allocator;
-  private DirectoryNamespace namespace;
+  protected BufferAllocator allocator;
+  protected LanceNamespace namespaceClient;
+  protected DirectoryNamespace innerNamespaceClient;
 
   @BeforeEach
   void setUp() {
     allocator = new RootAllocator(Long.MAX_VALUE);
-    namespace = new DirectoryNamespace();
+    innerNamespaceClient = new DirectoryNamespace();
 
     Map<String, String> config = new HashMap<>();
     config.put("root", tempDir.toString());
-    namespace.initialize(config, allocator);
+    innerNamespaceClient.initialize(config, allocator);
+    namespaceClient = wrapNamespace(innerNamespaceClient);
+  }
+
+  /**
+   * Factory method to wrap the DirectoryNamespace. Subclasses can override this to provide a custom
+   * namespace implementation.
+   *
+   * @param inner The DirectoryNamespace to wrap
+   * @return The namespace client to use in tests (may be the same as inner or a wrapper)
+   */
+  protected LanceNamespace wrapNamespace(DirectoryNamespace inner) {
+    return inner;
   }
 
   @AfterEach
   void tearDown() {
-    if (namespace != null) {
-      namespace.close();
+    if (namespaceClient != null && namespaceClient instanceof java.io.Closeable) {
+      try {
+        ((java.io.Closeable) namespaceClient).close();
+      } catch (Exception e) {
+        // Ignore
+      }
     }
     if (allocator != null) {
       allocator.close();
@@ -124,21 +182,23 @@ private byte[] createTestTableData() throws Exception {
 
   @Test
   void testNamespaceId() {
-    String namespaceId = namespace.namespaceId();
+    String namespaceId = namespaceClient.namespaceId();
     assertNotNull(namespaceId);
-    assertTrue(namespaceId.contains("DirectoryNamespace"));
+    assertTrue(
+        namespaceId.contains("DirectoryNamespace"),
+        "namespaceId should contain 'DirectoryNamespace', got: " + namespaceId);
   }
 
   @Test
   void testCreateAndListNamespaces() {
     // Create a namespace
     CreateNamespaceRequest createReq = new CreateNamespaceRequest().id(Arrays.asList("workspace"));
-    CreateNamespaceResponse createResp = namespace.createNamespace(createReq);
+    CreateNamespaceResponse createResp = namespaceClient.createNamespace(createReq);
     assertNotNull(createResp);
 
     // List namespaces
     ListNamespacesRequest listReq = new ListNamespacesRequest();
-    ListNamespacesResponse listResp = namespace.listNamespaces(listReq);
+    ListNamespacesResponse listResp = namespaceClient.listNamespaces(listReq);
     assertNotNull(listResp);
     assertNotNull(listResp.getNamespaces());
     assertTrue(listResp.getNamespaces().contains("workspace"));
@@ -148,12 +208,12 @@ void testCreateAndListNamespaces() {
   void testDescribeNamespace() {
     // Create a namespace
     CreateNamespaceRequest createReq = new CreateNamespaceRequest().id(Arrays.asList("workspace"));
-    namespace.createNamespace(createReq);
+    namespaceClient.createNamespace(createReq);
 
     // Describe namespace
     DescribeNamespaceRequest descReq =
         new DescribeNamespaceRequest().id(Arrays.asList("workspace"));
-    DescribeNamespaceResponse descResp = namespace.describeNamespace(descReq);
+    DescribeNamespaceResponse descResp = namespaceClient.describeNamespace(descReq);
     assertNotNull(descResp);
     assertNotNull(descResp.getProperties());
   }
@@ -162,32 +222,35 @@ void testDescribeNamespace() {
   void testNamespaceExists() {
     // Create a namespace
     CreateNamespaceRequest createReq = new CreateNamespaceRequest().id(Arrays.asList("workspace"));
-    namespace.createNamespace(createReq);
+    namespaceClient.createNamespace(createReq);
 
     // Check existence
     NamespaceExistsRequest existsReq = new NamespaceExistsRequest().id(Arrays.asList("workspace"));
-    assertDoesNotThrow(() -> namespace.namespaceExists(existsReq));
+    assertDoesNotThrow(() -> namespaceClient.namespaceExists(existsReq));
 
     // Check non-existent namespace
     NamespaceExistsRequest notExistsReq =
         new NamespaceExistsRequest().id(Arrays.asList("nonexistent"));
-    assertThrows(RuntimeException.class, () -> namespace.namespaceExists(notExistsReq));
+    LanceNamespaceException ex =
+        assertThrows(
+            LanceNamespaceException.class, () -> namespaceClient.namespaceExists(notExistsReq));
+    assertEquals(ErrorCode.NAMESPACE_NOT_FOUND, ex.getErrorCode());
   }
 
   @Test
   void testDropNamespace() {
     // Create a namespace
     CreateNamespaceRequest createReq = new CreateNamespaceRequest().id(Arrays.asList("workspace"));
-    namespace.createNamespace(createReq);
+    namespaceClient.createNamespace(createReq);
 
     // Drop namespace
     DropNamespaceRequest dropReq = new DropNamespaceRequest().id(Arrays.asList("workspace"));
-    DropNamespaceResponse dropResp = namespace.dropNamespace(dropReq);
+    DropNamespaceResponse dropResp = namespaceClient.dropNamespace(dropReq);
     assertNotNull(dropResp);
 
     // Verify it's gone
     NamespaceExistsRequest existsReq = new NamespaceExistsRequest().id(Arrays.asList("workspace"));
-    assertThrows(RuntimeException.class, () -> namespace.namespaceExists(existsReq));
+    assertThrows(LanceNamespaceException.class, () -> namespaceClient.namespaceExists(existsReq));
   }
 
   @Test
@@ -195,13 +258,13 @@ void testCreateTable() throws Exception {
     // Create parent namespace
     CreateNamespaceRequest createNsReq =
         new CreateNamespaceRequest().id(Arrays.asList("workspace"));
-    namespace.createNamespace(createNsReq);
+    namespaceClient.createNamespace(createNsReq);
 
     // Create table with data
     byte[] tableData = createTestTableData();
     CreateTableRequest createReq =
         new CreateTableRequest().id(Arrays.asList("workspace", "test_table"));
-    CreateTableResponse createResp = namespace.createTable(createReq, tableData);
+    CreateTableResponse createResp = namespaceClient.createTable(createReq, tableData);
 
     assertNotNull(createResp);
     assertNotNull(createResp.getLocation());
@@ -214,17 +277,17 @@ void testListTables() throws Exception {
     // Create parent namespace
     CreateNamespaceRequest createNsReq =
         new CreateNamespaceRequest().id(Arrays.asList("workspace"));
-    namespace.createNamespace(createNsReq);
+    namespaceClient.createNamespace(createNsReq);
 
     // Create a table
     byte[] tableData = createTestTableData();
     CreateTableRequest createReq =
         new CreateTableRequest().id(Arrays.asList("workspace", "test_table"));
-    namespace.createTable(createReq, tableData);
+    namespaceClient.createTable(createReq, tableData);
 
     // List tables
     ListTablesRequest listReq = new ListTablesRequest().id(Arrays.asList("workspace"));
-    ListTablesResponse listResp = namespace.listTables(listReq);
+    ListTablesResponse listResp = namespaceClient.listTables(listReq);
 
     assertNotNull(listResp);
     assertNotNull(listResp.getTables());
@@ -236,18 +299,18 @@ void testDescribeTable() throws Exception {
     // Create parent namespace
     CreateNamespaceRequest createNsReq =
         new CreateNamespaceRequest().id(Arrays.asList("workspace"));
-    namespace.createNamespace(createNsReq);
+    namespaceClient.createNamespace(createNsReq);
 
     // Create a table
     byte[] tableData = createTestTableData();
     CreateTableRequest createReq =
         new CreateTableRequest().id(Arrays.asList("workspace", "test_table"));
-    namespace.createTable(createReq, tableData);
+    namespaceClient.createTable(createReq, tableData);
 
     // Describe table
     DescribeTableRequest descReq =
         new DescribeTableRequest().id(Arrays.asList("workspace", "test_table"));
-    DescribeTableResponse descResp = namespace.describeTable(descReq);
+    DescribeTableResponse descResp = namespaceClient.describeTable(descReq);
 
     assertNotNull(descResp);
     assertNotNull(descResp.getLocation());
@@ -259,23 +322,23 @@ void testTableExists() throws Exception {
     // Create parent namespace
     CreateNamespaceRequest createNsReq =
         new CreateNamespaceRequest().id(Arrays.asList("workspace"));
-    namespace.createNamespace(createNsReq);
+    namespaceClient.createNamespace(createNsReq);
 
     // Create a table
     byte[] tableData = createTestTableData();
     CreateTableRequest createReq =
         new CreateTableRequest().id(Arrays.asList("workspace", "test_table"));
-    namespace.createTable(createReq, tableData);
+    namespaceClient.createTable(createReq, tableData);
 
     // Check existence
     TableExistsRequest existsReq =
         new TableExistsRequest().id(Arrays.asList("workspace", "test_table"));
-    assertDoesNotThrow(() -> namespace.tableExists(existsReq));
+    assertDoesNotThrow(() -> namespaceClient.tableExists(existsReq));
 
     // Check non-existent table
     TableExistsRequest notExistsReq =
         new TableExistsRequest().id(Arrays.asList("workspace", "nonexistent"));
-    assertThrows(RuntimeException.class, () -> namespace.tableExists(notExistsReq));
+    assertThrows(LanceNamespaceException.class, () -> namespaceClient.tableExists(notExistsReq));
   }
 
   @Test
@@ -283,23 +346,23 @@ void testDropTable() throws Exception {
     // Create parent namespace
     CreateNamespaceRequest createNsReq =
         new CreateNamespaceRequest().id(Arrays.asList("workspace"));
-    namespace.createNamespace(createNsReq);
+    namespaceClient.createNamespace(createNsReq);
 
     // Create a table
     byte[] tableData = createTestTableData();
     CreateTableRequest createReq =
         new CreateTableRequest().id(Arrays.asList("workspace", "test_table"));
-    namespace.createTable(createReq, tableData);
+    namespaceClient.createTable(createReq, tableData);
 
     // Drop table
     DropTableRequest dropReq = new DropTableRequest().id(Arrays.asList("workspace", "test_table"));
-    DropTableResponse dropResp = namespace.dropTable(dropReq);
+    DropTableResponse dropResp = namespaceClient.dropTable(dropReq);
     assertNotNull(dropResp);
 
     // Verify it's gone
     TableExistsRequest existsReq =
         new TableExistsRequest().id(Arrays.asList("workspace", "test_table"));
-    assertThrows(RuntimeException.class, () -> namespace.tableExists(existsReq));
+    assertThrows(LanceNamespaceException.class, () -> namespaceClient.tableExists(existsReq));
   }
 
   @Test
@@ -377,99 +440,47 @@ void testDescribeTableVersion() throws Exception {
   }
 
   /**
-   * Inner class that wraps DirectoryNamespace and tracks API calls for testing managed versioning.
+   * Creates a DirectoryNamespace configured for testing managed versioning with ops metrics.
+   *
+   * @param root The root path for the namespace
+   * @return A DirectoryNamespace with table_version_tracking_enabled and ops_metrics_enabled
    */
-  static class TableVersionTrackingNamespace implements LanceNamespace, java.io.Closeable {
-    private final DirectoryNamespace inner;
-    private final AtomicInteger createTableVersionCount = new AtomicInteger(0);
-    private final AtomicInteger describeTableVersionCount = new AtomicInteger(0);
-    private final AtomicInteger listTableVersionsCount = new AtomicInteger(0);
-
-    public TableVersionTrackingNamespace(Path root) {
-      Map<String, String> dirProps = new HashMap<>();
-      dirProps.put("root", root.toString());
-      dirProps.put("table_version_tracking_enabled", "true");
-      dirProps.put("manifest_enabled", "true");
-
-      this.inner = new DirectoryNamespace();
-      try (BufferAllocator allocator = new RootAllocator()) {
-        this.inner.initialize(dirProps, allocator);
-      }
-    }
-
-    public int getCreateTableVersionCount() {
-      return createTableVersionCount.get();
-    }
-
-    public int getDescribeTableVersionCount() {
-      return describeTableVersionCount.get();
-    }
-
-    public int getListTableVersionsCount() {
-      return listTableVersionsCount.get();
-    }
-
-    public long getNativeHandle() {
-      return inner.getNativeHandle();
-    }
-
-    @Override
-    public void initialize(Map<String, String> configProperties, BufferAllocator allocator) {
-      // Already initialized in constructor
-    }
-
-    @Override
-    public String namespaceId() {
-      return "TableVersionTrackingNamespace { inner: " + inner.namespaceId() + " }";
-    }
-
-    @Override
-    public DeclareTableResponse declareTable(DeclareTableRequest request) {
-      return inner.declareTable(request);
-    }
-
-    @Override
-    public DescribeTableResponse describeTable(DescribeTableRequest request) {
-      return inner.describeTable(request);
-    }
-
-    @Override
-    public CreateTableVersionResponse createTableVersion(CreateTableVersionRequest request) {
-      createTableVersionCount.incrementAndGet();
-      return inner.createTableVersion(request);
-    }
-
-    @Override
-    public DescribeTableVersionResponse describeTableVersion(DescribeTableVersionRequest request) {
-      describeTableVersionCount.incrementAndGet();
-      return inner.describeTableVersion(request);
+  private DirectoryNamespace createManagedVersioningNamespace(Path root) {
+    Map<String, String> dirProps = new HashMap<>();
+    dirProps.put("root", root.toString());
+    dirProps.put("table_version_tracking_enabled", "true");
+    dirProps.put("manifest_enabled", "true");
+    dirProps.put("ops_metrics_enabled", "true");
+
+    DirectoryNamespace ns = new DirectoryNamespace();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      ns.initialize(dirProps, allocator);
     }
+    return ns;
+  }
 
-    @Override
-    public ListTableVersionsResponse listTableVersions(ListTableVersionsRequest request) {
-      listTableVersionsCount.incrementAndGet();
-      return inner.listTableVersions(request);
-    }
+  private static int getCreateTableVersionCount(DirectoryNamespace ns) {
+    Map<String, Long> metrics = ns.retrieveOpsMetrics();
+    return metrics.getOrDefault("create_table_version", 0L).intValue();
+  }
 
-    @Override
-    public BatchDeleteTableVersionsResponse batchDeleteTableVersions(
-        BatchDeleteTableVersionsRequest request) {
-      return inner.batchDeleteTableVersions(request);
-    }
+  private static int getDescribeTableVersionCount(DirectoryNamespace ns) {
+    Map<String, Long> metrics = ns.retrieveOpsMetrics();
+    return metrics.getOrDefault("describe_table_version", 0L).intValue();
+  }
 
-    @Override
-    public void close() {
-      inner.close();
-    }
+  private static int getListTableVersionsCount(DirectoryNamespace ns) {
+    Map<String, Long> metrics = ns.retrieveOpsMetrics();
+    return metrics.getOrDefault("list_table_versions", 0L).intValue();
   }
 
   @Test
   void testExternalManifestStoreInvokesNamespaceApis(@TempDir Path managedVersioningTempDir)
       throws Exception {
     try (BufferAllocator allocator = new RootAllocator()) {
-      // Create namespace with table_version_tracking_enabled
-      TableVersionTrackingNamespace namespace =
-          new TableVersionTrackingNamespace(managedVersioningTempDir);
+      // Create namespace with table_version_tracking_enabled and ops_metrics_enabled
+      DirectoryNamespace namespaceClient =
+          createManagedVersioningNamespace(managedVersioningTempDir);
       String tableName = "test_table";
       java.util.List<String> tableId = Arrays.asList(tableName);
 
@@ -533,7 +544,7 @@ public VectorSchemaRoot getVectorSchemaRoot() {
             Dataset.write()
                 .allocator(allocator)
                 .reader(testReader)
-                .namespace(namespace)
+                .namespaceClient(namespaceClient)
                 .tableId(tableId)
                 .mode(WriteParams.WriteMode.CREATE)
                 .execute()) {
@@ -545,7 +556,7 @@ public VectorSchemaRoot getVectorSchemaRoot() {
       // Verify describe_table returns managed_versioning=true
       DescribeTableRequest descReq = new DescribeTableRequest();
       descReq.setId(tableId);
-      DescribeTableResponse descResp = namespace.describeTable(descReq);
+      DescribeTableResponse descResp = namespaceClient.describeTable(descReq);
 
       assertEquals(
           Boolean.TRUE,
@@ -553,22 +564,26 @@ public VectorSchemaRoot getVectorSchemaRoot() {
           "Expected managedVersioning=true when table_version_tracking_enabled");
 
       // Open dataset through namespace - this should call list_table_versions for latest
-      int initialListCount = namespace.getListTableVersionsCount();
+      int initialListCount = getListTableVersionsCount(namespaceClient);
       try (Dataset dsFromNamespace =
-          Dataset.open().allocator(allocator).namespace(namespace).tableId(tableId).build()) {
+          Dataset.open()
+              .allocator(allocator)
+              .namespaceClient(namespaceClient)
+              .tableId(tableId)
+              .build()) {
 
         assertEquals(2, dsFromNamespace.countRows());
         assertEquals(1, dsFromNamespace.version());
       }
       assertEquals(
           initialListCount + 1,
-          namespace.getListTableVersionsCount(),
+          getListTableVersionsCount(namespaceClient),
           "list_table_versions should have been called once when opening latest version");
 
       // Verify create_table_version was called once during CREATE
       assertEquals(
           1,
-          namespace.getCreateTableVersionCount(),
+          getCreateTableVersionCount(namespaceClient),
           "create_table_version should have been called once during CREATE");
 
       try (VectorSchemaRoot appendRoot = VectorSchemaRoot.create(schema, allocator)) {
@@ -624,7 +639,7 @@ public VectorSchemaRoot getVectorSchemaRoot() {
             Dataset.write()
                 .allocator(allocator)
                 .reader(appendReader)
-                .namespace(namespace)
+                .namespaceClient(namespaceClient)
                 .tableId(tableId)
                 .mode(WriteParams.WriteMode.APPEND)
                 .execute()) {
@@ -635,28 +650,32 @@ public VectorSchemaRoot getVectorSchemaRoot() {
 
       assertEquals(
           2,
-          namespace.getCreateTableVersionCount(),
+          getCreateTableVersionCount(namespaceClient),
           "create_table_version should have been called twice (once for CREATE, once for APPEND)");
 
       // Open latest version - should call list_table_versions
-      int listCountBeforeLatest = namespace.getListTableVersionsCount();
+      int listCountBeforeLatest = getListTableVersionsCount(namespaceClient);
       try (Dataset latestDs =
-          Dataset.open().allocator(allocator).namespace(namespace).tableId(tableId).build()) {
+          Dataset.open()
+              .allocator(allocator)
+              .namespaceClient(namespaceClient)
+              .tableId(tableId)
+              .build()) {
 
         assertEquals(4, latestDs.countRows());
         assertEquals(2, latestDs.version());
       }
       assertEquals(
           listCountBeforeLatest + 1,
-          namespace.getListTableVersionsCount(),
+          getListTableVersionsCount(namespaceClient),
           "list_table_versions should have been called once when opening latest version");
 
       // Open specific version (version 1) - should call describe_table_version
-      int describeCountBeforeV1 = namespace.getDescribeTableVersionCount();
+      int describeCountBeforeV1 = getDescribeTableVersionCount(namespaceClient);
       try (Dataset v1Ds =
           Dataset.open()
               .allocator(allocator)
-              .namespace(namespace)
+              .namespaceClient(namespaceClient)
               .tableId(tableId)
               .readOptions(new ReadOptions.Builder().setVersion(1L).build())
               .build()) {
@@ -666,10 +685,160 @@ public VectorSchemaRoot getVectorSchemaRoot() {
       }
       assertEquals(
           describeCountBeforeV1 + 1,
-          namespace.getDescribeTableVersionCount(),
+          getDescribeTableVersionCount(namespaceClient),
           "describe_table_version should have been called once when opening version 1");
 
-      namespace.close();
+      namespaceClient.close();
+    }
+  }
+
+  @Test
+  void testDatasetBasedCommitBuilderWithNamespace(@TempDir Path managedVersioningTempDir)
+      throws Exception {
+    try (BufferAllocator allocator = new RootAllocator()) {
+      DirectoryNamespace namespaceClient =
+          createManagedVersioningNamespace(managedVersioningTempDir);
+      String tableName = "test_table";
+      List<String> tableId = Arrays.asList(tableName);
+
+      Schema schema =
+          new Schema(
+              Arrays.asList(
+                  new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null),
+                  new Field("name", FieldType.nullable(new ArrowType.Utf8()), null)));
+
+      // Create initial dataset through namespace using WriteDatasetBuilder
+      try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
+        IntVector idVector = (IntVector) root.getVector("id");
+        VarCharVector nameVector = (VarCharVector) root.getVector("name");
+
+        idVector.allocateNew(2);
+        nameVector.allocateNew(2);
+        idVector.set(0, 1);
+        idVector.set(1, 2);
+        nameVector.set(0, "Alice".getBytes());
+        nameVector.set(1, "Bob".getBytes());
+        idVector.setValueCount(2);
+        nameVector.setValueCount(2);
+        root.setRowCount(2);
+
+        ArrowReader reader =
+            new ArrowReader(allocator) {
+              boolean firstRead = true;
+
+              @Override
+              public boolean loadNextBatch() {
+                if (firstRead) {
+                  firstRead = false;
+                  return true;
+                }
+                return false;
+              }
+
+              @Override
+              public long bytesRead() {
+                return 0;
+              }
+
+              @Override
+              protected void closeReadSource() {}
+
+              @Override
+              protected Schema readSchema() {
+                return schema;
+              }
+
+              @Override
+              public VectorSchemaRoot getVectorSchemaRoot() {
+                return root;
+              }
+            };
+
+        try (Dataset dataset =
+            Dataset.write()
+                .allocator(allocator)
+                .reader(reader)
+                .namespaceClient(namespaceClient)
+                .tableId(tableId)
+                .mode(WriteParams.WriteMode.CREATE)
+                .execute()) {
+          assertEquals(2, dataset.countRows());
+          assertEquals(1, dataset.version());
+        }
+      }
+
+      // Verify initial create used createTableVersion once
+      assertEquals(
+          1,
+          getCreateTableVersionCount(namespaceClient),
+          "create_table_version should be called once during CREATE");
+
+      // Open dataset through namespace (returns dataset with managed versioning)
+      Dataset existingDataset =
+          Dataset.open()
+              .allocator(allocator)
+              .namespaceClient(namespaceClient)
+              .tableId(tableId)
+              .build();
+
+      // Get the dataset URI for Fragment.create()
+      String datasetUri = existingDataset.uri();
+
+      // Create a new fragment independently (simulating Spark worker behavior)
+      List<FragmentMetadata> fragments;
+      try (VectorSchemaRoot appendRoot = VectorSchemaRoot.create(schema, allocator)) {
+        IntVector idVector = (IntVector) appendRoot.getVector("id");
+        VarCharVector nameVector = (VarCharVector) appendRoot.getVector("name");
+
+        idVector.allocateNew(2);
+        nameVector.allocateNew(2);
+        idVector.set(0, 3);
+        idVector.set(1, 4);
+        nameVector.set(0, "Charlie".getBytes());
+        nameVector.set(1, "Diana".getBytes());
+        idVector.setValueCount(2);
+        nameVector.setValueCount(2);
+        appendRoot.setRowCount(2);
+
+        fragments =
+            Fragment.create(datasetUri, allocator, appendRoot, new WriteParams.Builder().build());
+      }
+
+      // Commit using dataset-based CommitBuilder WITH namespace (the new path)
+      int createCountBefore = getCreateTableVersionCount(namespaceClient);
+      try (Transaction txn =
+              new Transaction.Builder()
+                  .readVersion(existingDataset.version())
+                  .operation(Append.builder().fragments(fragments).build())
+                  .build();
+          Dataset committed =
+              new CommitBuilder(existingDataset)
+                  .namespaceClient(namespaceClient)
+                  .tableId(tableId)
+                  .execute(txn)) {
+        assertEquals(2, committed.version());
+        assertEquals(4, committed.countRows());
+      }
+
+      // Verify createTableVersion was called for the dataset-based commit
+      assertEquals(
+          createCountBefore + 1,
+          getCreateTableVersionCount(namespaceClient),
+          "create_table_version should be called for dataset-based CommitBuilder with namespace");
+
+      // Verify the data is accessible through namespace
+      try (Dataset latestDs =
+          Dataset.open()
+              .allocator(allocator)
+              .namespaceClient(namespaceClient)
+              .tableId(tableId)
+              .build()) {
+        assertEquals(4, latestDs.countRows());
+        assertEquals(2, latestDs.version());
+      }
+
+      existingDataset.close();
+      namespaceClient.close();
     }
   }
 
@@ -678,7 +847,7 @@ void testConcurrentCreateAndDropWithSingleInstance() throws Exception {
     // Initialize namespace first - create parent namespace to ensure __manifest table
     // is created before concurrent operations
     CreateNamespaceRequest createNsReq = new CreateNamespaceRequest().id(Arrays.asList("test_ns"));
-    namespace.createNamespace(createNsReq);
+    namespaceClient.createNamespace(createNsReq);
 
     int numTables = 10;
     ExecutorService executor = Executors.newFixedThreadPool(numTables);
@@ -699,11 +868,11 @@ void testConcurrentCreateAndDropWithSingleInstance() throws Exception {
 
               CreateTableRequest createReq =
                   new CreateTableRequest().id(Arrays.asList("test_ns", tableName));
-              namespace.createTable(createReq, tableData);
+              namespaceClient.createTable(createReq, tableData);
 
               DropTableRequest dropReq =
                   new DropTableRequest().id(Arrays.asList("test_ns", tableName));
-              namespace.dropTable(dropReq);
+              namespaceClient.dropTable(dropReq);
 
               successCount.incrementAndGet();
             } catch (Exception e) {
@@ -724,7 +893,7 @@ void testConcurrentCreateAndDropWithSingleInstance() throws Exception {
     assertEquals(0, failCount.get(), "No tasks should fail");
 
     ListTablesRequest listReq = new ListTablesRequest().id(Arrays.asList("test_ns"));
-    ListTablesResponse listResp = namespace.listTables(listReq);
+    ListTablesResponse listResp = namespaceClient.listTables(listReq);
     assertEquals(0, listResp.getTables().size(), "All tables should be dropped");
   }
 
@@ -948,4 +1117,314 @@ void testConcurrentCreateThenDropFromDifferentInstance() throws Exception {
     assertEquals(numTables, dropSuccessCount.get(), "All drops should succeed");
     assertEquals(0, dropFailCount.get(), "No drops should fail");
   }
+
+  @Test
+  void testCountTableRows() throws Exception {
+    // Create parent namespace
+    CreateNamespaceRequest createNsReq =
+        new CreateNamespaceRequest().id(Arrays.asList("workspace"));
+    namespaceClient.createNamespace(createNsReq);
+
+    // Create a table with 3 rows
+    byte[] tableData = createTestTableData();
+    CreateTableRequest createReq =
+        new CreateTableRequest().id(Arrays.asList("workspace", "test_table"));
+    namespaceClient.createTable(createReq, tableData);
+
+    // Count rows
+    CountTableRowsRequest countReq =
+        new CountTableRowsRequest().id(Arrays.asList("workspace", "test_table"));
+    long count = namespaceClient.countTableRows(countReq);
+    assertEquals(3, count);
+  }
+
+  @Test
+  void testCountTableRowsWithFilter() throws Exception {
+    // Create parent namespace
+    CreateNamespaceRequest createNsReq =
+        new CreateNamespaceRequest().id(Arrays.asList("workspace"));
+    namespaceClient.createNamespace(createNsReq);
+
+    // Create a table with 3 rows (ages: 30, 25, 35)
+    byte[] tableData = createTestTableData();
+    CreateTableRequest createReq =
+        new CreateTableRequest().id(Arrays.asList("workspace", "test_table"));
+    namespaceClient.createTable(createReq, tableData);
+
+    // Count rows with predicate
+    CountTableRowsRequest countReq =
+        new CountTableRowsRequest()
+            .id(Arrays.asList("workspace", "test_table"))
+            .predicate("age > 28");
+    long count = namespaceClient.countTableRows(countReq);
+    assertEquals(2, count); // Alice (30) and Charlie (35)
+  }
+
+  @Test
+  void testInsertIntoTable() throws Exception {
+    // Create parent namespace
+    CreateNamespaceRequest createNsReq =
+        new CreateNamespaceRequest().id(Arrays.asList("workspace"));
+    namespaceClient.createNamespace(createNsReq);
+
+    // Create a table with 3 rows
+    byte[] tableData = createTestTableData();
+    CreateTableRequest createReq =
+        new CreateTableRequest().id(Arrays.asList("workspace", "test_table"));
+    namespaceClient.createTable(createReq, tableData);
+
+    // Insert more data
+    byte[] newData = createTestTableData(); // Another 3 rows
+    InsertIntoTableRequest insertReq =
+        new InsertIntoTableRequest().id(Arrays.asList("workspace", "test_table")).mode("append");
+    InsertIntoTableResponse insertResp = namespaceClient.insertIntoTable(insertReq, newData);
+    assertNotNull(insertResp);
+
+    // Verify row count increased
+    CountTableRowsRequest countReq =
+        new CountTableRowsRequest().id(Arrays.asList("workspace", "test_table"));
+    long count = namespaceClient.countTableRows(countReq);
+    assertEquals(6, count);
+  }
+
+  @Test
+  void testQueryTable() throws Exception {
+    // Create parent namespace
+    CreateNamespaceRequest createNsReq =
+        new CreateNamespaceRequest().id(Arrays.asList("workspace"));
+    namespaceClient.createNamespace(createNsReq);
+
+    // Create a table with 3 rows
+    byte[] tableData = createTestTableData();
+    CreateTableRequest createReq =
+        new CreateTableRequest().id(Arrays.asList("workspace", "test_table"));
+    namespaceClient.createTable(createReq, tableData);
+
+    // Query table - just verify we can execute a simple query
+    QueryTableRequest queryReq =
+        new QueryTableRequest()
+            .id(Arrays.asList("workspace", "test_table"))
+            .k(10)
+            .vector(new QueryTableRequestVector());
+    byte[] resultBytes = namespaceClient.queryTable(queryReq);
+    assertNotNull(resultBytes);
+    assertTrue(resultBytes.length > 0);
+  }
+
+  @Test
+  void testListTableVersions() throws Exception {
+    // Create parent namespace
+    CreateNamespaceRequest createNsReq =
+        new CreateNamespaceRequest().id(Arrays.asList("workspace"));
+    namespaceClient.createNamespace(createNsReq);
+
+    // Create a table
+    byte[] tableData = createTestTableData();
+    CreateTableRequest createReq =
+        new CreateTableRequest().id(Arrays.asList("workspace", "test_table"));
+    namespaceClient.createTable(createReq, tableData);
+
+    // List versions
+    ListTableVersionsRequest listReq =
+        new ListTableVersionsRequest().id(Arrays.asList("workspace", "test_table"));
+    ListTableVersionsResponse listResp = namespaceClient.listTableVersions(listReq);
+    assertNotNull(listResp);
+    assertNotNull(listResp.getVersions());
+    assertTrue(listResp.getVersions().size() >= 1);
+  }
+
+  @Test
+  void testRegisterTable() throws Exception {
+    // Create parent namespace
+    CreateNamespaceRequest createNsReq =
+        new CreateNamespaceRequest().id(Arrays.asList("workspace"));
+    namespaceClient.createNamespace(createNsReq);
+
+    // First create a table directly to get a valid location
+    byte[] tableData = createTestTableData();
+    CreateTableRequest createReq =
+        new CreateTableRequest().id(Arrays.asList("workspace", "source_table"));
+    CreateTableResponse createResp = namespaceClient.createTable(createReq, tableData);
+    String location = createResp.getLocation();
+
+    // Register the table under a new name (using relative path)
+    String relativePath = "source_table.lance";
+    RegisterTableRequest registerReq =
+        new RegisterTableRequest()
+            .id(Arrays.asList("workspace", "registered_table"))
+            .location(relativePath);
+    RegisterTableResponse registerResp = namespaceClient.registerTable(registerReq);
+    assertNotNull(registerResp);
+
+    // Verify the registered table exists
+    TableExistsRequest existsReq =
+        new TableExistsRequest().id(Arrays.asList("workspace", "registered_table"));
+    assertDoesNotThrow(() -> namespaceClient.tableExists(existsReq));
+  }
+
+  @Test
+  void testDeclareTable() throws Exception {
+    // Create parent namespace
+    CreateNamespaceRequest createNsReq =
+        new CreateNamespaceRequest().id(Arrays.asList("workspace"));
+    namespaceClient.createNamespace(createNsReq);
+
+    // Declare a table
+    DeclareTableRequest declareReq =
+        new DeclareTableRequest().id(Arrays.asList("workspace", "declared_table"));
+    DeclareTableResponse declareResp = namespaceClient.declareTable(declareReq);
+    assertNotNull(declareResp);
+    assertNotNull(declareResp.getLocation());
+  }
+
+  @Test
+  void testDeregisterTable() throws Exception {
+    // Create parent namespace
+    CreateNamespaceRequest createNsReq =
+        new CreateNamespaceRequest().id(Arrays.asList("workspace"));
+    namespaceClient.createNamespace(createNsReq);
+
+    // Create a table
+    byte[] tableData = createTestTableData();
+    CreateTableRequest createReq =
+        new CreateTableRequest().id(Arrays.asList("workspace", "test_table"));
+    namespaceClient.createTable(createReq, tableData);
+
+    // Deregister the table
+    DeregisterTableRequest deregisterReq =
+        new DeregisterTableRequest().id(Arrays.asList("workspace", "test_table"));
+    DeregisterTableResponse deregisterResp = namespaceClient.deregisterTable(deregisterReq);
+    assertNotNull(deregisterResp);
+
+    // Verify the table no longer exists
+    TableExistsRequest existsReq =
+        new TableExistsRequest().id(Arrays.asList("workspace", "test_table"));
+    assertThrows(LanceNamespaceException.class, () -> namespaceClient.tableExists(existsReq));
+  }
+
+  @Test
+  void testListTableIndices() throws Exception {
+    // Create parent namespace
+    CreateNamespaceRequest createNsReq =
+        new CreateNamespaceRequest().id(Arrays.asList("workspace"));
+    namespaceClient.createNamespace(createNsReq);
+
+    // Create a table
+    byte[] tableData = createTestTableData();
+    CreateTableRequest createReq =
+        new CreateTableRequest().id(Arrays.asList("workspace", "test_table"));
+    namespaceClient.createTable(createReq, tableData);
+
+    // List indices (should be empty initially)
+    ListTableIndicesRequest listReq =
+        new ListTableIndicesRequest().id(Arrays.asList("workspace", "test_table"));
+    ListTableIndicesResponse listResp = namespaceClient.listTableIndices(listReq);
+    assertNotNull(listResp);
+    assertNotNull(listResp.getIndexes());
+    assertEquals(0, listResp.getIndexes().size());
+  }
+
+  @Test
+  void testCreateScalarIndex() throws Exception {
+    // Create table at root level
+    byte[] tableData = createTestTableData();
+    CreateTableRequest createReq = new CreateTableRequest().id(Arrays.asList("test_table"));
+    namespaceClient.createTable(createReq, tableData);
+
+    // Create scalar index on 'id' column
+    CreateTableIndexRequest createIndexReq =
+        new CreateTableIndexRequest()
+            .id(Arrays.asList("test_table"))
+            .column("id")
+            .indexType("BTREE")
+            .name("id_idx");
+    CreateTableIndexResponse response = namespaceClient.createTableIndex(createIndexReq);
+    assertNotNull(response);
+
+    // List indices to verify
+    ListTableIndicesRequest listReq = new ListTableIndicesRequest().id(Arrays.asList("test_table"));
+    ListTableIndicesResponse listResp = namespaceClient.listTableIndices(listReq);
+    assertNotNull(listResp);
+    assertNotNull(listResp.getIndexes());
+    assertEquals(1, listResp.getIndexes().size());
+    assertEquals("id_idx", listResp.getIndexes().get(0).getIndexName());
+    assertTrue(listResp.getIndexes().get(0).getColumns().contains("id"));
+  }
+
+  @Test
+  void testCreateVectorIndex() throws Exception {
+    // Create table with vector data (256 rows with 8-dim vectors)
+    byte[] tableData = createVectorTableData(256, 8);
+    CreateTableRequest createReq = new CreateTableRequest().id(Arrays.asList("vector_table"));
+    namespaceClient.createTable(createReq, tableData);
+
+    // Create vector index using IVF_FLAT
+    CreateTableIndexRequest createIndexReq =
+        new CreateTableIndexRequest()
+            .id(Arrays.asList("vector_table"))
+            .column("vector")
+            .indexType("IVF_FLAT")
+            .name("vector_idx")
+            .distanceType("l2");
+    CreateTableIndexResponse response = namespaceClient.createTableIndex(createIndexReq);
+    assertNotNull(response);
+
+    // List indices to verify
+    ListTableIndicesRequest listReq =
+        new ListTableIndicesRequest().id(Arrays.asList("vector_table"));
+    ListTableIndicesResponse listResp = namespaceClient.listTableIndices(listReq);
+    assertNotNull(listResp);
+    assertNotNull(listResp.getIndexes());
+    assertEquals(1, listResp.getIndexes().size());
+    assertEquals("vector_idx", listResp.getIndexes().get(0).getIndexName());
+    assertTrue(listResp.getIndexes().get(0).getColumns().contains("vector"));
+  }
+
+  private byte[] createVectorTableData(int numRows, int dim) throws Exception {
+    Schema schema =
+        new Schema(
+            Arrays.asList(
+                new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null),
+                new Field(
+                    "vector",
+                    FieldType.nullable(new ArrowType.FixedSizeList(dim)),
+                    Arrays.asList(
+                        new Field(
+                            "item",
+                            FieldType.nullable(
+                                new ArrowType.FloatingPoint(
+                                    org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE)),
+                            null)))));
+
+    try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
+      IntVector idVector = (IntVector) root.getVector("id");
+      org.apache.arrow.vector.complex.FixedSizeListVector vectorCol =
+          (org.apache.arrow.vector.complex.FixedSizeListVector) root.getVector("vector");
+      org.apache.arrow.vector.Float4Vector itemsVector =
+          (org.apache.arrow.vector.Float4Vector) vectorCol.getDataVector();
+
+      idVector.allocateNew(numRows);
+      vectorCol.allocateNew();
+      itemsVector.allocateNew(numRows * dim);
+
+      for (int i = 0; i < numRows; i++) {
+        idVector.set(i, i);
+        for (int j = 0; j < dim; j++) {
+          itemsVector.set(i * dim + j, (float) (i * dim + j) * 0.01f);
+        }
+        vectorCol.setNotNull(i);
+      }
+
+      idVector.setValueCount(numRows);
+      itemsVector.setValueCount(numRows * dim);
+      vectorCol.setValueCount(numRows);
+      root.setRowCount(numRows);
+
+      ByteArrayOutputStream out = new ByteArrayOutputStream();
+      try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, out)) {
+        writer.writeBatch();
+      }
+      return out.toByteArray();
+    }
+  }
 }
diff --git a/java/src/test/java/org/lance/namespace/DynamicContextProviderTest.java b/java/src/test/java/org/lance/namespace/DynamicContextProviderTest.java
index 7959eb9be58..1b980a24d09 100644
--- a/java/src/test/java/org/lance/namespace/DynamicContextProviderTest.java
+++ b/java/src/test/java/org/lance/namespace/DynamicContextProviderTest.java
@@ -13,7 +13,10 @@
  */
 package org.lance.namespace;
 
-import org.lance.namespace.model.*;
+import org.lance.namespace.model.CreateNamespaceRequest;
+import org.lance.namespace.model.CreateNamespaceResponse;
+import org.lance.namespace.model.ListNamespacesRequest;
+import org.lance.namespace.model.ListNamespacesResponse;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
diff --git a/java/src/test/java/org/lance/namespace/RestNamespaceTest.java b/java/src/test/java/org/lance/namespace/RestNamespaceTest.java
index eab60969d2c..4ee8c701351 100644
--- a/java/src/test/java/org/lance/namespace/RestNamespaceTest.java
+++ b/java/src/test/java/org/lance/namespace/RestNamespaceTest.java
@@ -13,7 +13,24 @@
  */
 package org.lance.namespace;
 
-import org.lance.namespace.model.*;
+import org.lance.namespace.model.CreateNamespaceRequest;
+import org.lance.namespace.model.CreateNamespaceResponse;
+import org.lance.namespace.model.CreateTableRequest;
+import org.lance.namespace.model.CreateTableResponse;
+import org.lance.namespace.model.DescribeNamespaceRequest;
+import org.lance.namespace.model.DescribeNamespaceResponse;
+import org.lance.namespace.model.DescribeTableRequest;
+import org.lance.namespace.model.DescribeTableResponse;
+import org.lance.namespace.model.DropNamespaceRequest;
+import org.lance.namespace.model.DropNamespaceResponse;
+import org.lance.namespace.model.DropTableRequest;
+import org.lance.namespace.model.DropTableResponse;
+import org.lance.namespace.model.ListNamespacesRequest;
+import org.lance.namespace.model.ListNamespacesResponse;
+import org.lance.namespace.model.ListTablesRequest;
+import org.lance.namespace.model.ListTablesResponse;
+import org.lance.namespace.model.NamespaceExistsRequest;
+import org.lance.namespace.model.TableExistsRequest;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
diff --git a/java/src/test/java/org/lance/operation/MergeTest.java b/java/src/test/java/org/lance/operation/MergeTest.java
index 121d3a2e9d5..841ad41d5ea 100644
--- a/java/src/test/java/org/lance/operation/MergeTest.java
+++ b/java/src/test/java/org/lance/operation/MergeTest.java
@@ -246,7 +246,7 @@ void testMergeNewColumnWithNonContiguousFieldId(@TempDir Path tempDir) throws Ex
                   datasetPath,
                   addressRoot,
                   new int[] {addressFieldId, cityFieldId, countryFieldId},
-                  new int[] {0, 1, 2});
+                  new int[] {-1, 0, 1});
 
           FragmentMetadata fragmentMeta = initialDataset.getFragment(0).metadata();
           List<DataFile> dataFiles = fragmentMeta.getFiles();
diff --git a/java/src/test/java/org/lance/operation/OperationTestBase.java b/java/src/test/java/org/lance/operation/OperationTestBase.java
index 5f2c2a46d99..fc2571fb55c 100644
--- a/java/src/test/java/org/lance/operation/OperationTestBase.java
+++ b/java/src/test/java/org/lance/operation/OperationTestBase.java
@@ -35,7 +35,7 @@
 public class OperationTestBase {
 
   public static final int TEST_FILE_FORMAT_MAJOR_VERSION = 2;
-  public static final int TEST_FILE_FORMAT_MINOR_VERSION = 0;
+  public static final int TEST_FILE_FORMAT_MINOR_VERSION = 1;
   protected Dataset dataset;
 
   @BeforeAll
diff --git a/java/src/test/java/org/lance/operation/OverwriteTest.java b/java/src/test/java/org/lance/operation/OverwriteTest.java
index 3af691c5009..c1def711edc 100644
--- a/java/src/test/java/org/lance/operation/OverwriteTest.java
+++ b/java/src/test/java/org/lance/operation/OverwriteTest.java
@@ -19,15 +19,22 @@
 import org.lance.FragmentMetadata;
 import org.lance.TestUtils;
 import org.lance.Transaction;
+import org.lance.WriteParams;
 import org.lance.ipc.LanceScanner;
 
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -121,4 +128,67 @@ void testOverwrite(@TempDir Path tempDir) throws Exception {
       }
     }
   }
+
+  @Test
+  void testOverwriteWithDifferentFieldTypes(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("testOverwriteFieldTypes").toString();
+    try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
+      // Create initial dataset with schema: id (int32), name (utf8)
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      dataset = testDataset.createEmptyDataset();
+      FragmentMetadata fragmentMeta = testDataset.createNewFragment(10);
+      try (Transaction txn =
+          new Transaction.Builder()
+              .readVersion(dataset.version())
+              .operation(
+                  Overwrite.builder()
+                      .fragments(Collections.singletonList(fragmentMeta))
+                      .schema(testDataset.getSchema())
+                      .build())
+              .build()) {
+        dataset = new CommitBuilder(this.dataset).execute(txn);
+      }
+      assertEquals(2, dataset.version());
+      assertEquals(10, dataset.countRows());
+
+      // Overwrite with a new schema where "id" changes from int32 to int64
+      // and "name" changes from utf8 to int64
+      Schema newSchema =
+          new Schema(
+              Arrays.asList(
+                  Field.nullable("id", new ArrowType.Int(64, true)),
+                  Field.nullable("name", new ArrowType.Int(64, true))));
+
+      int newRowCount = 5;
+      List<FragmentMetadata> newFragments;
+      try (VectorSchemaRoot root = VectorSchemaRoot.create(newSchema, allocator)) {
+        root.allocateNew();
+        BigIntVector idVector = (BigIntVector) root.getVector("id");
+        BigIntVector nameVector = (BigIntVector) root.getVector("name");
+        for (int i = 0; i < newRowCount; i++) {
+          idVector.setSafe(i, (long) i * 100);
+          nameVector.setSafe(i, (long) i * 200);
+        }
+        root.setRowCount(newRowCount);
+        newFragments =
+            Fragment.create(datasetPath, allocator, root, new WriteParams.Builder().build());
+      }
+
+      try (Transaction txn =
+          new Transaction.Builder()
+              .readVersion(dataset.version())
+              .operation(Overwrite.builder().fragments(newFragments).schema(newSchema).build())
+              .build()) {
+        try (Dataset overwritten = new CommitBuilder(this.dataset).execute(txn)) {
+          assertEquals(3, overwritten.version());
+          assertEquals(newRowCount, overwritten.countRows());
+
+          // Verify the schema has the new types
+          Schema resultSchema = overwritten.getSchema();
+          assertEquals(newSchema, resultSchema);
+        }
+      }
+    }
+  }
 }
diff --git a/protos/ann.proto b/protos/ann.proto
new file mode 100644
index 00000000000..37f188a7826
--- /dev/null
+++ b/protos/ann.proto
@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+syntax = "proto3";
+
+package lance.pb;
+
+import "table_identifier.proto";
+import "table.proto";
+import "index.proto";
+
+// Serialized vector query parameters.
+message VectorQueryProto {
+  // Query vector as Arrow IPC bytes (supports Float16, Float32, Float64, UInt8, etc.)
+  bytes query_vector_arrow_ipc = 1;
+  string column = 2;
+  uint32 k = 3;
+  optional float lower_bound = 4;
+  optional float upper_bound = 5;
+  optional uint32 minimum_nprobes = 6;
+  optional uint32 maximum_nprobes = 7;
+  optional uint32 ef = 8;
+  optional uint32 refine_factor = 9;
+  // Distance metric type. Absent means None (use the index's default metric).
+  optional lance.index.pb.VectorMetricType metric_type = 10;
+  bool use_index = 11;
+  optional float dist_q_c = 12;
+}
+
+// Serializable form of ANNIvfSubIndexExec — the IVF sub-index search node.
+//
+// Note: ANNIvfSubIndexExec.prefilter_source (child ExecutionPlan) is NOT
+// serialized here. DataFusion's PhysicalExtensionCodec handles child plans
+// automatically via children() / with_new_children(). The codec receives
+// deserialized children in the `inputs` parameter of try_decode and
+// reconstructs the PreFilterSource from them.
+message ANNIvfSubIndexExecProto {
+  VectorQueryProto query = 1;
+  lance.datafusion.TableIdentifier table = 2;
+  repeated lance.table.IndexMetadata indices = 3;
+}
diff --git a/protos/table.proto b/protos/table.proto
index e7de867e46e..e73d22b6b93 100644
--- a/protos/table.proto
+++ b/protos/table.proto
@@ -280,6 +280,20 @@ message IndexMetadata {
   // The base path index of the data file. Used when the file is imported or referred from another dataset.
   // Lance use it as key of the base_paths field in Manifest to determine the actual base path of the data file.
   optional uint32 base_id = 9;
+
+  // List of files and their sizes for this index segment.
+  // This enables skipping HEAD calls when opening indices and allows reporting
+  // of index sizes without extra IO.
+  // If this is empty, the index files sizes are unknown.
+  repeated IndexFile files = 10;
+}
+
+// Metadata about a single file within an index segment.
+message IndexFile {
+  // Path relative to the index directory (e.g., "index.idx", "auxiliary.idx")
+  string path = 1;
+  // Size of the file in bytes
+  uint64 size_bytes = 2;
 }
 
 // Index Section, containing a list of index metadata for one dataset version.
@@ -508,23 +522,23 @@ message FragmentReuseIndexDetails {
 // MemWAL Index Types
 // ============================================================================
 
-// Region manifest containing epoch-based fencing and WAL state.
-// Each region has exactly one active writer at any time.
-message RegionManifest {
-  // Region identifier (UUID v4).
-  UUID region_id = 11;
+// Shard manifest containing epoch-based fencing and WAL state.
+// Each shard has exactly one active writer at any time.
+message ShardManifest {
+  // Shard identifier (UUID v4).
+  UUID shard_id = 11;
 
   // Manifest version number.
   // Matches the version encoded in the filename.
   uint64 version = 1;
 
-  // Region spec ID this region was created with.
-  // Set at region creation and immutable thereafter.
-  // A value of 0 indicates a manually-created region not governed by any spec.
-  uint32 region_spec_id = 10;
+  // Shard spec ID this shard was created with.
+  // Set at shard creation and immutable thereafter.
+  // A value of 0 indicates a manually-created shard not governed by any spec.
+  uint32 shard_spec_id = 10;
 
   // Writer fencing token - monotonically increasing.
-  // A writer must increment this when claiming the region.
+  // A writer must increment this when claiming the shard.
   uint64 writer_epoch = 2;
 
   // The most recent WAL entry position (0-based) that has been flushed to a MemTable.
@@ -550,16 +564,16 @@ message FlushedGeneration {
   // Generation number.
   uint64 generation = 1;
 
-  // Directory name relative to the region directory.
+  // Directory name relative to the shard directory.
   string path = 2;
 }
 
-// A region's merged generation, used in MemWalIndexDetails.
+// A shard's merged generation, used in MemWalIndexDetails.
 message MergedGeneration {
-  // Region identifier (UUID v4).
-  UUID region_id = 1;
+  // Shard identifier (UUID v4).
+  UUID shard_id = 1;
 
-  // Last generation merged to base table for this region.
+  // Last generation merged to base table for this shard.
   uint64 generation = 2;
 }
 
@@ -569,27 +583,27 @@ message IndexCatchupProgress {
   // Name of the base table index (must match an entry in maintained_indexes).
   string index_name = 1;
 
-  // Per-region progress: the generation up to which this index covers.
-  // If a region is not present, the index is assumed to be fully caught up
-  // (i.e., caught_up_generation >= merged_generation for that region).
+  // Per-shard progress: the generation up to which this index covers.
+  // If a shard is not present, the index is assumed to be fully caught up
+  // (i.e., caught_up_generation >= merged_generation for that shard).
   repeated MergedGeneration caught_up_generations = 2;
 }
 
 // Index details for MemWAL Index, stored in IndexMetadata.index_details.
 // This is the centralized structure for all MemWAL metadata:
-// - Configuration (region specs, indexes to maintain)
-// - Merge progress (merged generations per region)
-// - Region state snapshots
+// - Configuration (shard specs, indexes to maintain)
+// - Merge progress (merged generations per shard)
+// - Shard state snapshots
 //
 // Writers read this index to get configuration before writing.
-// Readers read this index to discover regions and their state.
-// A background process updates the index periodically to keep region snapshots current.
+// Readers read this index to discover shards and their state.
+// A background process updates the index periodically to keep shard snapshots current.
 //
-// Region snapshots are stored as a Lance file with one row per region.
-// The schema has one column per RegionManifest field, with region fields as columns:
-//   region_id: fixed_size_binary(16)  -- UUID bytes
+// Shard snapshots are stored as a Lance file with one row per shard.
+// The schema has one column per ShardManifest field, with shard fields as columns:
+//   shard_id: fixed_size_binary(16)  -- UUID bytes
 //   version: uint64
-//   region_spec_id: uint32
+//   shard_spec_id: uint32
 //   writer_epoch: uint64
 //   replay_after_wal_entry_position: uint64
 //   wal_entry_position_last_seen: uint64
@@ -600,19 +614,19 @@ message MemWalIndexDetails {
   // Snapshot timestamp (Unix timestamp in milliseconds).
   int64 snapshot_ts_millis = 1;
 
-  // Number of regions in the snapshot.
+  // Number of shards in the snapshot.
   // Used to determine storage format without reading the snapshot data.
-  uint32 num_regions = 2;
+  uint32 num_shards = 2;
 
-  // Inline region snapshots for small region counts.
-  // When num_regions <= threshold (implementation-defined, e.g., 100),
+  // Inline shard snapshots for small shard counts.
+  // When num_shards <= threshold (implementation-defined, e.g., 100),
   // snapshots are stored inline as serialized bytes.
-  // Format: Lance file bytes with the region snapshot schema.
+  // Format: Lance file bytes with the shard snapshot schema.
   optional bytes inline_snapshots = 3;
 
-  // Region specs defining how to derive region identifiers.
-  // This configuration determines how rows are partitioned into regions.
-  repeated RegionSpec region_specs = 7;
+  // Shard specs defining how to derive shard identifiers.
+  // This configuration determines how rows are partitioned into shards.
+  repeated ShardSpec shard_specs = 7;
 
   // Indexes from the base table to maintain in MemTables.
   // These are index names referencing indexes defined on the base table.
@@ -623,13 +637,13 @@ message MemWalIndexDetails {
   // SQ params) from the base table index to ensure distance comparability.
   repeated string maintained_indexes = 8;
 
-  // Last generation merged to base table for each region.
+  // Last generation merged to base table for each shard.
   // This is updated atomically with merge-insert data commits, enabling
   // conflict resolution when multiple mergers operate concurrently.
   //
-  // Note: This is separate from region snapshots because:
+  // Note: This is separate from shard snapshots because:
   // 1. merged_generations is updated by mergers (atomic with data commit)
-  // 2. region snapshots are updated by background index builder
+  // 2. shard snapshots are updated by background index builder
   repeated MergedGeneration merged_generations = 9;
 
   // Per-index catchup progress tracking.
@@ -644,25 +658,25 @@ message MemWalIndexDetails {
   repeated IndexCatchupProgress index_catchup = 10;
 }
 
-// Region spec definition.
-message RegionSpec {
+// Shard spec definition.
+message ShardSpec {
   // Unique identifier for this spec within the index.
   // IDs are never reused.
   uint32 spec_id = 1;
 
-  // Region field definitions that determine how to compute region identifiers.
-  repeated RegionField fields = 2;
+  // Shard field definitions that determine how to compute shard identifiers.
+  repeated ShardField fields = 2;
 }
 
-// Region field definition.
-message RegionField {
-  // Unique string identifier for this region field.
+// Shard field definition.
+message ShardField {
+  // Unique string identifier for this shard field.
   string field_id = 1;
 
   // Field IDs referencing source columns in the schema.
   repeated int32 source_ids = 2;
 
-  // Well-known region transform name (e.g., "identity", "year", "bucket").
+  // Well-known shard transform name (e.g., "identity", "year", "bucket").
   // Mutually exclusive with expression.
   optional string transform = 3;
 
@@ -670,7 +684,7 @@ message RegionField {
   // Mutually exclusive with transform.
   optional string expression = 4;
 
-  // Output type of the region value (Arrow type name).
+  // Output type of the shard value (Arrow type name).
   string result_type = 5;
 
   // Transform parameters (e.g., num_buckets for bucket transform).
diff --git a/protos/transaction.proto b/protos/transaction.proto
index 17d96486736..06268feb252 100644
--- a/protos/transaction.proto
+++ b/protos/transaction.proto
@@ -110,6 +110,9 @@ message Transaction {
       google.protobuf.Any new_index_details = 3;
       // the version of the new index
       uint32 new_index_version = 4;
+      // Files in the new index with their sizes.
+      // Empty if file sizes are not available (e.g. older writers).
+      repeated IndexFile new_index_files = 5;
     }
 
     // A group of rewrite files that are all part of the same rewrite.
@@ -235,7 +238,7 @@ message Transaction {
     repeated DataFragment new_fragments = 3;
     // The ids of the fields that have been modified.
     repeated uint32 fields_modified = 4;
-    /// List of MemWAL region generations to mark as merged after this transaction
+    /// List of MemWAL shard generations to mark as merged after this transaction
     repeated MergedGeneration merged_generations = 5;
     /// The fields that used to judge whether to preserve the new frag's id into
     /// the frag bitmap of the specified indices.
@@ -309,7 +312,7 @@ message Transaction {
   // This operation is used during merge-insert to atomically record which
   // generations have been merged to the base table.
   message UpdateMemWalState {
-    // Regions and generations being marked as merged.
+    // Shards and generations being marked as merged.
     repeated MergedGeneration merged_generations = 1;
   }
 
diff --git a/python/AGENTS.md b/python/AGENTS.md
index 8aaaa53bedc..9e74d7c0656 100644
--- a/python/AGENTS.md
+++ b/python/AGENTS.md
@@ -4,12 +4,19 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards.
 
 ## Commands
 
-* Build: `maturin develop` (required after Rust changes)
-* Test: `make test`
-* Run single test: `pytest python/tests/<test_file>.py::<test_name>`
-* Doctest: `make doctest`
-* Lint: `make lint`
-* Format: `make format`
+* Environment: use `uv` for all local Python environment setup in this repository.
+* First step in every new worktree or fresh checkout: run `uv sync --extra tests --extra dev` from `python/` before any Python command. Add other extras such as `benchmarks`, `torch`, or `geo` only when needed.
+* `uv sync` builds the local `pylance` Rust extension as part of environment setup. This can take a long time. Start it early, let it finish, and do not interrupt it or switch to a different setup path just because the build is slow.
+* Command execution: always use `uv run ...` for Python-related repository commands. Do not rely on a globally activated environment.
+* Never invoke bare `python`, `pytest`, `pip`, `maturin`, `make test`, `make doctest`, `make lint`, or `make format` for repository work.
+* If a Python command fails outside `uv run`, that does not count as a dependency or test failure. Fix the environment usage first and rerun correctly.
+* Build time expectations: `uv sync` and `uv run maturin develop` build the local `pylance` Rust extension as part of the environment workflow. This can be slow, especially on the first run or after Rust dependency changes; treat that as expected and do not switch to a different environment manager or shortcut around the build just because it takes time.
+* Build: `uv run maturin develop` (required after Rust changes)
+* Test: `uv run make test`
+* Run single test: `uv run pytest python/tests/<test_file>.py::<test_name>`
+* Doctest: `uv run make doctest`
+* Lint: `uv run make lint`
+* Format: `uv run make format`
 
 ## API Design
 
@@ -23,3 +30,8 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards.
 - Use `@pytest.mark.parametrize` for tests that differ only in inputs — extract shared setup into helpers.
 - Add tests to existing `test_{module}.py` files rather than creating new test files for the same module.
 - Replace `print()` in tests with `assert` statements.
+
+## Common Failure Mode
+
+- A missing module or missing command error from bare `python`, `pytest`, `pip`, `maturin`, or `make` is usually an environment usage mistake, not a repository issue.
+- Before reporting a Python dependency as unavailable, verify that `uv sync --extra tests --extra dev` has been run in the current worktree and that the failing command was executed with `uv run ...`.
diff --git a/python/Cargo.lock b/python/Cargo.lock
index 7f461c0348e..7a7bfc20e2d 100644
--- a/python/Cargo.lock
+++ b/python/Cargo.lock
@@ -56,12 +56,6 @@ version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
 
-[[package]]
-name = "adler32"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
-
 [[package]]
 name = "aes"
 version = "0.8.4"
@@ -70,7 +64,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
 dependencies = [
  "cfg-if",
  "cipher",
- "cpufeatures",
+ "cpufeatures 0.2.17",
 ]
 
 [[package]]
@@ -128,9 +122,9 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.6.21"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
+checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
 dependencies = [
  "anstyle",
  "anstyle-parse",
@@ -143,15 +137,15 @@ dependencies = [
 
 [[package]]
 name = "anstyle"
-version = "1.0.13"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.7"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
+checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
 dependencies = [
  "utf8parse",
 ]
@@ -200,15 +194,6 @@ dependencies = [
  "object",
 ]
 
-[[package]]
-name = "arc-swap"
-version = "1.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5"
-dependencies = [
- "rustversion",
-]
-
 [[package]]
 name = "arrayref"
 version = "0.3.9"
@@ -350,7 +335,7 @@ dependencies = [
  "arrow-schema",
  "arrow-select",
  "flatbuffers",
- "lz4_flex 0.12.0",
+ "lz4_flex",
  "zstd",
 ]
 
@@ -422,7 +407,7 @@ version = "57.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "serde_core",
  "serde_json",
 ]
@@ -610,9 +595,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-rs"
-version = "1.16.1"
+version = "1.16.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf"
+checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f"
 dependencies = [
  "aws-lc-sys",
  "zeroize",
@@ -620,9 +605,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-sys"
-version = "0.38.0"
+version = "0.40.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e"
+checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7"
 dependencies = [
  "cc",
  "cmake",
@@ -815,23 +800,17 @@ dependencies = [
  "aws-smithy-async",
  "aws-smithy-runtime-api",
  "aws-smithy-types",
- "h2 0.3.27",
- "h2 0.4.13",
- "http 0.2.12",
+ "h2",
  "http 1.4.0",
- "http-body 0.4.6",
- "hyper 0.14.32",
- "hyper 1.8.1",
- "hyper-rustls 0.24.2",
- "hyper-rustls 0.27.7",
+ "hyper",
+ "hyper-rustls",
  "hyper-util",
  "pin-project-lite",
- "rustls 0.21.12",
- "rustls 0.23.37",
+ "rustls",
  "rustls-native-certs",
  "rustls-pki-types",
  "tokio",
- "tokio-rustls 0.26.4",
+ "tokio-rustls",
  "tower",
  "tracing",
 ]
@@ -968,7 +947,7 @@ dependencies = [
  "http 1.4.0",
  "http-body 1.0.1",
  "http-body-util",
- "hyper 1.8.1",
+ "hyper",
  "hyper-util",
  "itoa",
  "matchit",
@@ -1084,9 +1063,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.11.0"
+version = "2.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
+checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
 
 [[package]]
 name = "bitpacking"
@@ -1120,16 +1099,16 @@ dependencies = [
 
 [[package]]
 name = "blake3"
-version = "1.8.3"
+version = "1.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d"
+checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e"
 dependencies = [
  "arrayref",
  "arrayvec",
  "cc",
  "cfg-if",
  "constant_time_eq",
- "cpufeatures",
+ "cpufeatures 0.3.0",
 ]
 
 [[package]]
@@ -1150,31 +1129,6 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "bon"
-version = "3.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d13a61f2963b88eef9c1be03df65d42f6996dfeac1054870d950fcf66686f83"
-dependencies = [
- "bon-macros",
- "rustversion",
-]
-
-[[package]]
-name = "bon-macros"
-version = "3.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d314cc62af2b6b0c65780555abb4d02a03dd3b799cd42419044f0c38d99738c0"
-dependencies = [
- "darling 0.23.0",
- "ident_case",
- "prettyplease",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "brotli"
 version = "8.0.2"
@@ -1250,9 +1204,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.56"
+version = "1.2.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
+checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1269,12 +1223,6 @@ dependencies = [
  "smallvec",
 ]
 
-[[package]]
-name = "census"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0"
-
 [[package]]
 name = "cfg-if"
 version = "1.0.4"
@@ -1323,18 +1271,18 @@ dependencies = [
 
 [[package]]
 name = "cmake"
-version = "0.1.57"
+version = "0.1.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d"
+checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
 dependencies = [
  "cc",
 ]
 
 [[package]]
 name = "colorchoice"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
+checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
 
 [[package]]
 name = "comfy-table"
@@ -1433,15 +1381,6 @@ version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
-[[package]]
-name = "core2"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "core_extensions"
 version = "1.5.4"
@@ -1466,6 +1405,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "cpufeatures"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "crc32c"
 version = "0.6.8"
@@ -1580,18 +1528,8 @@ version = "0.20.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
 dependencies = [
- "darling_core 0.20.11",
- "darling_macro 0.20.11",
-]
-
-[[package]]
-name = "darling"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
-dependencies = [
- "darling_core 0.23.0",
- "darling_macro 0.23.0",
+ "darling_core",
+ "darling_macro",
 ]
 
 [[package]]
@@ -1608,47 +1546,17 @@ dependencies = [
  "syn 2.0.117",
 ]
 
-[[package]]
-name = "darling_core"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
-dependencies = [
- "ident_case",
- "proc-macro2",
- "quote",
- "strsim",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "darling_macro"
 version = "0.20.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
 dependencies = [
- "darling_core 0.20.11",
- "quote",
- "syn 2.0.117",
-]
-
-[[package]]
-name = "darling_macro"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
-dependencies = [
- "darling_core 0.23.0",
+ "darling_core",
  "quote",
  "syn 2.0.117",
 ]
 
-[[package]]
-name = "dary_heap"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04"
-
 [[package]]
 name = "dashmap"
 version = "6.1.0"
@@ -1665,9 +1573,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503f1f4a9060ae6e650d3dff5dc7a21266fea1302d890768d45b4b28586e830f"
+checksum = "7541353e77dc7262b71ca27be07d8393661737e3a73b5d1b1c6f7d814c64fa2a"
 dependencies = [
  "arrow",
  "arrow-schema",
@@ -1708,7 +1616,7 @@ dependencies = [
  "object_store",
  "parking_lot",
  "parquet",
- "rand 0.9.2",
+ "rand 0.9.4",
  "regex",
  "sqlparser",
  "tempfile",
@@ -1720,9 +1628,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14417a3ee4ae3d092b56cd6c1d32e8ff3e2c9ec130ecb2276ec91c89fd599399"
+checksum = "9997731f90fa5398ef831ad0e69600f92c861b79c0d38bd1a29b6f0e3a0ce4c8"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1745,9 +1653,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog-listing"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d0eba824adb45a4b3ac6f0251d40df3f6a9382371cad136f4f14ac9ebc6bc10"
+checksum = "2b30a3dd50dec860c9559275c8d97d9de602e611237a6ecfbda0b3b63b872352"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1768,9 +1676,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0039deefbd00c56adf5168b7ca58568fb058e4ba4c5a03b09f8be371b4e434b6"
+checksum = "d551054acec0398ca604512310b77ce05c46f66e54b54d48200a686e385cca4e"
 dependencies = [
  "ahash",
  "arrow",
@@ -1792,9 +1700,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common-runtime"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ec7e3e60b813048331f8fb9673583173e5d2dd8fef862834ee871fc98b57ca7"
+checksum = "567d40e285f5b79f8737b576605721cd6c1133b5d2b00bdbd5d9838d90d0812f"
 dependencies = [
  "futures",
  "log",
@@ -1803,9 +1711,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "802068957f620302ecf05f84ff4019601aeafd36f5f3f1334984af2e34265129"
+checksum = "27d2668f51b3b30befae2207472569e37807fdedd1d14da58acc6f8ca6257eae"
 dependencies = [
  "arrow",
  "async-compression",
@@ -1829,7 +1737,7 @@ dependencies = [
  "liblzma",
  "log",
  "object_store",
- "rand 0.9.2",
+ "rand 0.9.4",
  "tokio",
  "tokio-util",
  "url",
@@ -1838,9 +1746,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-arrow"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90fc387d5067c62d494a6647d29c5ad4fcdd5a6e50ab4ea1d2568caa2d66f2cc"
+checksum = "e02e1b3e3a8ec55f1f62de4252b0407c8567363d056078769a197e24fc834a0f"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -1862,9 +1770,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-csv"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efd5e20579bb6c8bd4e6c620253972fb723822030c280dd6aa047f660d09eeba"
+checksum = "b559d7bf87d4f900f847baba8509634f838d9718695389e903604cdcccdb01f3"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1885,9 +1793,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-json"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0788b0d48fcef31880a02013ea3cc18e5a4e0eacc3b0abdd2cd0597b99dc96e"
+checksum = "250e2d7591ba8b638f063854650faa40bca4e8bd4059b2ece8836f6388d02db4"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1907,9 +1815,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-parquet"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66639b70f1f363f5f0950733170100e588f1acfacac90c1894e231194aa35957"
+checksum = "0b043149f2c3557ca94abc58de40f68a8d412ff53365c06126ed234f8596399d"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1937,15 +1845,15 @@ dependencies = [
 
 [[package]]
 name = "datafusion-doc"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e44b41f3e8267c6cf3eec982d63f34db9f1dd5f30abfd2e1f124f0871708952e"
+checksum = "b9496cb0db222dbb9a3735760ceca7fc56f35e1d5502c38d0caa77a81e9c1f6a"
 
 [[package]]
 name = "datafusion-execution"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e456f60e5d38db45335e84617006d90af14a8c8c5b8e959add708b2daaa0e2c"
+checksum = "dc45d23c516ed8d3637751e44e09e21b45b3f58b473c802dddd1f1ad4fe435ff"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1957,16 +1865,16 @@ dependencies = [
  "log",
  "object_store",
  "parking_lot",
- "rand 0.9.2",
+ "rand 0.9.4",
  "tempfile",
  "url",
 ]
 
 [[package]]
 name = "datafusion-expr"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6507c719804265a58043134580c1c20767e7c23ba450724393f03ec982769ad9"
+checksum = "63dd30526d2db4fda6440806a41e4676334a94bc0596cc9cc2a0efed20ef2c44"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1987,9 +1895,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-expr-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a413caa9c5885072b539337aed68488f0291653e8edd7d676c92df2480f6cab0"
+checksum = "1b486b5f6255d40976b88bb83813b0d035a8333e0ec39864824e78068cf42fa6"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2000,9 +1908,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-ffi"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ca486e22de2bb1512dda751fb490c1cabafa9aec67b456cd4038e812be527f7"
+checksum = "26b04181cffefd632e57acfc233ed239626863682dd8bb30ab366293f441bba8"
 dependencies = [
  "abi_stable",
  "arrow",
@@ -2030,9 +1938,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "189256495dc9cbbb8e20dbcf161f60422e628d201a78df8207e44bd4baefadb6"
+checksum = "07356c94118d881130dd0ffbff127540407d969c8978736e324edcd6c41cd48f"
 dependencies = [
  "arrow",
  "arrow-buffer",
@@ -2052,7 +1960,7 @@ dependencies = [
  "log",
  "md-5",
  "num-traits",
- "rand 0.9.2",
+ "rand 0.9.4",
  "regex",
  "sha2",
  "unicode-segmentation",
@@ -2061,9 +1969,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-aggregate"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12e73dfee4cd67c4a507ffff4c5a711d39983adf544adbc09c09bf06f789f413"
+checksum = "b644f9cf696df9233ce6958b9807666d78563b56f923267474dd6c07795f1f8f"
 dependencies = [
  "ahash",
  "arrow",
@@ -2082,9 +1990,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-aggregate-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87727bd9e65f4f9ac6d608c9810b7da9eaa3b18b26a4a4b76520592d49020acf"
+checksum = "c1de2deaaabe8923ce9ea9f29c47bbb4ee14f67ea2fe1ab5398d9bbebcf86e56"
 dependencies = [
  "ahash",
  "arrow",
@@ -2095,9 +2003,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-nested"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e5ef761359224b7c2b5a1bfad6296ac63225f8583d08ad18af9ba1a89ac3887"
+checksum = "552f8d92e4331ee91d23c02d12bb6acf32cbfd5215117e01c0fb63cd4b15af1a"
 dependencies = [
  "arrow",
  "arrow-ord",
@@ -2118,9 +2026,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-table"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b17dac25dfda2d2a90ff0ad1c054a11fb1523766226bec6e9bd8c410daee2ae"
+checksum = "970fd0cdd3df8802b9a9975ff600998289ba9d46682a4f7285cba4820c9ada78"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2134,9 +2042,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-window"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c594a29ddb22cbdbce500e4d99b5b2392c5cecb4c1086298b41d1ffec14dbb77"
+checksum = "40b4c21a7c8a986a1866c0a87ab756d0bbf7b5f41f306009fa2d9af79c52ed31"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2152,9 +2060,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-window-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aa1b15ed81c7543f62264a30dd49dec4b1b0b698053b968f53be32dfba4f729"
+checksum = "b1210ad73b8b3211aeaf4a42bef9bd7a2b7fce3ec119a478831f18c6ff7f7b93"
 dependencies = [
  "datafusion-common",
  "datafusion-physical-expr-common",
@@ -2162,9 +2070,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-macros"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c00c31c4795597aa25b74cab5174ac07a53051f27ce1e011ecaffa9eaeecef81"
+checksum = "aaa566a963013a38681ad82a727a654bc7feb19632426aea8c3412d415d200c5"
 dependencies = [
  "datafusion-doc",
  "quote",
@@ -2173,9 +2081,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-optimizer"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80ccf60767c09302b2e0fc3afebb3761a6d508d07316fab8c5e93312728a21bb"
+checksum = "ff9aa82b240252a88dee118372f9b9757c545ab9e53c0736bebab2e7da0ef1f2"
 dependencies = [
  "arrow",
  "chrono",
@@ -2193,9 +2101,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c64b7f277556944e4edd3558da01d9e9ff9f5416f1c0aa7fee088e57bd141a7e"
+checksum = "7d48022b8af9988c1d852644f9e8b5584c490659769a550c5e8d39457a1da0a5"
 dependencies = [
  "ahash",
  "arrow",
@@ -2217,9 +2125,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-adapter"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7abaee372ea2d19c016ee9ef8629c4415257d291cdd152bc7f0b75f28af1b63"
+checksum = "ae7a8abc0b4fe624000972a9b145b30b7f1b680bffaa950ea53f78d9b21c27c3"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2232,9 +2140,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42237efe621f92adc22d111b531fdbc2cc38ca9b5e02327535628fb103ae2157"
+checksum = "147253ca3e6b9d59c162de64c02800973018660e13340dd1886dd038d17ac429"
 dependencies = [
  "ahash",
  "arrow",
@@ -2249,9 +2157,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-optimizer"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd093498bd1319c6e5c76e9dfa905e78486f01b34579ce97f2e3a49f84c37fac"
+checksum = "689156bb2282107b6239db8d7ef44b4dab10a9b33d3491a0c74acac5e4fedd72"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2268,9 +2176,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-plan"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cbe61b12daf81a9f20ba03bd3541165d51f86e004ef37426b11881330eed261"
+checksum = "68253dc0ee5330aa558b2549c9b0da5af9fc17d753ae73022939014ad616fc28"
 dependencies = [
  "ahash",
  "arrow",
@@ -2299,9 +2207,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-proto"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33c055594ab7e3f5430aea1024055bc9bd29ba6479a9cae6fe29823a2f527470"
+checksum = "3f5ab57d0b5a368258fff1d828f1619a10541fa5c4ec4930a383deb3a23204c8"
 dependencies = [
  "arrow",
  "chrono",
@@ -2326,9 +2234,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-proto-common"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84b2523bb8e7269b943c9060a3ae91c5a61e6b1d800c014a9433547a9ce23e55"
+checksum = "bd21d2c804802ca4b1719191dfe8e3d0860686649de6375ddc9237f85beb82b3"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2337,9 +2245,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-pruning"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0124331116db7f79df92ebfd2c3b11a8f90240f253555c9bb084f10b6fecf1dd"
+checksum = "0fcad240a54d0b1d3e8f668398900260a53122d522b2102ab57218590decacd6"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2354,9 +2262,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-session"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1673e3c58ba618a6ea0568672f00664087b8982c581e9afd5aa6c3c79c9b431f"
+checksum = "f58e83a68bb67007a8fcbf005c44cefe441270c7ee7f6dee10c0e0109b556f6d"
 dependencies = [
  "async-trait",
  "datafusion-common",
@@ -2368,9 +2276,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-sql"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5272d256dab5347bb39d2040589f45d8c6b715b27edcb5fffe88cc8b9c3909cb"
+checksum = "be53e9eb55db0fbb8980bb6d87f2435b0524acf4c718ed54a57cabbb299b2ab3"
 dependencies = [
  "arrow",
  "bigdecimal",
@@ -2386,9 +2294,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-substrait"
-version = "52.2.0"
+version = "52.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "199790fd96e852997b30da4ff11109378c944841757d93875ea85fc69587ec91"
+checksum = "7dadf18107366b3470e47bdde8eeb9294b36444e451830d8e5fc8dc7b4ad9c58"
 dependencies = [
  "async-recursion",
  "async-trait",
@@ -2442,7 +2350,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
 dependencies = [
  "powerfmt",
- "serde_core",
 ]
 
 [[package]]
@@ -2460,7 +2367,7 @@ version = "0.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
 dependencies = [
- "darling 0.20.11",
+ "darling",
  "proc-macro2",
  "quote",
  "syn 2.0.117",
@@ -2529,12 +2436,6 @@ dependencies = [
  "const-random",
 ]
 
-[[package]]
-name = "downcast-rs"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc"
-
 [[package]]
 name = "dunce"
 version = "1.0.5"
@@ -2647,9 +2548,9 @@ dependencies = [
 
 [[package]]
 name = "env_filter"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f"
+checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef"
 dependencies = [
  "log",
  "regex",
@@ -2657,9 +2558,9 @@ dependencies = [
 
 [[package]]
 name = "env_logger"
-version = "0.11.9"
+version = "0.11.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d"
+checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a"
 dependencies = [
  "anstream",
  "anstyle",
@@ -2717,17 +2618,11 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
 
-[[package]]
-name = "fastdivide"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471"
-
 [[package]]
 name = "fastrand"
-version = "2.3.0"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
+checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
 
 [[package]]
 name = "filetime"
@@ -2758,7 +2653,7 @@ version = "25.12.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "rustc_version",
 ]
 
@@ -2806,16 +2701,6 @@ dependencies = [
  "percent-encoding",
 ]
 
-[[package]]
-name = "fs4"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8"
-dependencies = [
- "rustix 0.38.44",
- "windows-sys 0.52.0",
-]
-
 [[package]]
 name = "fs_extra"
 version = "1.3.0"
@@ -2824,10 +2709,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-array",
- "rand 0.9.2",
+ "rand 0.9.4",
 ]
 
 [[package]]
@@ -2996,9 +2881,9 @@ dependencies = [
 
 [[package]]
 name = "geo-types"
-version = "0.7.18"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24f8647af4005fa11da47cd56252c6ef030be8fa97bdbf355e7dfb6348f0a82c"
+checksum = "94776032c45f950d30a13af6113c2ad5625316c9abfbccee4dd5a6695f8fe0f5"
 dependencies = [
  "approx",
  "num-traits",
@@ -3147,25 +3032,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "h2"
-version = "0.3.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d"
-dependencies = [
- "bytes",
- "fnv",
- "futures-core",
- "futures-sink",
- "futures-util",
- "http 0.2.12",
- "indexmap",
- "slab",
- "tokio",
- "tokio-util",
- "tracing",
-]
-
 [[package]]
 name = "h2"
 version = "0.4.13"
@@ -3218,8 +3084,6 @@ version = "0.15.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
 dependencies = [
- "allocator-api2",
- "equivalent",
  "foldhash 0.1.5",
 ]
 
@@ -3234,6 +3098,12 @@ dependencies = [
  "foldhash 0.2.0",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"
+
 [[package]]
 name = "heapless"
 version = "0.8.0"
@@ -3280,12 +3150,6 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
-[[package]]
-name = "htmlescape"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
-
 [[package]]
 name = "http"
 version = "0.2.12"
@@ -3361,46 +3225,21 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
 
 [[package]]
 name = "hyper"
-version = "0.14.32"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7"
-dependencies = [
- "bytes",
- "futures-channel",
- "futures-core",
- "futures-util",
- "h2 0.3.27",
- "http 0.2.12",
- "http-body 0.4.6",
- "httparse",
- "httpdate",
- "itoa",
- "pin-project-lite",
- "socket2 0.5.10",
- "tokio",
- "tower-service",
- "tracing",
- "want",
-]
-
-[[package]]
-name = "hyper"
-version = "1.8.1"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"
+checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"
 dependencies = [
  "atomic-waker",
  "bytes",
  "futures-channel",
  "futures-core",
- "h2 0.4.13",
+ "h2",
  "http 1.4.0",
  "http-body 1.0.1",
  "httparse",
  "httpdate",
  "itoa",
  "pin-project-lite",
- "pin-utils",
  "smallvec",
  "tokio",
  "want",
@@ -3408,33 +3247,17 @@ dependencies = [
 
 [[package]]
 name = "hyper-rustls"
-version = "0.24.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
-dependencies = [
- "futures-util",
- "http 0.2.12",
- "hyper 0.14.32",
- "log",
- "rustls 0.21.12",
- "tokio",
- "tokio-rustls 0.24.1",
-]
-
-[[package]]
-name = "hyper-rustls"
-version = "0.27.7"
+version = "0.27.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58"
+checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
 dependencies = [
  "http 1.4.0",
- "hyper 1.8.1",
+ "hyper",
  "hyper-util",
- "rustls 0.23.37",
+ "rustls",
  "rustls-native-certs",
- "rustls-pki-types",
  "tokio",
- "tokio-rustls 0.26.4",
+ "tokio-rustls",
  "tower-service",
  "webpki-roots",
 ]
@@ -3451,12 +3274,12 @@ dependencies = [
  "futures-util",
  "http 1.4.0",
  "http-body 1.0.1",
- "hyper 1.8.1",
+ "hyper",
  "ipnet",
  "libc",
  "percent-encoding",
  "pin-project-lite",
- "socket2 0.6.2",
+ "socket2",
  "tokio",
  "tower-service",
  "tracing",
@@ -3540,12 +3363,13 @@ dependencies = [
 
 [[package]]
 name = "icu_collections"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43"
+checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"
 dependencies = [
  "displaydoc",
  "potential_utf",
+ "utf8_iter",
  "yoke",
  "zerofrom",
  "zerovec",
@@ -3553,9 +3377,9 @@ dependencies = [
 
 [[package]]
 name = "icu_locale_core"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6"
+checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
 dependencies = [
  "displaydoc",
  "litemap",
@@ -3566,9 +3390,9 @@ dependencies = [
 
 [[package]]
 name = "icu_normalizer"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599"
+checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
 dependencies = [
  "icu_collections",
  "icu_normalizer_data",
@@ -3580,15 +3404,15 @@ dependencies = [
 
 [[package]]
 name = "icu_normalizer_data"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"
+checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
 
 [[package]]
 name = "icu_properties"
-version = "2.1.2"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec"
+checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
 dependencies = [
  "icu_collections",
  "icu_locale_core",
@@ -3600,15 +3424,15 @@ dependencies = [
 
 [[package]]
 name = "icu_properties_data"
-version = "2.1.2"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af"
+checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
 
 [[package]]
 name = "icu_provider"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614"
+checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
 dependencies = [
  "displaydoc",
  "icu_locale_core",
@@ -3652,51 +3476,14 @@ dependencies = [
  "icu_properties",
 ]
 
-[[package]]
-name = "include-flate"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e01b7cb6ca682a621e7cda1c358c9724b53a7b4409be9be1dd443b7f3a26f998"
-dependencies = [
- "include-flate-codegen",
- "include-flate-compress",
- "libflate",
- "zstd",
-]
-
-[[package]]
-name = "include-flate-codegen"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f49bf5274aebe468d6e6eba14a977eaf1efa481dc173f361020de70c1c48050"
-dependencies = [
- "include-flate-compress",
- "libflate",
- "proc-macro-error",
- "proc-macro2",
- "quote",
- "syn 2.0.117",
- "zstd",
-]
-
-[[package]]
-name = "include-flate-compress"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eae6a40e716bcd5931f5dbb79cd921512a4f647e2e9413fded3171fca3824dbc"
-dependencies = [
- "libflate",
- "zstd",
-]
-
 [[package]]
 name = "indexmap"
-version = "2.13.0"
+version = "2.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
+checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
 dependencies = [
  "equivalent",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.0",
  "serde",
  "serde_core",
 ]
@@ -3726,6 +3513,17 @@ version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
+[[package]]
+name = "io-uring"
+version = "0.7.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdd7bddefd0a8833b88a4b68f90dae22c7450d11b354198baee3874fd811b344"
+dependencies = [
+ "bitflags 2.11.1",
+ "cfg-if",
+ "libc",
+]
+
 [[package]]
 name = "ipnet"
 version = "2.12.0"
@@ -3734,9 +3532,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
 
 [[package]]
 name = "iri-string"
-version = "0.7.10"
+version = "0.7.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a"
+checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20"
 dependencies = [
  "memchr",
  "serde",
@@ -3777,27 +3575,26 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.17"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "jieba-macros"
-version = "0.8.1"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "348294e44ee7e3c42685da656490f8febc7359632544019621588902216da95c"
+checksum = "a29cfc5dcd898604c6f80363411fa6b6b08e27d1d253d6225b9cb6702ea02fc0"
 dependencies = [
  "phf_codegen",
 ]
 
 [[package]]
 name = "jieba-rs"
-version = "0.8.1"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "766bd7012aa5ba49411ebdf4e93bddd59b182d2918e085d58dec5bb9b54b7105"
+checksum = "3245d6e9d1d5facbd6a23848d6b67e3439738ccbb4fa5a3d65da315ba1a910a2"
 dependencies = [
  "cedarwood",
- "include-flate",
  "jieba-macros",
  "phf 0.13.1",
  "regex",
@@ -3857,32 +3654,34 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.91"
+version = "0.3.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c"
+checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca"
 dependencies = [
+ "cfg-if",
+ "futures-util",
  "once_cell",
  "wasm-bindgen",
 ]
 
 [[package]]
 name = "jsonb"
-version = "0.5.5"
+version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a901f06163d352fbe41c3c2ff5e08b75330a003cc941e988fb501022f5421e6"
+checksum = "eb98fb29636087c40ad0d1274d9a30c0c1e83e03ae93f6e7e89247b37fcc6953"
 dependencies = [
  "byteorder",
  "ethnum",
  "fast-float2",
  "itoa",
  "jiff",
- "nom 8.0.0",
+ "nom",
  "num-traits",
- "ordered-float 5.1.0",
- "rand 0.9.2",
- "ryu",
+ "ordered-float 5.3.0",
+ "rand 0.9.4",
  "serde",
  "serde_json",
+ "zmij",
 ]
 
 [[package]]
@@ -3911,12 +3710,13 @@ dependencies = [
 
 [[package]]
 name = "lance"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-arith",
  "arrow-array",
  "arrow-buffer",
+ "arrow-cast",
  "arrow-ipc",
  "arrow-ord",
  "arrow-row",
@@ -3954,20 +3754,21 @@ dependencies = [
  "lance-linalg",
  "lance-namespace",
  "lance-table",
+ "lance-tokenizer",
  "log",
  "moka",
  "object_store",
  "permutation",
  "pin-project",
  "prost",
+ "prost-build",
  "prost-types",
- "rand 0.9.2",
+ "rand 0.9.4",
  "roaring",
  "semver",
  "serde",
  "serde_json",
  "snafu",
- "tantivy",
  "tokio",
  "tokio-stream",
  "tokio-util",
@@ -3978,12 +3779,13 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-cast",
  "arrow-data",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
@@ -3993,12 +3795,12 @@ dependencies = [
  "half",
  "jsonb",
  "num-traits",
- "rand 0.9.2",
+ "rand 0.9.4",
 ]
 
 [[package]]
 name = "lance-bitpacking"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrayref",
  "paste",
@@ -4007,7 +3809,7 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4030,7 +3832,7 @@ dependencies = [
  "object_store",
  "pin-project",
  "prost",
- "rand 0.9.2",
+ "rand 0.9.4",
  "roaring",
  "serde_json",
  "snafu",
@@ -4044,11 +3846,12 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-array",
  "arrow-buffer",
+ "arrow-cast",
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
@@ -4077,7 +3880,7 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4087,15 +3890,15 @@ dependencies = [
  "futures",
  "half",
  "hex",
- "rand 0.9.2",
- "rand_distr 0.5.1",
+ "rand 0.9.4",
+ "rand_distr",
  "rand_xoshiro",
  "random_word",
 ]
 
 [[package]]
 name = "lance-encoding"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4121,7 +3924,7 @@ dependencies = [
  "prost",
  "prost-build",
  "prost-types",
- "rand 0.9.2",
+ "rand 0.9.4",
  "snafu",
  "strum 0.26.3",
  "tokio",
@@ -4132,7 +3935,7 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4164,7 +3967,7 @@ dependencies = [
 
 [[package]]
 name = "lance-geo"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "datafusion",
  "geo-traits",
@@ -4178,7 +3981,7 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4192,6 +3995,7 @@ dependencies = [
  "bitpacking",
  "bitvec",
  "bytes",
+ "chrono",
  "crossbeam-queue",
  "datafusion",
  "datafusion-common",
@@ -4219,9 +4023,8 @@ dependencies = [
  "lance-io",
  "lance-linalg",
  "lance-table",
+ "lance-tokenizer",
  "libm",
- "lindera",
- "lindera-tantivy",
  "log",
  "ndarray",
  "num-traits",
@@ -4229,8 +4032,8 @@ dependencies = [
  "prost",
  "prost-build",
  "prost-types",
- "rand 0.9.2",
- "rand_distr 0.5.1",
+ "rand 0.9.4",
+ "rand_distr",
  "rangemap",
  "rayon",
  "roaring",
@@ -4238,7 +4041,6 @@ dependencies = [
  "serde_json",
  "smallvec",
  "snafu",
- "tantivy",
  "tempfile",
  "tokio",
  "tracing",
@@ -4248,7 +4050,7 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4268,17 +4070,20 @@ dependencies = [
  "deepsize",
  "futures",
  "http 1.4.0",
+ "io-uring",
  "lance-arrow",
  "lance-core",
  "lance-namespace",
+ "libc",
  "log",
+ "moka",
  "object_store",
  "object_store_opendal",
  "opendal",
  "path_abs",
  "pin-project",
  "prost",
- "rand 0.9.2",
+ "rand 0.9.4",
  "serde",
  "snafu",
  "tempfile",
@@ -4289,7 +4094,7 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4300,24 +4105,25 @@ dependencies = [
  "lance-arrow",
  "lance-core",
  "num-traits",
- "rand 0.9.2",
+ "rand 0.9.4",
 ]
 
 [[package]]
 name = "lance-namespace"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "async-trait",
  "bytes",
  "lance-core",
  "lance-namespace-reqwest-client",
+ "serde",
  "snafu",
 ]
 
 [[package]]
 name = "lance-namespace-impls"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -4331,11 +4137,12 @@ dependencies = [
  "lance-core",
  "lance-index",
  "lance-io",
+ "lance-linalg",
  "lance-namespace",
  "lance-table",
  "log",
  "object_store",
- "rand 0.9.2",
+ "rand 0.9.4",
  "reqwest",
  "serde",
  "serde_json",
@@ -4348,9 +4155,9 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-reqwest-client"
-version = "0.5.2"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ad4c947349acd6e37e984eba0254588bd894e6128434338b9e6904e56fb4633"
+checksum = "ee2e48de899e2931afb67fcddd0a08e439bf5d8b6ea2a2ed9cb8f4df669bd5cc"
 dependencies = [
  "reqwest",
  "serde",
@@ -4361,7 +4168,7 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4385,7 +4192,7 @@ dependencies = [
  "prost",
  "prost-build",
  "prost-types",
- "rand 0.9.2",
+ "rand 0.9.4",
  "rangemap",
  "roaring",
  "semver",
@@ -4398,6 +4205,17 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "lance-tokenizer"
+version = "6.0.0-beta.1"
+dependencies = [
+ "jieba-rs",
+ "lindera",
+ "rust-stemmers",
+ "serde",
+ "unicode-normalization",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.5.0"
@@ -4413,12 +4231,6 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
 
-[[package]]
-name = "levenshtein_automata"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
-
 [[package]]
 name = "lexical-core"
 version = "1.0.6"
@@ -4484,33 +4296,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7"
 
 [[package]]
 name = "libc"
-version = "0.2.182"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
-
-[[package]]
-name = "libflate"
-version = "2.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3248b8d211bd23a104a42d81b4fa8bb8ac4a3b75e7a43d85d2c9ccb6179cd74"
-dependencies = [
- "adler32",
- "core2",
- "crc32fast",
- "dary_heap",
- "libflate_lz77",
-]
-
-[[package]]
-name = "libflate_lz77"
-version = "2.2.0"
+version = "0.2.185"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a599cb10a9cd92b1300debcef28da8f70b935ec937f44fcd1b70a7c986a11c5c"
-dependencies = [
- "core2",
- "hashbrown 0.16.1",
- "rle-decode-fast",
-]
+checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"
 
 [[package]]
 name = "libloading"
@@ -4533,9 +4321,9 @@ dependencies = [
 
 [[package]]
 name = "liblzma-sys"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f2db66f3268487b5033077f266da6777d057949b8f93c8ad82e441df25e6186"
+checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6"
 dependencies = [
  "cc",
  "libc",
@@ -4550,14 +4338,14 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 
 [[package]]
 name = "libredox"
-version = "0.1.14"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a"
+checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "libc",
  "plain",
- "redox_syscall 0.7.3",
+ "redox_syscall 0.7.4",
 ]
 
 [[package]]
@@ -4623,7 +4411,7 @@ dependencies = [
  "md5",
  "memmap2",
  "once_cell",
- "rand 0.9.2",
+ "rand 0.9.4",
  "reqwest",
  "serde",
  "tar",
@@ -4672,18 +4460,7 @@ dependencies = [
 ]
 
 [[package]]
-name = "lindera-tantivy"
-version = "0.44.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d0d17afa51b4f4cd2f3e50d4c0845800f3a774ed7f706612fbd91eba0c84b41"
-dependencies = [
- "lindera",
- "tantivy",
- "tantivy-tokenizer-api",
-]
-
-[[package]]
-name = "lindera-unidic"
+name = "lindera-unidic"
 version = "0.44.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "52672945166c14276bbba25e4ec79d7e126db1b503c0a6aa07ffc0141ae15cfa"
@@ -4695,12 +4472,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "linux-raw-sys"
-version = "0.4.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
-
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -4709,9 +4480,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
 
 [[package]]
 name = "litemap"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
+checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
 
 [[package]]
 name = "lock_api"
@@ -4741,15 +4512,6 @@ dependencies = [
  "tracing-subscriber",
 ]
 
-[[package]]
-name = "lru"
-version = "0.12.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
-dependencies = [
- "hashbrown 0.15.5",
-]
-
 [[package]]
 name = "lru-slab"
 version = "0.1.2"
@@ -4777,15 +4539,9 @@ dependencies = [
 
 [[package]]
 name = "lz4_flex"
-version = "0.11.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a"
-
-[[package]]
-name = "lz4_flex"
-version = "0.12.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e"
+checksum = "98c23545df7ecf1b16c303910a69b079e8e251d60f7dd2cc9b4177f2afaf1746"
 dependencies = [
  "twox-hash",
 ]
@@ -4834,15 +4590,6 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0"
 
-[[package]]
-name = "measure_time"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e"
-dependencies = [
- "log",
-]
-
 [[package]]
 name = "memchr"
 version = "2.8.0"
@@ -4883,12 +4630,6 @@ dependencies = [
  "unicase",
 ]
 
-[[package]]
-name = "minimal-lexical"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
-
 [[package]]
 name = "miniz_oxide"
 version = "0.8.9"
@@ -4901,9 +4642,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "1.1.1"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc"
+checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
 dependencies = [
  "libc",
  "wasi",
@@ -4918,9 +4659,9 @@ checksum = "dce6dd36094cac388f119d2e9dc82dc730ef91c32a6222170d630e5414b956e6"
 
 [[package]]
 name = "moka"
-version = "0.12.14"
+version = "0.12.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85f8024e1c8e71c778968af91d43700ce1d11b219d127d79fb2934153b82b42b"
+checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046"
 dependencies = [
  "async-lock",
  "crossbeam-channel",
@@ -4942,12 +4683,6 @@ version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
 
-[[package]]
-name = "murmurhash32"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
-
 [[package]]
 name = "ndarray"
 version = "0.16.1"
@@ -4963,16 +4698,6 @@ dependencies = [
  "rawpointer",
 ]
 
-[[package]]
-name = "nom"
-version = "7.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
-dependencies = [
- "memchr",
- "minimal-lexical",
-]
-
 [[package]]
 name = "nom"
 version = "8.0.0"
@@ -5028,9 +4753,9 @@ dependencies = [
 
 [[package]]
 name = "num-conv"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
+checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
 
 [[package]]
 name = "num-integer"
@@ -5074,9 +4799,9 @@ dependencies = [
 
 [[package]]
 name = "num_enum"
-version = "0.7.5"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c"
+checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26"
 dependencies = [
  "num_enum_derive",
  "rustversion",
@@ -5084,9 +4809,9 @@ dependencies = [
 
 [[package]]
 name = "num_enum_derive"
-version = "0.7.5"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7"
+checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8"
 dependencies = [
  "proc-macro-crate",
  "proc-macro2",
@@ -5119,13 +4844,13 @@ dependencies = [
  "http-body-util",
  "httparse",
  "humantime",
- "hyper 1.8.1",
+ "hyper",
  "itertools 0.14.0",
  "md-5",
  "parking_lot",
  "percent-encoding",
  "quick-xml 0.38.4",
- "rand 0.9.2",
+ "rand 0.9.4",
  "reqwest",
  "ring",
  "rustls-pemfile",
@@ -5159,9 +4884,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.21.3"
+version = "1.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "once_cell_polyfill"
@@ -5169,12 +4894,6 @@ version = "1.70.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
 
-[[package]]
-name = "oneshot"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
-
 [[package]]
 name = "opendal"
 version = "0.55.0"
@@ -5228,9 +4947,9 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "5.1.0"
+version = "5.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d"
+checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e"
 dependencies = [
  "num-traits",
 ]
@@ -5251,15 +4970,6 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
 
-[[package]]
-name = "ownedbytes"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e"
-dependencies = [
- "stable_deref_trait",
-]
-
 [[package]]
 name = "parking"
 version = "2.2.1"
@@ -5311,7 +5021,7 @@ dependencies = [
  "futures",
  "half",
  "hashbrown 0.16.1",
- "lz4_flex 0.12.0",
+ "lz4_flex",
  "num-bigint",
  "num-integer",
  "num-traits",
@@ -5563,9 +5273,9 @@ dependencies = [
 
 [[package]]
 name = "pkg-config"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
 
 [[package]]
 name = "plain"
@@ -5581,18 +5291,18 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
 
 [[package]]
 name = "portable-atomic-util"
-version = "0.2.5"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5"
+checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3"
 dependencies = [
  "portable-atomic",
 ]
 
 [[package]]
 name = "potential_utf"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77"
+checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"
 dependencies = [
  "zerovec",
 ]
@@ -5624,37 +5334,13 @@ dependencies = [
 
 [[package]]
 name = "proc-macro-crate"
-version = "3.4.0"
+version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983"
+checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
 dependencies = [
  "toml_edit",
 ]
 
-[[package]]
-name = "proc-macro-error"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
-dependencies = [
- "proc-macro-error-attr",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
- "version_check",
-]
-
-[[package]]
-name = "proc-macro-error-attr"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
-dependencies = [
- "proc-macro2",
- "quote",
- "version_check",
-]
-
 [[package]]
 name = "proc-macro2"
 version = "1.0.106"
@@ -5727,10 +5413,11 @@ dependencies = [
 
 [[package]]
 name = "pylance"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 dependencies = [
  "arrow",
  "arrow-array",
+ "arrow-cast",
  "arrow-data",
  "arrow-schema",
  "async-trait",
@@ -5880,8 +5567,8 @@ dependencies = [
  "quinn-proto",
  "quinn-udp",
  "rustc-hash",
- "rustls 0.23.37",
- "socket2 0.6.2",
+ "rustls",
+ "socket2",
  "thiserror 2.0.18",
  "tokio",
  "tracing",
@@ -5890,17 +5577,17 @@ dependencies = [
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.13"
+version = "0.11.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31"
+checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
 dependencies = [
  "bytes",
  "getrandom 0.3.4",
  "lru-slab",
- "rand 0.9.2",
+ "rand 0.9.4",
  "ring",
  "rustc-hash",
- "rustls 0.23.37",
+ "rustls",
  "rustls-pki-types",
  "slab",
  "thiserror 2.0.18",
@@ -5918,7 +5605,7 @@ dependencies = [
  "cfg_aliases",
  "libc",
  "once_cell",
- "socket2 0.6.2",
+ "socket2",
  "tracing",
  "windows-sys 0.60.2",
 ]
@@ -5963,9 +5650,9 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.9.2"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
+checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
 dependencies = [
  "rand_chacha 0.9.0",
  "rand_core 0.9.5",
@@ -6009,16 +5696,6 @@ dependencies = [
  "getrandom 0.3.4",
 ]
 
-[[package]]
-name = "rand_distr"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
-dependencies = [
- "num-traits",
- "rand 0.8.5",
-]
-
 [[package]]
 name = "rand_distr"
 version = "0.5.1"
@@ -6026,7 +5703,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
 dependencies = [
  "num-traits",
- "rand 0.9.2",
+ "rand 0.9.4",
 ]
 
 [[package]]
@@ -6047,7 +5724,7 @@ dependencies = [
  "ahash",
  "brotli",
  "paste",
- "rand 0.9.2",
+ "rand 0.9.4",
  "unicase",
 ]
 
@@ -6065,9 +5742,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
 
 [[package]]
 name = "rayon"
-version = "1.11.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
 dependencies = [
  "either",
  "rayon-core",
@@ -6109,16 +5786,16 @@ version = "0.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
 ]
 
 [[package]]
 name = "redox_syscall"
-version = "0.7.3"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16"
+checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
 ]
 
 [[package]]
@@ -6229,12 +5906,12 @@ dependencies = [
  "encoding_rs",
  "futures-core",
  "futures-util",
- "h2 0.4.13",
+ "h2",
  "http 1.4.0",
  "http-body 1.0.1",
  "http-body-util",
- "hyper 1.8.1",
- "hyper-rustls 0.27.7",
+ "hyper",
+ "hyper-rustls",
  "hyper-util",
  "js-sys",
  "log",
@@ -6243,7 +5920,7 @@ dependencies = [
  "percent-encoding",
  "pin-project-lite",
  "quinn",
- "rustls 0.23.37",
+ "rustls",
  "rustls-native-certs",
  "rustls-pki-types",
  "serde",
@@ -6251,7 +5928,7 @@ dependencies = [
  "serde_urlencoded",
  "sync_wrapper",
  "tokio",
- "tokio-rustls 0.26.4",
+ "tokio-rustls",
  "tokio-util",
  "tower",
  "tower-http 0.6.8",
@@ -6278,12 +5955,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "rle-decode-fast"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
-
 [[package]]
 name = "roaring"
 version = "0.11.3"
@@ -6354,9 +6025,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-hash"
-version = "2.1.1"
+version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
 
 [[package]]
 name = "rustc_version"
@@ -6367,55 +6038,30 @@ dependencies = [
  "semver",
 ]
 
-[[package]]
-name = "rustix"
-version = "0.38.44"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
-dependencies = [
- "bitflags 2.11.0",
- "errno",
- "libc",
- "linux-raw-sys 0.4.15",
- "windows-sys 0.59.0",
-]
-
 [[package]]
 name = "rustix"
 version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "errno",
  "libc",
- "linux-raw-sys 0.12.1",
+ "linux-raw-sys",
  "windows-sys 0.61.2",
 ]
 
 [[package]]
 name = "rustls"
-version = "0.21.12"
+version = "0.23.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e"
-dependencies = [
- "log",
- "ring",
- "rustls-webpki 0.101.7",
- "sct",
-]
-
-[[package]]
-name = "rustls"
-version = "0.23.37"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
+checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21"
 dependencies = [
  "aws-lc-rs",
  "once_cell",
  "ring",
  "rustls-pki-types",
- "rustls-webpki 0.103.9",
+ "rustls-webpki",
  "subtle",
  "zeroize",
 ]
@@ -6453,19 +6099,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.101.7"
+version = "0.103.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
-dependencies = [
- "ring",
- "untrusted",
-]
-
-[[package]]
-name = "rustls-webpki"
-version = "0.103.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53"
+checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06"
 dependencies = [
  "aws-lc-rs",
  "ring",
@@ -6505,9 +6141,9 @@ dependencies = [
 
 [[package]]
 name = "schannel"
-version = "0.1.28"
+version = "0.1.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1"
+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
 dependencies = [
  "windows-sys 0.61.2",
 ]
@@ -6559,23 +6195,13 @@ dependencies = [
  "sha2",
 ]
 
-[[package]]
-name = "sct"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
-dependencies = [
- "ring",
- "untrusted",
-]
-
 [[package]]
 name = "security-framework"
 version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "core-foundation",
  "core-foundation-sys",
  "libc",
@@ -6594,9 +6220,9 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.27"
+version = "1.0.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
 dependencies = [
  "serde",
  "serde_core",
@@ -6728,7 +6354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "digest",
 ]
 
@@ -6739,7 +6365,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "digest",
 ]
 
@@ -6780,9 +6406,9 @@ dependencies = [
 
 [[package]]
 name = "simd-adler32"
-version = "0.3.8"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
+checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
 
 [[package]]
 name = "simdutf8"
@@ -6808,15 +6434,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
 
-[[package]]
-name = "sketches-ddsketch"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "slab"
 version = "0.4.12"
@@ -6858,31 +6475,21 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
 
 [[package]]
 name = "socket2"
-version = "0.5.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
-dependencies = [
- "libc",
- "windows-sys 0.52.0",
-]
-
-[[package]]
-name = "socket2"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0"
+checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
 dependencies = [
  "libc",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
 name = "spade"
-version = "2.15.0"
+version = "2.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb313e1c8afee5b5647e00ee0fe6855e3d529eb863a0fdae1d60006c4d1e9990"
+checksum = "9699399fd9349b00b184f5635b074f9ec93afffef30c853f8c875b32c0f8c7fa"
 dependencies = [
- "hashbrown 0.15.5",
+ "hashbrown 0.16.1",
  "num-traits",
  "robust",
  "smallvec",
@@ -7085,152 +6692,6 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
 
-[[package]]
-name = "tantivy"
-version = "0.24.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64a966cb0e76e311f09cf18507c9af192f15d34886ee43d7ba7c7e3803660c43"
-dependencies = [
- "aho-corasick",
- "arc-swap",
- "base64",
- "bitpacking",
- "bon",
- "byteorder",
- "census",
- "crc32fast",
- "crossbeam-channel",
- "downcast-rs",
- "fastdivide",
- "fnv",
- "fs4",
- "htmlescape",
- "hyperloglogplus",
- "itertools 0.14.0",
- "levenshtein_automata",
- "log",
- "lru",
- "lz4_flex 0.11.5",
- "measure_time",
- "memmap2",
- "once_cell",
- "oneshot",
- "rayon",
- "regex",
- "rust-stemmers",
- "rustc-hash",
- "serde",
- "serde_json",
- "sketches-ddsketch",
- "smallvec",
- "tantivy-bitpacker",
- "tantivy-columnar",
- "tantivy-common",
- "tantivy-fst",
- "tantivy-query-grammar",
- "tantivy-stacker",
- "tantivy-tokenizer-api",
- "tempfile",
- "thiserror 2.0.18",
- "time",
- "uuid",
- "winapi",
-]
-
-[[package]]
-name = "tantivy-bitpacker"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1adc286a39e089ae9938935cd488d7d34f14502544a36607effd2239ff0e2494"
-dependencies = [
- "bitpacking",
-]
-
-[[package]]
-name = "tantivy-columnar"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6300428e0c104c4f7db6f95b466a6f5c1b9aece094ec57cdd365337908dc7344"
-dependencies = [
- "downcast-rs",
- "fastdivide",
- "itertools 0.14.0",
- "serde",
- "tantivy-bitpacker",
- "tantivy-common",
- "tantivy-sstable",
- "tantivy-stacker",
-]
-
-[[package]]
-name = "tantivy-common"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e91b6ea6090ce03dc72c27d0619e77185d26cc3b20775966c346c6d4f7e99d7f"
-dependencies = [
- "async-trait",
- "byteorder",
- "ownedbytes",
- "serde",
- "time",
-]
-
-[[package]]
-name = "tantivy-fst"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
-dependencies = [
- "byteorder",
- "regex-syntax",
- "utf8-ranges",
-]
-
-[[package]]
-name = "tantivy-query-grammar"
-version = "0.24.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e810cdeeebca57fc3f7bfec5f85fdbea9031b2ac9b990eb5ff49b371d52bbe6a"
-dependencies = [
- "nom 7.1.3",
- "serde",
- "serde_json",
-]
-
-[[package]]
-name = "tantivy-sstable"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "709f22c08a4c90e1b36711c1c6cad5ae21b20b093e535b69b18783dd2cb99416"
-dependencies = [
- "futures-util",
- "itertools 0.14.0",
- "tantivy-bitpacker",
- "tantivy-common",
- "tantivy-fst",
- "zstd",
-]
-
-[[package]]
-name = "tantivy-stacker"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bcdebb267671311d1e8891fd9d1301803fdb8ad21ba22e0a30d0cab49ba59c1"
-dependencies = [
- "murmurhash32",
- "rand_distr 0.4.3",
- "tantivy-common",
-]
-
-[[package]]
-name = "tantivy-tokenizer-api"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfa942fcee81e213e09715bbce8734ae2180070b97b33839a795ba1de201547d"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "tap"
 version = "1.0.1"
@@ -7239,9 +6700,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "tar"
-version = "0.4.44"
+version = "0.4.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"
+checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973"
 dependencies = [
  "filetime",
  "libc",
@@ -7256,14 +6717,14 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tempfile"
-version = "3.26.0"
+version = "3.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0"
+checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
  "getrandom 0.4.2",
  "once_cell",
- "rustix 1.1.4",
+ "rustix",
  "windows-sys 0.61.2",
 ]
 
@@ -7378,9 +6839,9 @@ dependencies = [
 
 [[package]]
 name = "tinystr"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869"
+checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
 dependencies = [
  "displaydoc",
  "zerovec",
@@ -7388,9 +6849,9 @@ dependencies = [
 
 [[package]]
 name = "tinyvec"
-version = "1.10.0"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
+checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
 dependencies = [
  "tinyvec_macros",
 ]
@@ -7403,9 +6864,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.50.0"
+version = "1.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d"
+checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776"
 dependencies = [
  "bytes",
  "libc",
@@ -7413,39 +6874,29 @@ dependencies = [
  "parking_lot",
  "pin-project-lite",
  "signal-hook-registry",
- "socket2 0.6.2",
+ "socket2",
  "tokio-macros",
  "windows-sys 0.61.2",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "2.6.1"
+version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c"
+checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
 dependencies = [
  "proc-macro2",
  "quote",
  "syn 2.0.117",
 ]
 
-[[package]]
-name = "tokio-rustls"
-version = "0.24.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
-dependencies = [
- "rustls 0.21.12",
- "tokio",
-]
-
 [[package]]
 name = "tokio-rustls"
 version = "0.26.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
 dependencies = [
- "rustls 0.23.37",
+ "rustls",
  "tokio",
 ]
 
@@ -7475,18 +6926,18 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "0.7.5+spec-1.1.0"
+version = "1.1.1+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
 dependencies = [
  "serde_core",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.23.10+spec-1.0.0"
+version = "0.25.11+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269"
+checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b"
 dependencies = [
  "indexmap",
  "toml_datetime",
@@ -7496,9 +6947,9 @@ dependencies = [
 
 [[package]]
 name = "toml_parser"
-version = "1.0.9+spec-1.1.0"
+version = "1.1.2+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4"
+checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
 dependencies = [
  "winnow",
 ]
@@ -7525,7 +6976,7 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "bytes",
  "http 1.4.0",
  "http-body 1.0.1",
@@ -7543,7 +6994,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
 dependencies = [
  "async-compression",
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "bytes",
  "futures-core",
  "futures-util",
@@ -7628,9 +7079,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-subscriber"
-version = "0.3.22"
+version = "0.3.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e"
+checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
 dependencies = [
  "matchers",
  "nu-ansi-term",
@@ -7671,7 +7122,7 @@ version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
 dependencies = [
- "rand 0.9.2",
+ "rand 0.9.4",
 ]
 
 [[package]]
@@ -7688,9 +7139,9 @@ checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
 
 [[package]]
 name = "typewit"
-version = "1.14.2"
+version = "1.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8c1ae7cc0fdb8b842d65d127cb981574b0d2b249b74d1c7a2986863dc134f71"
+checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6"
 
 [[package]]
 name = "typify"
@@ -7768,9 +7219,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.12.0"
+version = "1.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
+checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c"
 
 [[package]]
 name = "unicode-width"
@@ -7846,9 +7297,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.21.0"
+version = "1.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb"
+checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9"
 dependencies = [
  "getrandom 0.4.2",
  "js-sys",
@@ -7925,9 +7376,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.114"
+version = "0.2.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e"
+checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -7938,23 +7389,19 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.64"
+version = "0.4.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8"
+checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8"
 dependencies = [
- "cfg-if",
- "futures-util",
  "js-sys",
- "once_cell",
  "wasm-bindgen",
- "web-sys",
 ]
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.114"
+version = "0.2.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6"
+checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -7962,9 +7409,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.114"
+version = "0.2.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3"
+checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904"
 dependencies = [
  "bumpalo",
  "proc-macro2",
@@ -7975,9 +7422,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.114"
+version = "0.2.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16"
+checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129"
 dependencies = [
  "unicode-ident",
 ]
@@ -8023,7 +7470,7 @@ version = "0.244.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "hashbrown 0.15.5",
  "indexmap",
  "semver",
@@ -8031,9 +7478,9 @@ dependencies = [
 
 [[package]]
 name = "web-sys"
-version = "0.3.91"
+version = "0.3.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9"
+checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -8315,9 +7762,9 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
 
 [[package]]
 name = "winnow"
-version = "0.7.14"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829"
+checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5"
 dependencies = [
  "memchr",
 ]
@@ -8380,7 +7827,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
 dependencies = [
  "anyhow",
- "bitflags 2.11.0",
+ "bitflags 2.11.1",
  "indexmap",
  "log",
  "serde",
@@ -8437,9 +7884,9 @@ dependencies = [
 
 [[package]]
 name = "writeable"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
+checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
 
 [[package]]
 name = "wyz"
@@ -8457,7 +7904,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
 dependencies = [
  "libc",
- "rustix 1.1.4",
+ "rustix",
 ]
 
 [[package]]
@@ -8480,9 +7927,9 @@ checksum = "aed111bd9e48a802518765906cbdadf0b45afb72b9c81ab049a3b86252adffdd"
 
 [[package]]
 name = "yoke"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954"
+checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
 dependencies = [
  "stable_deref_trait",
  "yoke-derive",
@@ -8491,9 +7938,9 @@ dependencies = [
 
 [[package]]
 name = "yoke-derive"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
+checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -8503,18 +7950,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.40"
+version = "0.8.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5"
+checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.40"
+version = "0.8.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953"
+checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -8523,18 +7970,18 @@ dependencies = [
 
 [[package]]
 name = "zerofrom"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"
+checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df"
 dependencies = [
  "zerofrom-derive",
 ]
 
 [[package]]
 name = "zerofrom-derive"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
+checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -8550,9 +7997,9 @@ checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
 
 [[package]]
 name = "zerotrie"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851"
+checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
 dependencies = [
  "displaydoc",
  "yoke",
@@ -8561,9 +8008,9 @@ dependencies = [
 
 [[package]]
 name = "zerovec"
-version = "0.11.5"
+version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002"
+checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
 dependencies = [
  "yoke",
  "zerofrom",
@@ -8572,9 +8019,9 @@ dependencies = [
 
 [[package]]
 name = "zerovec-derive"
-version = "0.11.2"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
+checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
 dependencies = [
  "proc-macro2",
  "quote",
diff --git a/python/Cargo.toml b/python/Cargo.toml
index 7dbc96f9ecb..4a242f6bd9c 100644
--- a/python/Cargo.toml
+++ b/python/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pylance"
-version = "4.0.0-beta.7"
+version = "6.0.0-beta.1"
 edition = "2024"
 authors = ["Lance Devs <dev@lance.org>"]
 license = "Apache-2.0"
@@ -15,6 +15,7 @@ crate-type = ["cdylib"]
 [dependencies]
 arrow = { version = "57.0.0", features = ["pyarrow"] }
 arrow-array = "57.0.0"
+arrow-cast = "57.0.0"
 arrow-data = "57.0.0"
 arrow-schema = "57.0.0"
 object_store = "0.12.4"
diff --git a/python/DEVELOPMENT.md b/python/DEVELOPMENT.md
index 9effb0fb456..12c56549608 100644
--- a/python/DEVELOPMENT.md
+++ b/python/DEVELOPMENT.md
@@ -1,5 +1,15 @@
 # Development
 
+For local development, prefer [uv](https://docs.astral.sh/uv/) to create and manage the Python environment:
+
+```shell
+uv sync --extra tests --extra dev
+```
+
+Add extras such as `benchmarks`, `torch`, or `geo` only when you need them. After the environment is initialized, either activate it or use `uv run ...` for commands.
+
+`uv sync` is not just downloading Python packages here. It also builds the local `pylance` Rust extension as part of the editable environment, so the first run, cache misses, or Rust dependency changes can make it noticeably slow. This is expected; let the build finish instead of interrupting it and switching to a different environment setup.
+
 ## Building the project
 
 This project is built with [maturin](https://github.com/PyO3/maturin).
@@ -7,31 +17,27 @@ This project is built with [maturin](https://github.com/PyO3/maturin).
 It can be built in development mode with:
 
 ```shell
-maturin develop
+uv run maturin develop
 ```
 
 This builds the Rust native module in place. You will need to re-run this
 whenever you change the Rust code. But changing the Python code doesn't require
 re-building.
 
-## Running tests
-
-To run the tests, first install the test packages:
+As with `uv sync`, this may take a while because it is compiling Rust code for the local `pylance` extension. Slow builds are expected here, especially after dependency or toolchain changes.
 
-```shell
-pip install '.[tests]'
-```
+## Running tests
 
-then:
+To run the tests:
 
 ```shell
-make test
+uv run make test
 ```
 
 To check the documentation examples, use
 
 ```shell
-make doctest
+uv run make doctest
 ```
 
 ## Formatting and linting
@@ -39,7 +45,7 @@ make doctest
 To run formatters, run:
 
 ```shell
-make format
+uv run make format
 ```
 
 (To run for just Python or just Rust, use `make format-python` or `cargo fmt`.)
@@ -47,7 +53,7 @@ make format
 To run format checker and linters, run:
 
 ```shell
-make lint
+uv run make lint
 ```
 
 (To run for just Python or just Rust, use `make lint-python` or `make lint-rust`.)
@@ -59,7 +65,7 @@ then you can use the pre-commit tool. The project includes a pre-commit config
 file already. First, install the pre-commit tool:
 
 ```shell
-pip install pre-commit
+uv tool install pre-commit
 ```
 
 Then install the hooks:
@@ -98,7 +104,8 @@ benchmarks added there should run in less than 5 seconds.
 Before running benchmarks, you should build pylance in release mode:
 
 ```shell
-maturin develop --profile release-with-debug --extras benchmarks --features datagen
+uv sync --extra tests --extra dev --extra benchmarks
+uv run maturin develop --profile release-with-debug --extras benchmarks --features datagen
 ```
 
 (You can also use `--release` or `--profile release`, but `--profile release-with-debug`
@@ -107,7 +114,7 @@ will provide debug symbols for profiling.)
 Then you can run the benchmarks with
 
 ```shell
-pytest python/benchmarks -m "not slow"
+uv run pytest python/benchmarks -m "not slow"
 ```
 
 Note: the first time you run the benchmarks, they may take a while, since they
@@ -118,7 +125,7 @@ Some benchmarks are especially slow, so they are skipped `-m "not slow"`. To run
 the slow benchmarks, use:
 
 ```shell
-pytest python/benchmarks
+uv run pytest python/benchmarks
 ```
 
 ### Run a particular benchmark
@@ -127,7 +134,7 @@ To filter benchmarks by name, use the usual pytest `-k` flag (this can be a
 substring match, so you don't need to type the full name):
 
 ```shell
-pytest python/benchmarks -k test_ivf_pq_index_search
+uv run pytest python/benchmarks -k test_ivf_pq_index_search
 ```
 
 ### Profile a benchmark
@@ -136,7 +143,7 @@ If you have [cargo-flamegraph](https://github.com/flamegraph-rs/flamegraph)
 installed, you can create a flamegraph of a benchmark by running:
 
 ```shell
-flamegraph -F 100 --no-inline -- $(which python) \
+flamegraph -F 100 --no-inline -- $(uv run which python) \
     -m pytest python/benchmarks \
     --benchmark-min-time=2 \
     -k test_ivf_pq_index_search
@@ -165,12 +172,14 @@ the benchmarks again with `--benchmark-compare`.
 ```shell
 CURRENT_BRANCH=$(git branch --show-current)
 git checkout main
-maturin develop --profile release-with-debug  --features datagen
-pytest --benchmark-save=baseline python/benchmarks -m "not slow"
+uv sync --extra tests --extra dev --extra benchmarks
+uv run maturin develop --profile release-with-debug --features datagen
+uv run pytest --benchmark-save=baseline python/benchmarks -m "not slow"
 COMPARE_ID=$(ls .benchmarks/*/ | tail -1 | cut -c1-4)
 git checkout $CURRENT_BRANCH
-maturin develop --profile release-with-debug  --features datagen
-pytest --benchmark-compare=$COMPARE_ID python/benchmarks -m "not slow"
+uv sync --extra tests --extra dev --extra benchmarks
+uv run maturin develop --profile release-with-debug --features datagen
+uv run pytest --benchmark-compare=$COMPARE_ID python/benchmarks -m "not slow"
 ```
 
 ## Tracing
diff --git a/python/pyproject.toml b/python/pyproject.toml
index f105144e933..613ff5b8d5b 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "pylance"
 dynamic = ["version"]
-dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.5.2"]
+dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.6.1"]
 description = "python wrapper for Lance columnar format"
 authors = [{ name = "Lance Devs", email = "dev@lance.org" }]
 license = { file = "LICENSE" }
diff --git a/python/python/ci_benchmarks/datagen/lineitems.py b/python/python/ci_benchmarks/datagen/lineitems.py
index 9a6bb27eaed..19e9e1fe745 100644
--- a/python/python/ci_benchmarks/datagen/lineitems.py
+++ b/python/python/ci_benchmarks/datagen/lineitems.py
@@ -21,14 +21,13 @@ def _gen_data(tmpdir: str, scale_factor: int):
     con.execute("INSTALL tpch; LOAD tpch")
     con.execute(f"CALL dbgen(sf={scale_factor})")
     res = con.query("SELECT * FROM lineitem")
-    return res.fetch_arrow_reader()
+    return res.to_arrow_reader()
 
 
 def _create(dataset_uri: str, data_storage_version: str, scale_factor: int = 10):
     tmpdir = tempfile.mkdtemp(prefix=f"tpch-scale-factor-{scale_factor}-")
     try:
         ds = lance.dataset(dataset_uri)
-        print(ds.count_rows())
         if ds.count_rows() == NUM_ROWS:
             return
         elif ds.count_rows() == 0:
diff --git a/python/python/lance/__init__.py b/python/python/lance/__init__.py
index d3f1e46a017..5b39bfcf2fa 100644
--- a/python/python/lance/__init__.py
+++ b/python/python/lance/__init__.py
@@ -6,7 +6,7 @@
 import logging
 import os
 import warnings
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
 from . import io, log
 from .blob import Blob, BlobArray, BlobColumn, BlobFile, blob_array, blob_field
@@ -14,6 +14,7 @@
     DataStatistics,
     FieldStatistics,
     Index,
+    IndexFile,
     LanceDataset,
     LanceOperation,
     LanceScanner,
@@ -35,8 +36,8 @@
 from .namespace import (
     DescribeTableRequest,
     LanceNamespace,
-    LanceNamespaceStorageOptionsProvider,
 )
+from .progress import IndexProgress
 from .schema import json_to_schema, schema_to_json
 from .util import sanitize_ts
 
@@ -62,11 +63,11 @@
     "FieldStatistics",
     "FragmentMetadata",
     "Index",
+    "IndexFile",
     "LanceDataset",
     "LanceFragment",
     "LanceOperation",
     "LanceScanner",
-    "LanceNamespaceStorageOptionsProvider",
     "MergeInsertBuilder",
     "ScanStatistics",
     "Transaction",
@@ -81,6 +82,7 @@
     "set_logger",
     "write_dataset",
     "FFILanceTableProvider",
+    "IndexProgress",
 ]
 
 
@@ -97,9 +99,8 @@ def dataset(
     index_cache_size_bytes: Optional[int] = None,
     read_params: Optional[Dict[str, any]] = None,
     session: Optional[Session] = None,
-    namespace: Optional[LanceNamespace] = None,
+    namespace_client: Optional[LanceNamespace] = None,
     table_id: Optional[List[str]] = None,
-    storage_options_provider: Optional[Any] = None,
 ) -> LanceDataset:
     """
     Opens the Lance dataset from the address specified.
@@ -109,7 +110,7 @@ def dataset(
     uri : str, optional
         Address to the Lance dataset. It can be a local file path `/tmp/data.lance`,
         or a cloud object store URI, i.e., `s3://bucket/data.lance`.
-        Either `uri` or (`namespace` + `table_id`) must be provided, but not both.
+        Either `uri` or (`namespace_client` + `table_id`) must be provided.
     version : optional, int | str
         If specified, load a specific version of the Lance dataset. Else, loads the
         latest version. A version number (`int`) or a tag (`str`) can be provided.
@@ -159,69 +160,64 @@ def dataset(
     session : optional, lance.Session
         A session to use for this dataset. This contains the caches used by the
         across multiple datasets.
-    namespace : optional, LanceNamespace
-        A namespace instance from which to fetch table location and storage options.
+    namespace_client : optional, LanceNamespace
+        A namespace client from which to fetch table location and storage options.
         Use lance.namespace.connect() to create a namespace instance.
         Must be provided together with `table_id`. Cannot be used with `uri`.
         When provided, the table location will be fetched automatically from the
         namespace via describe_table().
     table_id : optional, List[str]
         The table identifier when using a namespace (e.g., ["my_table"]).
-        Must be provided together with `namespace`. Cannot be used with `uri`.
-    storage_options_provider : optional
-        A storage options provider for automatic credential refresh. Must implement
-        `fetch_storage_options()` method that returns a dict of storage options.
-        If provided along with `namespace`, this takes precedence over the
-        namespace-created provider.
+        Must be provided together with `namespace_client`. Cannot be used with `uri`.
 
     Notes
     -----
-    When using `namespace` and `table_id`:
+    When using `namespace_client` and `table_id`:
     - The `uri` parameter is optional and will be fetched from the namespace
     - Storage options from describe_table() will be used automatically
     - A dynamic storage options provider will be created to refresh credentials
     - Initial storage options from describe_table() will be merged with
       any provided `storage_options`
     """
-    # Validate that user provides either uri OR (namespace + table_id), not both
+    # Validate that user provides either uri OR (namespace_client + table_id), not both
     has_uri = uri is not None
-    has_namespace = namespace is not None or table_id is not None
+    has_namespace = namespace_client is not None or table_id is not None
 
     if has_uri and has_namespace:
         raise ValueError(
-            "Cannot specify both 'uri' and 'namespace/table_id'. "
-            "Please provide either 'uri' or both 'namespace' and 'table_id'."
+            "Cannot specify both 'uri' and 'namespace_client/table_id'. "
+            "Please provide either 'uri' or both 'namespace_client' and 'table_id'."
         )
     elif not has_uri and not has_namespace:
         raise ValueError(
-            "Must specify either 'uri' or both 'namespace' and 'table_id'."
+            "Must specify either 'uri' or both 'namespace_client' and 'table_id'."
         )
 
     # Handle namespace resolution in Python
-    managed_versioning = False
-    if namespace is not None:
+    namespace_client_managed_versioning = False
+    if namespace_client is not None:
         if table_id is None:
             raise ValueError(
-                "Both 'namespace' and 'table_id' must be provided together."
+                "Both 'namespace_client' and 'table_id' must be provided together."
             )
 
         request = DescribeTableRequest(id=table_id, version=version)
-        response = namespace.describe_table(request)
+        response = namespace_client.describe_table(request)
 
         uri = response.location
         if uri is None:
             raise ValueError("Namespace did not return a 'location' for the table")
 
         # Check if namespace manages versioning (commits go through namespace API)
-        managed_versioning = getattr(response, "managed_versioning", None) is True
+        namespace_client_managed_versioning = (
+            getattr(response, "managed_versioning", None) is True
+        )
 
         namespace_storage_options = response.storage_options
 
-        if namespace_storage_options:
-            if storage_options_provider is None:
-                storage_options_provider = LanceNamespaceStorageOptionsProvider(
-                    namespace=namespace, table_id=table_id
-                )
+        # Merge namespace storage options with user-provided options
+        # Namespace options take precedence
+        if namespace_storage_options is not None:
             if storage_options is None:
                 storage_options = namespace_storage_options
             else:
@@ -229,7 +225,9 @@ def dataset(
                 merged_options.update(namespace_storage_options)
                 storage_options = merged_options
     elif table_id is not None:
-        raise ValueError("Both 'namespace' and 'table_id' must be provided together.")
+        raise ValueError(
+            "Both 'namespace_client' and 'table_id' must be provided together."
+        )
 
     ds = LanceDataset(
         uri,
@@ -243,9 +241,9 @@ def dataset(
         index_cache_size_bytes=index_cache_size_bytes,
         read_params=read_params,
         session=session,
-        storage_options_provider=storage_options_provider,
-        namespace=namespace if managed_versioning else None,
-        table_id=table_id if managed_versioning else None,
+        namespace_client=namespace_client,
+        table_id=table_id,
+        namespace_client_managed_versioning=namespace_client_managed_versioning,
     )
     if version is None and asof is not None:
         ts_cutoff = sanitize_ts(asof)
@@ -269,7 +267,9 @@ def dataset(
                 index_cache_size_bytes=index_cache_size_bytes,
                 read_params=read_params,
                 session=session,
-                storage_options_provider=storage_options_provider,
+                namespace_client=namespace_client,
+                table_id=table_id,
+                namespace_client_managed_versioning=namespace_client_managed_versioning,
             )
     else:
         return ds
diff --git a/python/python/lance/blob.py b/python/python/lance/blob.py
index 1a3f4e946fb..02adfdc95e3 100644
--- a/python/python/lance/blob.py
+++ b/python/python/lance/blob.py
@@ -282,6 +282,10 @@ def size(self) -> int:
     def readall(self) -> bytes:
         return self.inner.readall()
 
+    def read_range(self, offset: int, length: int) -> bytes:
+        """Read a blob-local byte range without changing the current cursor."""
+        return self.inner.read_range(offset, length)
+
     def readinto(self, b: bytearray) -> int:
         return self.inner.read_into(b)
 
diff --git a/python/python/lance/dataset.py b/python/python/lance/dataset.py
index fa013d6dcdd..79641ab4438 100644
--- a/python/python/lance/dataset.py
+++ b/python/python/lance/dataset.py
@@ -30,6 +30,7 @@
     Tuple,
     TypedDict,
     Union,
+    cast,
 )
 
 import pyarrow as pa
@@ -78,9 +79,8 @@
     from lance.namespace import LanceNamespace
 
     from .commit import CommitLock
-    from .io import StorageOptionsProvider
     from .lance.indices import IndexDescription
-    from .progress import FragmentWriteProgress
+    from .progress import FragmentWriteProgress, IndexProgress
     from .types import ReaderLike
 
     QueryVectorLike = Union[
@@ -91,6 +91,157 @@
         Iterable[float],
     ]
 LANCE_COMMIT_MESSAGE_KEY = "__lance_commit_message"
+_BLOB_PANDAS_MODE_LAZY = "lazy"
+_BLOB_PANDAS_MODE_BYTES = "bytes"
+_BLOB_PANDAS_MODE_DESCRIPTIONS = "descriptions"
+_BLOB_PANDAS_MODES = frozenset(
+    {
+        _BLOB_PANDAS_MODE_LAZY,
+        _BLOB_PANDAS_MODE_BYTES,
+        _BLOB_PANDAS_MODE_DESCRIPTIONS,
+    }
+)
+_BLOB_ROW_ADDR_COLUMN = "_rowaddr"
+
+
+def _field_metadata_value(field: pa.Field, key: str) -> Optional[bytes]:
+    metadata = field.metadata
+    if metadata is None:
+        return None
+    return metadata.get(key.encode("utf-8"))
+
+
+def _is_blob_field(field: pa.Field) -> bool:
+    return (
+        _field_metadata_value(field, "lance-encoding:blob") == b"true"
+        or _field_metadata_value(field, "ARROW:extension:name") == b"lance.blob.v2"
+    )
+
+
+def _blob_columns_in_schema(schema: pa.Schema) -> set[str]:
+    return {field.name for field in schema if _is_blob_field(field)}
+
+
+def _normalize_blob_pandas_mode(
+    blob_mode: str,
+) -> Literal["lazy", "bytes", "descriptions"]:
+    if blob_mode not in _BLOB_PANDAS_MODES:
+        raise ValueError("blob_mode must be one of: 'lazy', 'bytes', 'descriptions'")
+    return cast("Literal['lazy', 'bytes', 'descriptions']", blob_mode)
+
+
+def _simple_source_column(expr: str) -> Optional[str]:
+    expr = expr.strip()
+    if expr.startswith("`") and expr.endswith("`") and expr.count("`") == 2:
+        return expr[1:-1]
+    if "." in expr or any(ch.isspace() for ch in expr):
+        return None
+    if not expr:
+        return None
+    allowed = set("_$")
+    if not (expr[0].isalpha() or expr[0] == "_"):
+        return None
+    if any(not (ch.isalnum() or ch in allowed) for ch in expr[1:]):
+        return None
+    return expr
+
+
+def _blob_column_sources(
+    schema: pa.Schema,
+    snapshot: Dict[str, Any],
+    dataset_schema: pa.Schema,
+) -> dict[str, str]:
+    blob_columns = {}
+    output_blob_columns = [field.name for field in schema if _is_blob_field(field)]
+    columns_with_transform = snapshot.get("_columns_with_transform")
+    if not columns_with_transform:
+        return {name: name for name in output_blob_columns}
+
+    source_is_blob = {field.name for field in dataset_schema if _is_blob_field(field)}
+    for name in output_blob_columns:
+        expr = dict(columns_with_transform).get(name)
+        source = _simple_source_column(expr) if expr is not None else name
+        if source is None or source not in source_is_blob:
+            raise NotImplementedError(
+                "blob-aware to_pandas only supports direct blob column references "
+                "for transformed projections"
+            )
+        blob_columns[name] = source
+    return blob_columns
+
+
+def _snapshot_scanner_builder(builder: "ScannerBuilder") -> Dict[str, Any]:
+    """Capture Python-side scanner config needed to rebuild the scan later.
+
+    The native scanner object does not preserve the original builder arguments.
+    We need these values to recreate the same scan when `to_pandas` switches blob
+    handling modes or injects `_rowaddr` for lazy blob export.
+    """
+
+    def snapshot_value(value: Any) -> Any:
+        try:
+            return copy.deepcopy(value)
+        except (TypeError, AttributeError):
+            return value
+
+    return {
+        key: snapshot_value(value)
+        for key, value in vars(builder).items()
+        if key != "ds"
+    }
+
+
+def _scanner_from_snapshot(
+    ds: "LanceDataset", snapshot: Dict[str, Any]
+) -> "LanceScanner":
+    builder = ScannerBuilder(ds)
+    for key, value in snapshot.items():
+        if key == "_columns" and value is not None:
+            setattr(builder, key, list(value))
+        elif key == "_columns_with_transform" and value is not None:
+            setattr(builder, key, list(value))
+        elif key == "_fragments" and value is not None:
+            setattr(builder, key, list(value))
+        elif key == "_orderings" and value is not None:
+            setattr(builder, key, list(value))
+        else:
+            setattr(builder, key, value)
+    return builder.to_scanner()
+
+
+def _is_null_blob_description(description: Any) -> bool:
+    if description is None:
+        return True
+    if not isinstance(description, dict):
+        return False
+    if description.keys() == {"position", "size"}:
+        return description["position"] == 1 and description["size"] == 0
+    if description.keys() == {"kind", "position", "size", "blob_id", "blob_uri"}:
+        return (
+            description["kind"] == 0
+            and description["position"] == 0
+            and description["size"] == 0
+            and description["blob_id"] == 0
+            and description["blob_uri"] == ""
+        )
+    return False
+
+
+def _resolve_blob_selection(
+    ids: Optional[Union[List[int], pa.Array]],
+    addresses: Optional[Union[List[int], pa.Array]],
+    indices: Optional[Union[List[int], pa.Array]],
+) -> Tuple[str, Union[List[int], pa.Array]]:
+    if sum([bool(v is not None) for v in [ids, addresses, indices]]) != 1:
+        raise ValueError("Exactly one of ids, indices, or addresses must be specified")
+
+    if ids is not None:
+        return "ids", ids
+    if addresses is not None:
+        return "addresses", addresses
+    if indices is not None:
+        return "indices", indices
+    raise ValueError("Either ids, addresses, or indices must be specified")
 
 
 class MergeInsertBuilder(_MergeInsertBuilder):
@@ -299,12 +450,12 @@ def explain_plan(
         >>> print(plan) # doctest: +ELLIPSIS
         MergeInsert: on=[id], when_matched=UpdateAll, when_not_matched=InsertAll, ...
           CoalescePartitionsExec
-            ProjectionExec: expr=[_rowid@1 as _rowid, _rowaddr@2 as _rowaddr, ...]
-              ProjectionExec: expr=[id@2 IS NOT NULL as __common_expr_1, ...]
-                HashJoinExec: mode=CollectLeft, join_type=Right, ...
-                  CooperativeExec
-                    LanceRead: uri=test_dataset/data, projection=[id], ...
-                  RepartitionExec: ...
+            ProjectionExec: expr=[...]
+              HashJoinExec: mode=CollectLeft, join_type=Right, ...
+                CooperativeExec
+                  LanceRead: uri=test_dataset/data, projection=[id], ...
+                RepartitionExec: ...
+                  ProjectionExec: expr=[..., true as __merge_source_sentinel]
                     StreamingTableExec: partition_sizes=1, ...
         <BLANKLINE>
 
@@ -318,10 +469,9 @@ def explain_plan(
         >>> print(plan) # doctest: +ELLIPSIS
         MergeInsert: on=[id], when_matched=UpdateAll, when_not_matched=InsertAll, ...
           CoalescePartitionsExec
-            ProjectionExec: expr=[_rowid@1 as _rowid, _rowaddr@2 as _rowaddr, ...]
-              ProjectionExec: expr=[id@2 IS NOT NULL as __common_expr_1, ...]
-                HashJoinExec: mode=CollectLeft, join_type=Right, ...
-                  ...
+            ProjectionExec: expr=[...]
+              HashJoinExec: mode=CollectLeft, join_type=Right, ...
+                ...
         """
         return super(MergeInsertBuilder, self).explain_plan(schema, verbose=verbose)
 
@@ -382,12 +532,12 @@ def analyze_plan(
         >>> print(analysis) # doctest: +ELLIPSIS
             MergeInsert: elapsed=..., on=[id], ..., metrics=[..., bytes_written=..., ...]
               CoalescePartitionsExec, elapsed=..., metrics=[output_rows=..., elapsed_compute=...]
-                ProjectionExec: elapsed=..., expr=[_rowid@1 as _rowid, ...], metrics=[...]
-                  ProjectionExec: elapsed=..., expr=[id@2 IS NOT NULL as __common_expr_1, ...], metrics=[...]
-                    HashJoinExec: elapsed=..., mode=CollectLeft, join_type=Right, ...
-                      CooperativeExec, elapsed=..., metrics=[]
-                        LanceRead: elapsed=..., ..., metrics=[..., bytes_read=..., ...]
-                      RepartitionExec: ...
+                ProjectionExec: elapsed=..., expr=[...], metrics=[...]
+                  HashJoinExec: elapsed=..., mode=CollectLeft, join_type=Right, ...
+                    CooperativeExec, elapsed=..., metrics=[]
+                      LanceRead: elapsed=..., ..., metrics=[..., bytes_read=..., ...]
+                    RepartitionExec: ...
+                      ProjectionExec: elapsed=..., expr=[..., true as __merge_source_sentinel], metrics=[...]
                         StreamingTableExec: ..., metrics=[]
 
         The two key parts of the plan analysis are LanceRead and MergeInsert.
@@ -430,14 +580,13 @@ def __init__(
         index_cache_size_bytes: Optional[int] = None,
         read_params: Optional[Dict[str, Any]] = None,
         session: Optional[Session] = None,
-        storage_options_provider: Optional[Any] = None,
-        namespace: Optional[Any] = None,
+        namespace_client: Optional[Any] = None,
         table_id: Optional[List[str]] = None,
+        namespace_client_managed_versioning: bool = False,
     ):
         uri = os.fspath(uri) if isinstance(uri, Path) else uri
         self._uri = uri
         self._storage_options = storage_options
-        self._storage_options_provider = storage_options_provider
 
         # Handle deprecation warning for index_cache_size
         if index_cache_size is not None:
@@ -449,6 +598,13 @@ def __init__(
                 stacklevel=2,
             )
 
+        # Store namespace_client and table_id for credential refresh in file operations
+        self._namespace_client = namespace_client
+        self._table_id = table_id
+        self._namespace_client_managed_versioning = namespace_client_managed_versioning
+
+        # Storage options provider is automatically created in Rust when
+        # namespace_client and table_id are provided
         self._ds = _Dataset(
             uri,
             version,
@@ -462,9 +618,9 @@ def __init__(
             index_cache_size_bytes=index_cache_size_bytes,
             read_params=read_params,
             session=session,
-            storage_options_provider=storage_options_provider,
-            namespace=namespace,
+            namespace_client=namespace_client,
             table_id=table_id,
+            namespace_client_managed_versioning=namespace_client_managed_versioning,
         )
         self._default_scan_options = default_scan_options
         self._read_params = read_params
@@ -529,13 +685,19 @@ def __setstate__(self, state):
         )
         self._default_scan_options = default_scan_options
         self._read_params = read_params
-        self._storage_options_provider = None
+        self._namespace_client = None
+        self._table_id = None
+        self._namespace_client_managed_versioning = False
 
     def __copy__(self):
         ds = LanceDataset.__new__(LanceDataset)
         ds._uri = self._uri
         ds._storage_options = self._storage_options
-        ds._storage_options_provider = self._storage_options_provider
+        ds._namespace_client = self._namespace_client
+        ds._table_id = self._table_id
+        ds._namespace_client_managed_versioning = (
+            self._namespace_client_managed_versioning
+        )
         ds._ds = copy.copy(self._ds)
         ds._default_scan_options = self._default_scan_options
         ds._read_params = self._read_params.copy() if self._read_params else None
@@ -630,7 +792,8 @@ def create_branch(
         ds._ds = new_ds
         ds._uri = new_ds.uri
         ds._storage_options = self._storage_options
-        ds._storage_options_provider = self._storage_options_provider
+        ds._namespace_client = self._namespace_client
+        ds._table_id = self._table_id
         ds._default_scan_options = self._default_scan_options
         ds._read_params = self._read_params
         return ds
@@ -641,12 +804,13 @@ def checkout_latest(self):
 
     def list_indices(self) -> List[Index]:
         """
-        Returns index information for all indices in the dataset.
+        Returns physical index segment information for all indices in the dataset.
 
         This method is deprecated as it requires loading the statistics for each index
-        which can be a very expensive operation.  Instead use describe_indices() to
-        list index information and index_statistics() to get the statistics for
-        individual indexes of interest.
+        which can be a very expensive operation.  It also exposes physical index
+        segments directly.  Instead use describe_indices() for logical index
+        descriptions and index_statistics() to get the statistics for individual
+        indexes of interest.
         """
         warnings.warn(
             "The 'list_indices' method is deprecated. It may be removed in a future "
@@ -657,7 +821,7 @@ def list_indices(self) -> List[Index]:
         return self._ds.load_indices()
 
     def describe_indices(self) -> List[IndexDescription]:
-        """Returns index information for all indices in the dataset."""
+        """Returns logical index information aggregated across all segments."""
         return self._ds.describe_indices()
 
     def index_statistics(self, index_name: str) -> Dict[str, Any]:
@@ -687,6 +851,7 @@ def scanner(
         offset: Optional[int] = None,
         nearest: Optional[dict] = None,
         batch_size: Optional[int] = None,
+        batch_size_bytes: Optional[int] = None,
         batch_readahead: Optional[int] = None,
         fragment_readahead: Optional[int] = None,
         scan_in_order: Optional[bool] = None,
@@ -782,9 +947,16 @@ def scanner(
                 }
 
         batch_size: int, default None
-            The target size of batches returned.  In some cases batches can be up to
-            twice this size (but never larger than this).  In some cases batches can
-            be smaller than this size.
+            The maximum number of rows per batch.  In some cases batches can be
+            smaller than this size.  Note: this can be overridden by
+            ``batch_size_bytes`` or by a dataset-level ``batch_size_bytes``
+            configured via ``FileReaderOptions``.
+        batch_size_bytes: int, default None
+            If set, the scanner will produce batches whose total size in bytes
+            is approximately this value, overriding the row-based ``batch_size``.
+            This can also be configured at the dataset level via
+            ``FileReaderOptions``.  A scanner-level setting takes precedence
+            over the dataset-level default.
         io_buffer_size: int, default None
             The size of the IO buffer.  See ``ScannerBuilder.io_buffer_size``
             for more information.
@@ -920,6 +1092,7 @@ def setopt(opt, val):
         setopt(builder.limit, limit)
         setopt(builder.offset, offset)
         setopt(builder.batch_size, batch_size)
+        setopt(builder.batch_size_bytes, batch_size_bytes)
         setopt(builder.io_buffer_size, io_buffer_size)
         setopt(builder.batch_readahead, batch_readahead)
         setopt(builder.fragment_readahead, fragment_readahead)
@@ -1003,6 +1176,7 @@ def to_table(
         offset: Optional[int] = None,
         nearest: Optional[dict] = None,
         batch_size: Optional[int] = None,
+        batch_size_bytes: Optional[int] = None,
         batch_readahead: Optional[int] = None,
         fragment_readahead: Optional[int] = None,
         scan_in_order: Optional[bool] = None,
@@ -1130,6 +1304,7 @@ def to_table(
             offset=offset,
             nearest=nearest,
             batch_size=batch_size,
+            batch_size_bytes=batch_size_bytes,
             io_buffer_size=io_buffer_size,
             batch_readahead=batch_readahead,
             fragment_readahead=fragment_readahead,
@@ -1148,6 +1323,73 @@ def to_table(
             disable_scoring_autoprojection=disable_scoring_autoprojection,
         ).to_table()
 
+    def to_pandas(
+        self,
+        columns: Optional[Union[List[str], Dict[str, str]]] = None,
+        filter: Optional[Union[str, pa.compute.Expression]] = None,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        nearest: Optional[dict] = None,
+        batch_size: Optional[int] = None,
+        batch_readahead: Optional[int] = None,
+        fragment_readahead: Optional[int] = None,
+        scan_in_order: Optional[bool] = None,
+        *,
+        prefilter: Optional[bool] = None,
+        with_row_id: Optional[bool] = None,
+        with_row_address: Optional[bool] = None,
+        use_stats: Optional[bool] = None,
+        fast_search: Optional[bool] = None,
+        full_text_query: Optional[Union[str, dict, FullTextQuery]] = None,
+        io_buffer_size: Optional[int] = None,
+        late_materialization: Optional[bool | List[str]] = None,
+        blob_mode: str = _BLOB_PANDAS_MODE_LAZY,
+        use_scalar_index: Optional[bool] = None,
+        include_deleted_rows: Optional[bool] = None,
+        order_by: Optional[List[ColumnOrdering]] = None,
+        disable_scoring_autoprojection: Optional[bool] = None,
+        **kwargs,
+    ) -> "pd.DataFrame":
+        """Read the data into a :py:class:`pandas.DataFrame`.
+
+        Parameters are the same as :meth:`to_table`, except pandas export uses
+        ``blob_mode`` instead of Arrow-facing ``blob_handling``.
+
+        Parameters
+        ----------
+        blob_mode: str, default "lazy"
+            Controls how blob columns are returned.
+
+            - ``"lazy"``: return :class:`lance.BlobFile` objects
+            - ``"bytes"``: return Python ``bytes``
+            - ``"descriptions"``: preserve ``to_table().to_pandas()`` behavior
+        **kwargs
+            Forwarded to :meth:`pyarrow.Table.to_pandas` for non-blob columns.
+        """
+        return self.scanner(
+            columns=columns,
+            filter=filter,
+            limit=limit,
+            offset=offset,
+            nearest=nearest,
+            batch_size=batch_size,
+            io_buffer_size=io_buffer_size,
+            batch_readahead=batch_readahead,
+            fragment_readahead=fragment_readahead,
+            late_materialization=late_materialization,
+            use_scalar_index=use_scalar_index,
+            scan_in_order=scan_in_order,
+            prefilter=prefilter,
+            with_row_id=with_row_id,
+            with_row_address=with_row_address,
+            use_stats=use_stats,
+            fast_search=fast_search,
+            full_text_query=full_text_query,
+            include_deleted_rows=include_deleted_rows,
+            order_by=order_by,
+            disable_scoring_autoprojection=disable_scoring_autoprojection,
+        ).to_pandas(blob_mode=blob_mode, **kwargs)
+
     @property
     def partition_expression(self):
         """
@@ -1506,6 +1748,7 @@ def to_batches(
         offset: Optional[int] = None,
         nearest: Optional[dict] = None,
         batch_size: Optional[int] = None,
+        batch_size_bytes: Optional[int] = None,
         batch_readahead: Optional[int] = None,
         fragment_readahead: Optional[int] = None,
         scan_in_order: Optional[bool] = None,
@@ -1542,6 +1785,7 @@ def to_batches(
             offset=offset,
             nearest=nearest,
             batch_size=batch_size,
+            batch_size_bytes=batch_size_bytes,
             io_buffer_size=io_buffer_size,
             batch_readahead=batch_readahead,
             fragment_readahead=fragment_readahead,
@@ -1683,21 +1927,77 @@ def take_blobs(
         -------
         blob_files : List[BlobFile]
         """
-        if sum([bool(v is not None) for v in [ids, addresses, indices]]) != 1:
-            raise ValueError(
-                "Exactly one of ids, indices, or addresses must be specified"
-            )
+        selection_kind, selection_values = _resolve_blob_selection(
+            ids, addresses, indices
+        )
 
-        if ids is not None:
-            lance_blob_files = self._ds.take_blobs(ids, blob_column)
-        elif addresses is not None:
-            lance_blob_files = self._ds.take_blobs_by_addresses(addresses, blob_column)
-        elif indices is not None:
-            lance_blob_files = self._ds.take_blobs_by_indices(indices, blob_column)
+        if selection_kind == "ids":
+            lance_blob_files = self._ds.take_blobs(selection_values, blob_column)
+        elif selection_kind == "addresses":
+            lance_blob_files = self._ds.take_blobs_by_addresses(
+                selection_values, blob_column
+            )
         else:
-            raise ValueError("Either ids, addresses, or indices must be specified")
+            lance_blob_files = self._ds.take_blobs_by_indices(
+                selection_values, blob_column
+            )
         return [BlobFile(lance_blob_file) for lance_blob_file in lance_blob_files]
 
+    def read_blobs(
+        self,
+        blob_column: str,
+        ids: Optional[Union[List[int], pa.Array]] = None,
+        addresses: Optional[Union[List[int], pa.Array]] = None,
+        indices: Optional[Union[List[int], pa.Array]] = None,
+        *,
+        io_buffer_size: Optional[int] = None,
+        preserve_order: Optional[bool] = None,
+    ) -> List[Tuple[int, bytes]]:
+        """
+        Read blobs directly into memory using Lance's planned blob reader.
+
+        Unlike :py:meth:`take_blobs`, which returns file-like :py:class:`lance.BlobFile`
+        handles for random access, this API plans and executes batched reads and
+        returns materialized blob payloads.
+
+        Exactly one of ids, addresses, or indices must be specified.
+
+        Parameters
+        ----------
+        blob_column : str
+            The name of the blob column to read.
+        ids : Integer Array or array-like
+            Row IDs to read in the dataset.
+        addresses : Integer Array or array-like
+            The (unstable) row addresses to read in the dataset.
+        indices : Integer Array or array-like
+            The offset / indices of the row in the dataset.
+        io_buffer_size : int, optional
+            Override the scheduler I/O buffer size used while materializing blobs.
+        preserve_order : bool, optional
+            If True, returned rows follow the requested selection order.
+
+        Returns
+        -------
+        blobs : List[Tuple[int, bytes]]
+            A list of ``(row_address, blob_bytes)`` pairs.
+        """
+        selection_kind, selection_values = _resolve_blob_selection(
+            ids, addresses, indices
+        )
+
+        kwargs = {
+            "io_buffer_size": io_buffer_size,
+            "preserve_order": preserve_order,
+        }
+        if selection_kind == "ids":
+            return self._ds.read_blobs(selection_values, blob_column, **kwargs)
+        if selection_kind == "addresses":
+            return self._ds.read_blobs_by_addresses(
+                selection_values, blob_column, **kwargs
+            )
+        return self._ds.read_blobs_by_indices(selection_values, blob_column, **kwargs)
+
     def head(self, num_rows, **kwargs):
         """
         Load the first N rows of the dataset.
@@ -2348,7 +2648,8 @@ def new_file_session(self):
         return LanceFileSession(
             base_path=self._uri,
             storage_options=self.latest_storage_options(),
-            storage_options_provider=self._storage_options_provider,
+            namespace_client=self._namespace_client,
+            table_id=self._table_id,
         )
 
     def checkout_version(
@@ -2498,6 +2799,7 @@ def create_scalar_index(
         train: bool = True,
         fragment_ids: Optional[List[int]] = None,
         index_uuid: Optional[str] = None,
+        progress_callback: Optional[Callable[[IndexProgress], None]] = None,
         **kwargs,
     ):
         """Create a scalar index on a column.
@@ -2591,14 +2893,17 @@ def create_scalar_index(
         fragment_ids : List[int], optional
             If provided, the index will be created only on the specified fragments.
             This enables distributed/fragment-level indexing. When provided, the
-            method returns an IndexMetadata object but does not commit the index
-            to the dataset. The index can be committed later using the commit API.
+            method returns metadata for one segment but does not commit
+            the index to the dataset. The segment can be planned, merged, and
+            committed later using the segment builder and commit APIs.
             This parameter is passed via kwargs internally.
         index_uuid : str, optional
-            A UUID to use for fragment-level distributed indexing
-            multiple fragment-level indices need to share UUID for later merging.
-            If not provided, a new UUID will be generated. This parameter is passed via
-            kwargs internally.
+            A UUID to use for the segment written by this call.
+            If not provided, a new UUID will be generated. This parameter is
+            passed via kwargs internally.
+        progress_callback : callable, optional
+            A callback that receives :class:`lance.progress.IndexProgress` events while
+            the index is being built.
 
         with_position: bool, default False
             This is for the ``INVERTED`` index. If True, the index will store the
@@ -2606,10 +2911,21 @@ def create_scalar_index(
             query. This will significantly increase the index size.
             It won't impact the performance of non-phrase queries even if it is set to
             True.
-        skip_merge: bool, default False
-            This is for the ``INVERTED`` index. If True, the index will skip the
-            partition merge stage after indexing. This can be useful for
-            distributed/fragment-level indexing where a later merge is desired.
+        memory_limit: int, optional
+            This is for the ``INVERTED`` index. Total build-time memory limit in MiB.
+            If set, Lance divides this budget evenly across the workers. If unset,
+            the default will be 2 GiB per worker. This parameter is only used for the
+            current build and is not persisted with the index.
+
+            A larger memory limit will create an index with fewer shards which will
+            be easier to search so this is a trade-off between build resources and
+            search cost.
+        num_workers: int, optional
+            This is for the ``INVERTED`` index. Number of workers to use for
+            the current build. The effective worker count is clamped to
+            ``[1, num_compute_cpus]``. If unset, Lance uses ``num_compute_cpus``
+            workers unless ``LANCE_FTS_NUM_SHARDS`` is set. This parameter is
+            only used for the current build and is not persisted with the index.
         base_tokenizer: str, default "simple"
             This is for the ``INVERTED`` index. The base tokenizer to use. The
             value can be:
@@ -2771,10 +3087,12 @@ def create_scalar_index(
             kwargs["fragment_ids"] = fragment_ids
         if index_uuid is not None:
             kwargs["index_uuid"] = index_uuid
+        if progress_callback is not None:
+            kwargs["progress_callback"] = progress_callback
 
         self._ds.create_index([column], index_type, name, replace, train, None, kwargs)
 
-    def create_index(
+    def _create_index_impl(
         self,
         column: Union[str, List[str]],
         index_type: str,
@@ -2793,244 +3111,71 @@ def create_index(
         index_cache_size: Optional[int] = None,
         shuffle_partition_batches: Optional[int] = None,
         shuffle_partition_concurrency: Optional[int] = None,
-        # experimental parameters
         ivf_centroids_file: Optional[str] = None,
         precomputed_partition_dataset: Optional[str] = None,
         storage_options: Optional[Dict[str, str]] = None,
         filter_nan: bool = True,
         train: bool = True,
-        # distributed indexing parameters
         fragment_ids: Optional[List[int]] = None,
         index_uuid: Optional[str] = None,
         *,
         target_partition_size: Optional[int] = None,
         skip_transpose: bool = False,
+        require_commit: bool = True,
         **kwargs,
-    ) -> LanceDataset:
-        """Create index on column.
-
-        **Experimental API**
-
-        Parameters
-        ----------
-        column : str
-            The column to be indexed.
-        index_type : str
-            The type of the index.
-            ``"IVF_PQ, IVF_HNSW_PQ and IVF_HNSW_SQ"`` are supported now.
-        name : str, optional
-            The index name. If not provided, it will be generated from the
-            column name.
-        metric : str
-            The distance metric type, i.e., "L2" (alias to "euclidean"), "cosine"
-            or "dot" (dot product). Default is "L2".
-        replace : bool
-            Replace the existing index if it exists.
-        num_partitions : int, optional
-            The number of partitions of IVF (Inverted File Index).
-            Deprecated. Use target_partition_size instead.
-        ivf_centroids : optional
-            It can be either :py:class:`np.ndarray`,
-            :py:class:`pyarrow.FixedSizeListArray` or
-            :py:class:`pyarrow.FixedShapeTensorArray`.
-            A ``num_partitions x dimension`` array of existing K-mean centroids
-            for IVF clustering. If not provided, a new KMeans model will be trained.
-        pq_codebook : optional,
-            It can be :py:class:`np.ndarray`, :py:class:`pyarrow.FixedSizeListArray`,
-            or :py:class:`pyarrow.FixedShapeTensorArray`.
-            A ``num_sub_vectors x (2 ^ nbits * dimensions // num_sub_vectors)``
-            array of K-mean centroids for PQ codebook.
-
-            Note: ``nbits`` is always 8 for now.
-            If not provided, a new PQ model will be trained.
-        num_sub_vectors : int, optional
-            The number of sub-vectors for PQ (Product Quantization).
-        accelerator : str or ``torch.Device``, optional
-            If set, use an accelerator to speed up the training process.
-            Accepted accelerator: "cuda" (Nvidia GPU) and "mps" (Apple Silicon GPU).
-            If not set, use the CPU.
-        index_cache_size : int, optional
-            The size of the index cache in number of entries. Default value is 256.
-        shuffle_partition_batches : int, optional
-            The number of batches, using the row group size of the dataset, to include
-            in each shuffle partition. Default value is 10240.
+    ) -> Index:
+        if not require_commit and fragment_ids is None:
+            raise ValueError(
+                "create_index_uncommitted requires fragment_ids "
+                "for distributed index build"
+            )
 
-            Assuming the row group size is 1024, each shuffle partition will hold
-            10240 * 1024 = 10,485,760 rows. By making this value smaller, this shuffle
-            will consume less memory but will take longer to complete, and vice versa.
-        shuffle_partition_concurrency : int, optional
-            The number of shuffle partitions to process concurrently. Default value is 2
+        # Only support building index for 1 column from the API aspect, however
+        # the internal implementation might support building multi-column index later.
+        if isinstance(column, str):
+            column = [column]
 
-            By making this value smaller, this shuffle will consume less memory but will
-            take longer to complete, and vice versa.
-        storage_options : optional, dict
-            Extra options that make sense for a particular storage connection. This is
-            used to store connection parameters like credentials, endpoint, etc.
-        filter_nan: bool
-            Defaults to True. False is UNSAFE, and will cause a crash if any null/nan
-            values are present (and otherwise will not). Disables the null filter used
-            for nullable columns. Obtains a small speed boost.
-        train : bool, default True
-            If True, the index will be trained on the data (e.g., compute IVF
-            centroids, PQ codebooks). If False, an empty index structure will be
-            created without training, which can be populated later.
-        fragment_ids : List[int], optional
-            If provided, the index will be created only on the specified fragments.
-            This enables distributed/fragment-level indexing. When provided, the
-            method creates temporary index metadata but does not commit the index
-            to the dataset. The index can be committed later using
-            merge_index_metadata(index_uuid, "VECTOR", column=..., index_name=...).
-        index_uuid : str, optional
-            A UUID to use for fragment-level distributed indexing. Multiple
-            fragment-level indices need to share UUID for later merging.
-            If not provided, a new UUID will be generated.
-        target_partition_size: int, optional
-            The target partition size. If set, the number of partitions will be computed
-            based on the target partition size.
-            Otherwise, the target partition size will be set by index type.
-        kwargs :
-            Parameters passed to the index building process.
+        # validate args
+        for c in column:
+            lance_field = self._ds.lance_schema.field_case_insensitive(c)
+            if lance_field is None:
+                raise KeyError(f"{c} not found in schema")
+            field = lance_field.to_arrow()
+            is_multivec = False
+            if pa.types.is_fixed_size_list(field.type):
+                dimension = field.type.list_size
+            elif pa.types.is_list(field.type) and pa.types.is_fixed_size_list(
+                field.type.value_type
+            ):
+                dimension = field.type.value_type.list_size
+                is_multivec = True
+            elif (
+                isinstance(field.type, pa.FixedShapeTensorType)
+                and len(field.type.shape) == 1
+            ):
+                dimension = field.type.shape[0]
+            else:
+                raise TypeError(
+                    f"Vector column {c} must be FixedSizeListArray "
+                    f"1-dimensional FixedShapeTensorArray, got {field.type}"
+                )
 
+            if num_sub_vectors is not None and dimension % num_sub_vectors != 0:
+                raise ValueError(
+                    f"dimension ({dimension}) must be divisible by num_sub_vectors"
+                    f" ({num_sub_vectors})"
+                )
 
-
-        The SQ (Scalar Quantization) is available for only ``IVF_HNSW_SQ`` index type,
-        this quantization method is used to reduce the memory usage of the index,
-        it maps the float vectors to integer vectors, each integer is of ``num_bits``,
-        now only 8 bits are supported.
-
-        If ``index_type`` is "IVF_*", then the following parameters are required:
-            num_partitions
-
-        If ``index_type`` is with "PQ", then the following parameters are required:
-            num_sub_vectors
-
-        Optional parameters for `IVF_PQ`:
-
-            - ivf_centroids
-                Existing K-mean centroids for IVF clustering.
-            - num_bits
-                The number of bits for PQ (Product Quantization). Default is 8.
-                Only 4, 8 are supported.
-            - index_file_version
-                The version of the index file. Default is "V3".
-
-        Optional parameters for `IVF_RQ`:
-
-            - num_bits
-                The number of bits for RQ (Rabit Quantization). Default is 1.
-
-        Optional parameters for `IVF_HNSW_*`:
-            max_level
-                Int, the maximum number of levels in the graph.
-            m
-                Int, the number of edges per node in the graph.
-            ef_construction
-                Int, the number of nodes to examine during the construction.
-
-        Examples
-        --------
-
-        .. code-block:: python
-
-            import lance
-
-            dataset = lance.dataset("/tmp/sift.lance")
-            dataset.create_index(
-                "vector",
-                "IVF_PQ",
-                num_partitions=256,
-                num_sub_vectors=16
-            )
-
-        .. code-block:: python
-
-            import lance
-
-            dataset = lance.dataset("/tmp/sift.lance")
-            dataset.create_index(
-                "vector",
-                "IVF_HNSW_SQ",
-                num_partitions=256,
-            )
-
-        Experimental Accelerator (GPU) support:
-
-        - *accelerate*: use GPU to train IVF partitions.
-            Only supports CUDA (Nvidia) or MPS (Apple) currently.
-            Requires PyTorch being installed.
-
-        .. code-block:: python
-
-            import lance
-
-            dataset = lance.dataset("/tmp/sift.lance")
-            dataset.create_index(
-                "vector",
-                "IVF_PQ",
-                num_partitions=256,
-                num_sub_vectors=16,
-                accelerator="cuda"
-            )
-
-        Note: GPU acceleration is currently supported only for the ``IVF_PQ`` index
-        type. Providing an accelerator for other index types will fall back to CPU
-        index building.
-
-        References
-        ----------
-        * `Faiss Index <https://github.com/facebookresearch/faiss/wiki/Faiss-indexes>`_
-        * IVF introduced in `Video Google: a text retrieval approach to object matching
-          in videos <https://ieeexplore.ieee.org/abstract/document/1238663>`_
-        * `Product quantization for nearest neighbor search
-          <https://hal.inria.fr/inria-00514462v2/document>`_
-
-        """
-        # Only support building index for 1 column from the API aspect, however
-        # the internal implementation might support building multi-column index later.
-        if isinstance(column, str):
-            column = [column]
-
-        # validate args
-        for c in column:
-            lance_field = self._ds.lance_schema.field_case_insensitive(c)
-            if lance_field is None:
-                raise KeyError(f"{c} not found in schema")
-            field = lance_field.to_arrow()
-            is_multivec = False
-            if pa.types.is_fixed_size_list(field.type):
-                dimension = field.type.list_size
-            elif pa.types.is_list(field.type) and pa.types.is_fixed_size_list(
-                field.type.value_type
-            ):
-                dimension = field.type.value_type.list_size
-                is_multivec = True
-            elif (
-                isinstance(field.type, pa.FixedShapeTensorType)
-                and len(field.type.shape) == 1
-            ):
-                dimension = field.type.shape[0]
-            else:
-                raise TypeError(
-                    f"Vector column {c} must be FixedSizeListArray "
-                    f"1-dimensional FixedShapeTensorArray, got {field.type}"
-                )
-
-            if num_sub_vectors is not None and dimension % num_sub_vectors != 0:
-                raise ValueError(
-                    f"dimension ({dimension}) must be divisible by num_sub_vectors"
-                    f" ({num_sub_vectors})"
-                )
-
-            element_type = field.type.value_type
-            if is_multivec:
-                element_type = field.type.value_type.value_type
-            if not (
-                pa.types.is_floating(element_type) or pa.types.is_uint8(element_type)
-            ):
-                raise TypeError(
-                    f"Vector column {c} must have floating value type, "
-                    f"got {field.type.value_type}"
-                )
+            element_type = field.type.value_type
+            if is_multivec:
+                element_type = field.type.value_type.value_type
+            if not (
+                pa.types.is_floating(element_type) or pa.types.is_uint8(element_type)
+            ):
+                raise TypeError(
+                    f"Vector column {c} must have floating value type, "
+                    f"got {field.type.value_type}"
+                )
 
         if not isinstance(metric, str) or metric.lower() not in [
             "l2",
@@ -3092,13 +3237,21 @@ def create_index(
             pass
 
         if torch_detected:
-            if fragment_ids is not None or index_uuid is not None:
-                LOGGER.info(
-                    "Torch detected; "
-                    "enforce single-node indexing (distributed is CPU-only)."
-                )
-            fragment_ids = None
-            index_uuid = None
+            if require_commit:
+                if fragment_ids is not None or index_uuid is not None:
+                    LOGGER.info(
+                        "Torch detected; "
+                        "enforce single-node indexing (distributed is CPU-only)."
+                    )
+                fragment_ids = None
+                index_uuid = None
+            else:
+                if index_uuid is not None:
+                    LOGGER.info(
+                        "Torch detected; "
+                        "enforce single-node indexing (distributed is CPU-only)."
+                    )
+                index_uuid = None
 
         if accelerator is not None:
             from .vector import (
@@ -3291,7 +3444,7 @@ def create_index(
             kwargs["index_uuid"] = index_uuid
 
         timers["final_create_index:start"] = time.time()
-        self._ds.create_index(
+        index = self._ds.create_index(
             column, index_type, name, replace, train, storage_options, kwargs
         )
         timers["final_create_index:end"] = time.time()
@@ -3307,8 +3460,340 @@ def create_index(
                 "Temporary shuffle buffers stored at %s, you may want to delete it.",
                 kwargs["precomputed_shuffle_buffers_path"],
             )
+        return index
+
+    def create_index(
+        self,
+        column: Union[str, List[str]],
+        index_type: str,
+        name: Optional[str] = None,
+        metric: str = "L2",
+        replace: bool = False,
+        num_partitions: Optional[int] = None,
+        ivf_centroids: Optional[
+            Union[np.ndarray, pa.FixedSizeListArray, pa.FixedShapeTensorArray]
+        ] = None,
+        pq_codebook: Optional[
+            Union[np.ndarray, pa.FixedSizeListArray, pa.FixedShapeTensorArray]
+        ] = None,
+        num_sub_vectors: Optional[int] = None,
+        accelerator: Optional[Union[str, "torch.Device"]] = None,
+        index_cache_size: Optional[int] = None,
+        shuffle_partition_batches: Optional[int] = None,
+        shuffle_partition_concurrency: Optional[int] = None,
+        # experimental parameters
+        ivf_centroids_file: Optional[str] = None,
+        precomputed_partition_dataset: Optional[str] = None,
+        storage_options: Optional[Dict[str, str]] = None,
+        filter_nan: bool = True,
+        train: bool = True,
+        # distributed indexing parameters
+        fragment_ids: Optional[List[int]] = None,
+        index_uuid: Optional[str] = None,
+        *,
+        target_partition_size: Optional[int] = None,
+        skip_transpose: bool = False,
+        progress_callback: Optional[Callable[[IndexProgress], None]] = None,
+        **kwargs,
+    ) -> LanceDataset:
+        """Create index on column.
+
+        **Experimental API**
+
+        Parameters
+        ----------
+        column : str
+            The column to be indexed.
+        index_type : str
+            The type of the index.
+            ``"IVF_PQ, IVF_HNSW_PQ and IVF_HNSW_SQ"`` are supported now.
+        name : str, optional
+            The index name. If not provided, it will be generated from the
+            column name.
+        metric : str
+            The distance metric type, i.e., "L2" (alias to "euclidean"), "cosine"
+            or "dot" (dot product). Default is "L2".
+        replace : bool
+            Replace the existing index if it exists.
+        num_partitions : int, optional
+            The number of partitions of IVF (Inverted File Index).
+            Deprecated. Use target_partition_size instead.
+        ivf_centroids : optional
+            It can be either :py:class:`np.ndarray`,
+            :py:class:`pyarrow.FixedSizeListArray` or
+            :py:class:`pyarrow.FixedShapeTensorArray`.
+            A ``num_partitions x dimension`` array of existing K-mean centroids
+            for IVF clustering. If not provided, a new KMeans model will be trained.
+        pq_codebook : optional,
+            It can be :py:class:`np.ndarray`, :py:class:`pyarrow.FixedSizeListArray`,
+            or :py:class:`pyarrow.FixedShapeTensorArray`.
+            A ``num_sub_vectors x (2 ^ nbits * dimensions // num_sub_vectors)``
+            array of K-mean centroids for PQ codebook.
+
+            Note: ``nbits`` is always 8 for now.
+            If not provided, a new PQ model will be trained.
+        num_sub_vectors : int, optional
+            The number of sub-vectors for PQ (Product Quantization).
+        accelerator : str or ``torch.Device``, optional
+            If set, use an accelerator to speed up the training process.
+            Accepted accelerator: "cuda" (Nvidia GPU) and "mps" (Apple Silicon GPU).
+            If not set, use the CPU.
+        index_cache_size : int, optional
+            The size of the index cache in number of entries. Default value is 256.
+        shuffle_partition_batches : int, optional
+            The number of batches, using the row group size of the dataset, to include
+            in each shuffle partition. Default value is 10240.
+
+            Assuming the row group size is 1024, each shuffle partition will hold
+            10240 * 1024 = 10,485,760 rows. By making this value smaller, this shuffle
+            will consume less memory but will take longer to complete, and vice versa.
+        shuffle_partition_concurrency : int, optional
+            The number of shuffle partitions to process concurrently. Default value is 2
+
+            By making this value smaller, this shuffle will consume less memory but will
+            take longer to complete, and vice versa.
+        storage_options : optional, dict
+            Extra options that make sense for a particular storage connection. This is
+            used to store connection parameters like credentials, endpoint, etc.
+        filter_nan: bool
+            Defaults to True. False is UNSAFE, and will cause a crash if any null/nan
+            values are present (and otherwise will not). Disables the null filter used
+            for nullable columns. Obtains a small speed boost.
+        train : bool, default True
+            If True, the index will be trained on the data (e.g., compute IVF
+            centroids, PQ codebooks). If False, an empty index structure will be
+            created without training, which can be populated later.
+        fragment_ids : List[int], optional
+            If provided, the index will be created only on the specified fragments.
+            This enables distributed/fragment-level indexing. When provided, the
+            method creates one segment but does not commit the index
+            to the dataset. The returned metadata can be passed to
+            optionally merged with ``merge_existing_index_segments(...)``
+            and then committed with ``commit_existing_index_segments(...)``.
+        index_uuid : str, optional
+            A UUID to use for the segment written by this call.
+            If not provided, a new UUID will be generated.
+        progress_callback : callable, optional
+            A callback that receives :class:`lance.progress.IndexProgress` events while
+            the index is being built.
+        target_partition_size: int, optional
+            The target partition size. If set, the number of partitions will be computed
+            based on the target partition size.
+            Otherwise, the target partition size will be set by index type.
+        kwargs :
+            Parameters passed to the index building process.
+
+
+
+        The SQ (Scalar Quantization) is available for only ``IVF_HNSW_SQ`` index type,
+        this quantization method is used to reduce the memory usage of the index,
+        it maps the float vectors to integer vectors, each integer is of ``num_bits``,
+        now only 8 bits are supported.
+
+        If ``index_type`` is "IVF_*", then the following parameters are required:
+            num_partitions
+
+        If ``index_type`` is with "PQ", then the following parameters are required:
+            num_sub_vectors
+
+        Optional parameters for `IVF_PQ`:
+
+            - ivf_centroids
+                Existing K-mean centroids for IVF clustering.
+            - num_bits
+                The number of bits for PQ (Product Quantization). Default is 8.
+                Only 4, 8 are supported.
+            - index_file_version
+                The version of the index file. Default is "V3".
+
+        Optional parameters for `IVF_RQ`:
+
+            - num_bits
+                The number of bits for RQ (Rabit Quantization). Default is 1.
+
+        Optional parameters for `IVF_HNSW_*`:
+            max_level
+                Int, the maximum number of levels in the graph.
+            m
+                Int, the number of edges per node in the graph.
+            ef_construction
+                Int, the number of nodes to examine during the construction.
+
+        Examples
+        --------
+
+        .. code-block:: python
+
+            import lance
+
+            dataset = lance.dataset("/tmp/sift.lance")
+            dataset.create_index(
+                "vector",
+                "IVF_PQ",
+                num_partitions=256,
+                num_sub_vectors=16
+            )
+
+        .. code-block:: python
+
+            import lance
+
+            dataset = lance.dataset("/tmp/sift.lance")
+            dataset.create_index(
+                "vector",
+                "IVF_HNSW_SQ",
+                num_partitions=256,
+            )
+
+        Experimental Accelerator (GPU) support:
+
+        - *accelerate*: use GPU to train IVF partitions.
+            Only supports CUDA (Nvidia) or MPS (Apple) currently.
+            Requires PyTorch being installed.
+
+        .. code-block:: python
+
+            import lance
+
+            dataset = lance.dataset("/tmp/sift.lance")
+            dataset.create_index(
+                "vector",
+                "IVF_PQ",
+                num_partitions=256,
+                num_sub_vectors=16,
+                accelerator="cuda"
+            )
+
+        Note: GPU acceleration is currently supported only for the ``IVF_PQ`` index
+        type. Providing an accelerator for other index types will fall back to CPU
+        index building.
+
+        References
+        ----------
+        * `Faiss Index <https://github.com/facebookresearch/faiss/wiki/Faiss-indexes>`_
+        * IVF introduced in `Video Google: a text retrieval approach to object matching
+          in videos <https://ieeexplore.ieee.org/abstract/document/1238663>`_
+        * `Product quantization for nearest neighbor search
+          <https://hal.inria.fr/inria-00514462v2/document>`_
+
+        """
+        if progress_callback is not None:
+            kwargs["progress_callback"] = progress_callback
+        self._create_index_impl(
+            column,
+            index_type,
+            name=name,
+            metric=metric,
+            replace=replace,
+            num_partitions=num_partitions,
+            ivf_centroids=ivf_centroids,
+            pq_codebook=pq_codebook,
+            num_sub_vectors=num_sub_vectors,
+            accelerator=accelerator,
+            index_cache_size=index_cache_size,
+            shuffle_partition_batches=shuffle_partition_batches,
+            shuffle_partition_concurrency=shuffle_partition_concurrency,
+            ivf_centroids_file=ivf_centroids_file,
+            precomputed_partition_dataset=precomputed_partition_dataset,
+            storage_options=storage_options,
+            filter_nan=filter_nan,
+            train=train,
+            fragment_ids=fragment_ids,
+            index_uuid=index_uuid,
+            target_partition_size=target_partition_size,
+            skip_transpose=skip_transpose,
+            require_commit=True,
+            **kwargs,
+        )
         return self
 
+    def create_index_uncommitted(
+        self,
+        column: Union[str, List[str]],
+        index_type: str,
+        name: Optional[str] = None,
+        metric: str = "L2",
+        replace: bool = False,
+        num_partitions: Optional[int] = None,
+        ivf_centroids: Optional[
+            Union[np.ndarray, pa.FixedSizeListArray, pa.FixedShapeTensorArray]
+        ] = None,
+        pq_codebook: Optional[
+            Union[np.ndarray, pa.FixedSizeListArray, pa.FixedShapeTensorArray]
+        ] = None,
+        num_sub_vectors: Optional[int] = None,
+        accelerator: Optional[Union[str, "torch.Device"]] = None,
+        index_cache_size: Optional[int] = None,
+        shuffle_partition_batches: Optional[int] = None,
+        shuffle_partition_concurrency: Optional[int] = None,
+        ivf_centroids_file: Optional[str] = None,
+        precomputed_partition_dataset: Optional[str] = None,
+        storage_options: Optional[Dict[str, str]] = None,
+        filter_nan: bool = True,
+        train: bool = True,
+        fragment_ids: Optional[List[int]] = None,
+        index_uuid: Optional[str] = None,
+        *,
+        target_partition_size: Optional[int] = None,
+        skip_transpose: bool = False,
+        **kwargs,
+    ) -> Index:
+        """
+        Create one segment without publishing it and return its metadata.
+
+        This is the public distributed-build API for vector index
+        construction. Unlike :meth:`create_index`, this method does not publish
+        the index into the dataset manifest. Instead, it writes one segment
+        under ``_indices/<segment_uuid>/`` and returns the resulting
+        :class:`Index` metadata.
+
+        Callers should:
+
+        1. run :meth:`create_index_uncommitted` on each worker with that worker's
+           assigned ``fragment_ids``
+        2. collect the returned :class:`Index` objects
+        3. optionally merge one or more caller-defined groups with
+           :meth:`merge_existing_index_segments`
+        4. commit the final segment list with
+           :meth:`commit_existing_index_segments`
+
+        Parameters are the same as :meth:`create_index`, with one additional
+        requirement:
+
+        - ``fragment_ids`` must be provided
+
+        Returns
+        -------
+        Index
+            Metadata for the segment that was written by this call.
+        """
+        return self._create_index_impl(
+            column,
+            index_type,
+            name=name,
+            metric=metric,
+            replace=replace,
+            num_partitions=num_partitions,
+            ivf_centroids=ivf_centroids,
+            pq_codebook=pq_codebook,
+            num_sub_vectors=num_sub_vectors,
+            accelerator=accelerator,
+            index_cache_size=index_cache_size,
+            shuffle_partition_batches=shuffle_partition_batches,
+            shuffle_partition_concurrency=shuffle_partition_concurrency,
+            ivf_centroids_file=ivf_centroids_file,
+            precomputed_partition_dataset=precomputed_partition_dataset,
+            storage_options=storage_options,
+            filter_nan=filter_nan,
+            train=train,
+            fragment_ids=fragment_ids,
+            index_uuid=index_uuid,
+            target_partition_size=target_partition_size,
+            skip_transpose=skip_transpose,
+            require_commit=False,
+            **kwargs,
+        )
+
     def drop_index(self, name: str):
         """
         Drops an index from the dataset
@@ -3320,7 +3805,7 @@ def drop_index(self, name: str):
         """
         return self._ds.drop_index(name)
 
-    def prewarm_index(self, name: str):
+    def prewarm_index(self, name: str, *, with_position: bool = False):
         """
         Prewarm an index
 
@@ -3332,26 +3817,32 @@ def prewarm_index(self, name: str):
         ----------
         name: str
             The name of the index to prewarm.
+        with_position: bool, default False
+            This is only supported for ``INVERTED`` indices. If True, positions are
+            also loaded into the cache during prewarm so phrase queries do not need a
+            separate lazy positions read.
         """
-        return self._ds.prewarm_index(name)
+        return self._ds.prewarm_index(name, with_position=with_position)
 
     def merge_index_metadata(
         self,
         index_uuid: str,
         index_type: str,
         batch_readhead: Optional[int] = None,
+        progress_callback: Optional[Callable[[IndexProgress], None]] = None,
     ):
         """
-        Merge distributed index metadata for supported scalar
-        and vector index types.
+        Merge distributed scalar index metadata.
 
-        This method supports all index types defined in
-        :class:`lance.indices.SupportedDistributedIndices`,
-        including scalar indices and precise vector index types.
+        Vector distributed indexing no longer uses this API. For vector indices,
+        build segments with :meth:`create_index_uncommitted`, optionally merge
+        caller-defined groups with :meth:`merge_existing_index_segments`, and
+        publish them with :meth:`commit_existing_index_segments`.
 
         This method does NOT commit changes.
 
-        This API merges temporary index files (e.g., per-fragment partials).
+        This API merges temporary scalar index files (for example per-fragment
+        BTree or inverted index outputs).
         After this method returns, callers MUST explicitly commit
         the index manifest using lance.LanceDataset.commit(...)
         with a LanceOperation.CreateIndex.
@@ -3359,13 +3850,16 @@ def merge_index_metadata(
         Parameters
         ----------
         index_uuid: str
-            The shared UUID used when building fragment-level indices.
+            The shared UUID used when building fragment-level scalar indices.
         index_type: str
             Index type name. Must be one of the enum values in
             :class:`lance.indices.SupportedDistributedIndices`
-            (for example ``"IVF_PQ"``).
+            supported by scalar distributed merge.
         batch_readhead: int, optional
             Prefetch concurrency used by BTREE merge reader. Default: 1.
+        progress_callback: callable, optional
+            A callback that receives :class:`lance.progress.IndexProgress` events while
+            metadata is being merged.
         """
         # Normalize type
         t = index_type.upper()
@@ -3377,9 +3871,24 @@ def merge_index_metadata(
             )
 
         # Merge physical index files at the index directory
-        self._ds.merge_index_metadata(index_uuid, t, batch_readhead)
+        self._ds.merge_index_metadata(index_uuid, t, batch_readhead, progress_callback)
         return None
 
+    def merge_existing_index_segments(self, segments: List[Index]) -> Index:
+        """
+        Merge one caller-defined group of existing uncommitted segments.
+        """
+        return self._ds.merge_existing_index_segments(segments)
+
+    def commit_existing_index_segments(
+        self, index_name: str, column: str, segments: List[Index]
+    ) -> LanceDataset:
+        """
+        Commit built index segments as one logical index.
+        """
+        self._ds.commit_existing_index_segments(index_name, column, segments)
+        return self
+
     def session(self) -> Session:
         """
         Return the dataset session, which holds the dataset's state.
@@ -3406,15 +3915,15 @@ def commit(
         read_version: Optional[int] = None,
         commit_lock: Optional[CommitLock] = None,
         storage_options: Optional[Dict[str, str]] = None,
-        storage_options_provider: Optional["StorageOptionsProvider"] = None,
         enable_v2_manifest_paths: Optional[bool] = None,
         detached: Optional[bool] = False,
         max_retries: int = 20,
         *,
         commit_message: Optional[str] = None,
         enable_stable_row_ids: Optional[bool] = None,
-        namespace: Optional["LanceNamespace"] = None,
+        namespace_client: Optional["LanceNamespace"] = None,
         table_id: Optional[List[str]] = None,
+        namespace_client_managed_versioning: bool = False,
     ) -> LanceDataset:
         """Create a new version of dataset
 
@@ -3454,8 +3963,6 @@ def commit(
         storage_options : optional, dict
             Extra options that make sense for a particular storage connection. This is
             used to store connection parameters like credentials, endpoint, etc.
-        storage_options_provider : StorageOptionsProvider, optional
-            A provider for dynamic storage options with automatic credential refresh.
         enable_v2_manifest_paths : bool, optional
             If True, and this is a new dataset, uses the new V2 manifest paths.
             These paths provide more efficient opening of datasets with many
@@ -3481,12 +3988,12 @@ def commit(
             row IDs assign each row a monotonically increasing id that persists
             across compaction and other maintenance operations.  This option is
             ignored for existing datasets.
-        namespace : LanceNamespace, optional
-            A namespace instance. Must be provided together with table_id.
+        namespace_client : LanceNamespace, optional
+            A namespace client. Must be provided together with table_id.
             Use lance.namespace.connect() to create a namespace.
         table_id : List[str], optional
             The table identifier within the namespace (e.g., ["workspace", "table"]).
-            Must be provided together with namespace.
+            Must be provided together with namespace_client.
 
         Returns
         -------
@@ -3540,6 +4047,9 @@ def commit(
                 "read_version is required for all operations except "
                 "Overwrite and Restore"
             )
+
+        # Storage options provider is automatically created in Rust when
+        # namespace_client and table_id are provided
         if isinstance(operation, Transaction):
             if commit_message is not None:
                 raise ValueError(
@@ -3552,13 +4062,13 @@ def commit(
                 operation,
                 commit_lock,
                 storage_options=storage_options,
-                storage_options_provider=storage_options_provider,
                 enable_v2_manifest_paths=enable_v2_manifest_paths,
                 detached=detached,
                 max_retries=max_retries,
                 enable_stable_row_ids=enable_stable_row_ids,
-                namespace=namespace,
+                namespace_client=namespace_client,
                 table_id=table_id,
+                namespace_client_managed_versioning=namespace_client_managed_versioning,
             )
         elif isinstance(operation, LanceOperation.BaseOperation):
             new_ds = _Dataset.commit(
@@ -3567,14 +4077,14 @@ def commit(
                 read_version,
                 commit_lock,
                 storage_options=storage_options,
-                storage_options_provider=storage_options_provider,
                 enable_v2_manifest_paths=enable_v2_manifest_paths,
                 detached=detached,
                 max_retries=max_retries,
                 commit_message=commit_message,
                 enable_stable_row_ids=enable_stable_row_ids,
-                namespace=namespace,
+                namespace_client=namespace_client,
                 table_id=table_id,
+                namespace_client_managed_versioning=namespace_client_managed_versioning,
             )
         else:
             raise TypeError(
@@ -3584,7 +4094,9 @@ def commit(
 
         ds = LanceDataset.__new__(LanceDataset)
         ds._storage_options = storage_options
-        ds._storage_options_provider = storage_options_provider
+        ds._namespace_client = namespace_client
+        ds._table_id = table_id
+        ds._namespace_client_managed_versioning = namespace_client_managed_versioning
         ds._ds = new_ds
         ds._uri = new_ds.uri
         ds._default_scan_options = None
@@ -3597,7 +4109,6 @@ def commit_batch(
         transactions: Sequence[Transaction],
         commit_lock: Optional[CommitLock] = None,
         storage_options: Optional[Dict[str, str]] = None,
-        storage_options_provider: Optional["StorageOptionsProvider"] = None,
         enable_v2_manifest_paths: Optional[bool] = None,
         detached: Optional[bool] = False,
         max_retries: int = 20,
@@ -3626,8 +4137,6 @@ def commit_batch(
         storage_options : optional, dict
             Extra options that make sense for a particular storage connection. This is
             used to store connection parameters like credentials, endpoint, etc.
-        storage_options_provider : StorageOptionsProvider, optional
-            A provider for dynamic storage options with automatic credential refresh.
         enable_v2_manifest_paths : bool, optional
             If True, and this is a new dataset, uses the new V2 manifest paths.
             These paths provide more efficient opening of datasets with many
@@ -3674,7 +4183,6 @@ def commit_batch(
             transactions,
             commit_lock,
             storage_options=storage_options,
-            storage_options_provider=storage_options_provider,
             enable_v2_manifest_paths=enable_v2_manifest_paths,
             detached=detached,
             max_retries=max_retries,
@@ -3683,7 +4191,8 @@ def commit_batch(
         ds._ds = new_ds
         ds._uri = new_ds.uri
         ds._storage_options = storage_options
-        ds._storage_options_provider = storage_options_provider
+        ds._namespace_client = None
+        ds._table_id = None
         ds._default_scan_options = None
         ds._read_params = None
         return BulkCommitResult(
@@ -4183,6 +4692,7 @@ class Tag(TypedDict):
 
 class Branch(TypedDict):
     parent_branch: Optional[str]
+    branch_identifier: List[Tuple[int, str]]
     parent_version: int
     create_at: int
     manifest_size: int
@@ -4215,6 +4725,14 @@ class ExecuteResult(TypedDict):
     num_deleted_rows: int
 
 
+@dataclass
+class IndexFile:
+    """Metadata about a file in an index segment."""
+
+    path: str
+    size_bytes: int
+
+
 @dataclass
 class Index:
     """Represents an index in the dataset."""
@@ -4227,6 +4745,8 @@ class Index:
     index_version: int
     created_at: Optional[datetime] = None
     base_id: Optional[int] = None
+    files: Optional[List["IndexFile"]] = None
+    index_details: Optional[tuple[str, bytes]] = None
 
 
 class AutoCleanupConfig(TypedDict):
@@ -4755,6 +5275,7 @@ def __init__(self, ds: LanceDataset):
         self._columns_with_transform = None
         self._nearest = None
         self._batch_size: Optional[int] = None
+        self._batch_size_bytes: Optional[int] = None
         self._io_buffer_size: Optional[int] = None
         self._batch_readahead: Optional[int] = None
         self._fragment_readahead: Optional[int] = None
@@ -4785,10 +5306,28 @@ def apply_defaults(self, default_opts: Dict[str, Any]) -> ScannerBuilder:
         return self
 
     def batch_size(self, batch_size: int) -> ScannerBuilder:
-        """Set batch size for Scanner"""
+        """Set the maximum number of rows per batch.
+
+        Note: this can be overridden by ``batch_size_bytes`` or by a
+        dataset-level ``batch_size_bytes`` configured via
+        ``FileReaderOptions``.
+        """
         self._batch_size = batch_size
         return self
 
+    def batch_size_bytes(self, batch_size_bytes: int) -> ScannerBuilder:
+        """Set the target batch size in bytes.
+
+        When set, the scanner will produce batches whose total size in bytes
+        is approximately this value, overriding the row-based ``batch_size``.
+
+        This can also be configured at the dataset level via
+        ``FileReaderOptions``.  A scanner-level setting takes precedence
+        over the dataset-level default.
+        """
+        self._batch_size_bytes = batch_size_bytes
+        return self
+
     def io_buffer_size(self, io_buffer_size: int) -> ScannerBuilder:
         """
         Set the I/O buffer size for the Scanner
@@ -5173,6 +5712,7 @@ def to_scanner(self) -> LanceScanner:
             self._offset,
             self._nearest,
             self._batch_size,
+            self._batch_size_bytes,
             self._io_buffer_size,
             self._batch_readahead,
             self._fragment_readahead,
@@ -5194,13 +5734,19 @@ def to_scanner(self) -> LanceScanner:
             self._disable_scoring_autoprojection,
             self._substrait_aggregate,
         )
-        return LanceScanner(scanner, self.ds)
+        return LanceScanner(scanner, self.ds, _snapshot_scanner_builder(self))
 
 
 class LanceScanner(pa.dataset.Scanner):
-    def __init__(self, scanner: _Scanner, dataset: LanceDataset):
+    def __init__(
+        self,
+        scanner: _Scanner,
+        dataset: LanceDataset,
+        snapshot: Optional[Dict[str, Any]] = None,
+    ):
         self._scanner = scanner
         self._ds = dataset
+        self._snapshot = snapshot
 
     def to_table(self) -> pa.Table:
         """
@@ -5214,6 +5760,83 @@ def to_reader(self) -> pa.RecordBatchReader:
     def to_batches(self) -> Iterator[RecordBatch]:
         yield from self.to_reader()
 
+    def to_pandas(
+        self, *, blob_mode: str = _BLOB_PANDAS_MODE_LAZY, **kwargs: Any
+    ) -> "pd.DataFrame":
+        """Read the scan results into a :py:class:`pandas.DataFrame`.
+
+        ``blob_mode`` is pandas-specific and does not replace Arrow's
+        ``blob_handling`` setting used by :meth:`to_table`.
+        """
+        blob_mode = _normalize_blob_pandas_mode(blob_mode)
+        schema = self.projected_schema
+        blob_columns = _blob_columns_in_schema(schema)
+        if not blob_columns or blob_mode == _BLOB_PANDAS_MODE_DESCRIPTIONS:
+            return self.to_table().to_pandas(**kwargs)
+
+        if self._snapshot is None:
+            raise NotImplementedError(
+                "blob-aware to_pandas requires a scanner created from the Python API"
+            )
+
+        snapshot = dict(self._snapshot)
+        if blob_mode == _BLOB_PANDAS_MODE_BYTES:
+            snapshot["_blob_handling"] = "all_binary"
+            return (
+                _scanner_from_snapshot(self._ds, snapshot)
+                .to_table()
+                .to_pandas(**kwargs)
+            )
+
+        blob_sources = _blob_column_sources(schema, self._snapshot, self._ds.schema)
+        snapshot["_with_row_address"] = True
+        snapshot["_blob_handling"] = "blobs_descriptions"
+        table = _scanner_from_snapshot(self._ds, snapshot).to_table()
+
+        requested_rowaddr = bool(self._snapshot.get("_with_row_address", False))
+        if _BLOB_ROW_ADDR_COLUMN not in table.schema.names:
+            raise RuntimeError("blob-aware to_pandas expected _rowaddr in scan results")
+
+        row_addrs = table.column(_BLOB_ROW_ADDR_COLUMN).to_pylist()
+        columns_to_drop = [name for name in blob_columns if name in table.schema.names]
+        if not requested_rowaddr:
+            columns_to_drop.append(_BLOB_ROW_ADDR_COLUMN)
+        non_blob_table = (
+            table.drop_columns(columns_to_drop) if columns_to_drop else table
+        )
+        if non_blob_table.num_columns == 0:
+            dataframe = pd.DataFrame(index=range(table.num_rows))
+        else:
+            dataframe = non_blob_table.to_pandas(**kwargs)
+
+        output_names = [field.name for field in schema]
+        for index, name in enumerate(output_names):
+            if name not in blob_columns:
+                continue
+
+            descriptions = table.column(name).to_pylist()
+            non_null_positions = [
+                pos
+                for pos, description in enumerate(descriptions)
+                if not _is_null_blob_description(description)
+            ]
+            non_null_addrs = [row_addrs[pos] for pos in non_null_positions]
+            blob_files = (
+                self._ds.take_blobs(blob_sources[name], addresses=non_null_addrs)
+                if non_null_addrs
+                else []
+            )
+            blob_iter = iter(blob_files)
+            values = []
+            for description in descriptions:
+                if _is_null_blob_description(description):
+                    values.append(None)
+                    continue
+                values.append(next(blob_iter))
+            dataframe.insert(index, name, values)
+
+        return dataframe
+
     @property
     def projected_schema(self) -> Schema:
         return self._scanner.schema
@@ -5331,12 +5954,12 @@ def __init__(self, dataset: LanceDataset):
     def compact_files(
         self,
         *,
-        target_rows_per_fragment: int = 1024 * 1024,
-        max_rows_per_group: int = 1024,
+        target_rows_per_fragment: Optional[int] = None,
+        max_rows_per_group: Optional[int] = None,
         max_bytes_per_file: Optional[int] = None,
-        materialize_deletions: bool = True,
-        materialize_deletions_threshold: float = 0.1,
-        defer_index_remap: bool = False,
+        materialize_deletions: Optional[bool] = None,
+        materialize_deletions_threshold: Optional[float] = None,
+        defer_index_remap: Optional[bool] = None,
         num_threads: Optional[int] = None,
         batch_size: Optional[int] = None,
         compaction_mode: Optional[
@@ -5358,14 +5981,33 @@ def compact_files(
         not be compacted because the fragments it is adjacent to do not need
         compaction.
 
+        Default values for these options can be stored in the dataset manifest
+        config using keys prefixed with ``lance.compaction.``. For example,
+        setting the config key ``lance.compaction.target_rows_per_fragment`` to
+        ``"500000"`` will use 500,000 as the default target rows per fragment.
+        Explicitly provided parameters take precedence over manifest config
+        values, which in turn take precedence over hardcoded defaults.
+
+        Supported config keys: ``lance.compaction.target_rows_per_fragment``,
+        ``lance.compaction.max_rows_per_group``,
+        ``lance.compaction.max_bytes_per_file``,
+        ``lance.compaction.materialize_deletions``,
+        ``lance.compaction.materialize_deletions_threshold``,
+        ``lance.compaction.defer_index_remap``,
+        ``lance.compaction.batch_size``,
+        ``lance.compaction.compaction_mode``,
+        ``lance.compaction.binary_copy_read_batch_bytes``.
+
         Parameters
         ----------
-        target_rows_per_fragment: int, default 1024*1024
+        target_rows_per_fragment: int, optional
             The target number of rows per fragment. This is the number of rows
-            that will be in each fragment after compaction.
-        max_rows_per_group: int, default 1024
+            that will be in each fragment after compaction. If not specified,
+            uses the manifest config value, or 1024*1024.
+        max_rows_per_group: int, optional
             Max number of rows per group. This does not affect which fragments
             need compaction, but does affect how they are re-written if selected.
+            If not specified, uses the manifest config value, or 1024.
 
             This setting only affects datasets using the legacy storage format.
             The newer format does not require row groups.
@@ -5376,14 +6018,17 @@ def compact_files(
             that are smaller than `target_rows_per_fragment`.
 
             The default will use the default from ``write_dataset``.
-        materialize_deletions: bool, default True
+        materialize_deletions: bool, optional
             Whether to compact fragments with soft deleted rows so they are no
-            longer present in the file.
-        materialize_deletions_threshold: float, default 0.1
+            longer present in the file. If not specified, uses the manifest
+            config value, or True.
+        materialize_deletions_threshold: float, optional
             The fraction of original rows that are soft deleted in a fragment
-            before the fragment is a candidate for compaction.
-        defer_index_remap: bool, default False
-            Whether to defer index remapping during compaction.
+            before the fragment is a candidate for compaction. If not specified,
+            uses the manifest config value, or 0.1.
+        defer_index_remap: bool, optional
+            Whether to defer index remapping during compaction. If not specified,
+            uses the manifest config value, or False.
         num_threads: int, optional
             The number of threads to use when performing compaction. If not
             specified, defaults to the number of cores on the machine.
@@ -5414,18 +6059,22 @@ def compact_files(
         --------
         lance.optimize.Compaction
         """
-        opts = dict(
-            target_rows_per_fragment=target_rows_per_fragment,
-            max_rows_per_group=max_rows_per_group,
-            max_bytes_per_file=max_bytes_per_file,
-            materialize_deletions=materialize_deletions,
-            materialize_deletions_threshold=materialize_deletions_threshold,
-            defer_index_remap=defer_index_remap,
-            num_threads=num_threads,
-            batch_size=batch_size,
-            compaction_mode=compaction_mode,
-            binary_copy_read_batch_bytes=binary_copy_read_batch_bytes,
-        )
+        opts = {
+            k: v
+            for k, v in dict(
+                target_rows_per_fragment=target_rows_per_fragment,
+                max_rows_per_group=max_rows_per_group,
+                max_bytes_per_file=max_bytes_per_file,
+                materialize_deletions=materialize_deletions,
+                materialize_deletions_threshold=materialize_deletions_threshold,
+                defer_index_remap=defer_index_remap,
+                num_threads=num_threads,
+                batch_size=batch_size,
+                compaction_mode=compaction_mode,
+                binary_copy_read_batch_bytes=binary_copy_read_batch_bytes,
+            ).items()
+            if v is not None
+        }
         return Compaction.execute(self._dataset, opts)
 
     def optimize_indices(self, **kwargs):
@@ -5705,8 +6354,10 @@ def write_dataset(
     transaction_properties: Optional[Dict[str, str]] = None,
     initial_bases: Optional[List[DatasetBasePath]] = None,
     target_bases: Optional[List[str]] = None,
+    external_blob_mode: Literal["reference", "ingest"] = "reference",
     allow_external_blob_outside_bases: bool = False,
-    namespace: Optional[LanceNamespace] = None,
+    blob_pack_file_size_threshold: Optional[int] = None,
+    namespace_client: Optional[LanceNamespace] = None,
     table_id: Optional[List[str]] = None,
 ) -> LanceDataset:
     """Write a given data_obj to the given uri
@@ -5720,7 +6371,7 @@ def write_dataset(
     uri: str, Path, LanceDataset, or None
         Where to write the dataset to (directory). If a LanceDataset is passed,
         the session will be reused.
-        Either `uri` or (`namespace` + `table_id`) must be provided, but not both.
+        Either `uri` or (`namespace_client` + `table_id`) must be provided.
     schema: Schema, optional
         If specified and the input is a pandas DataFrame, use this schema
         instead of the default pandas to arrow table conversion.
@@ -5800,48 +6451,58 @@ def write_dataset(
 
         **CREATE mode**: References must match bases in `initial_bases`
         **APPEND/OVERWRITE modes**: References must match bases in the existing manifest
+    external_blob_mode: {"reference", "ingest"}, default "reference"
+        How external blob URIs are handled on write.
+
+        - ``"reference"`` stores the URI as an external blob reference.
+        - ``"ingest"`` reads the external bytes during write and stores them in
+          Lance-managed storage using the normal inline / packed / dedicated
+          thresholds.
     allow_external_blob_outside_bases: bool, default False
         If False, external blob URIs must map to the dataset root or a registered
         base path. If True, external blob URIs outside registered bases are allowed.
-    namespace : optional, LanceNamespace
-        A namespace instance from which to fetch table location and storage options.
+        This option only applies when ``external_blob_mode="reference"``.
+    blob_pack_file_size_threshold: optional, int, default None
+        Maximum size in bytes for blob v2 pack (.blob) sidecar files. When a pack
+        file reaches this size, a new one is started. If not set, defaults to 1 GiB.
+    namespace_client : optional, LanceNamespace
+        A namespace client from which to fetch table location and storage options.
         Must be provided together with `table_id`. Cannot be used with `uri`.
         When provided, the table location will be fetched automatically from the
         namespace via describe_table(). Storage options will be automatically refreshed
         before they expire.
     table_id : optional, List[str]
         The table identifier when using a namespace (e.g., ["my_table"]).
-        Must be provided together with `namespace`. Cannot be used with `uri`.
+        Must be provided together with `namespace_client`. Cannot be used with `uri`.
 
     Notes
     -----
-    When using `namespace` and `table_id`:
+    When using `namespace_client` and `table_id`:
     - The `uri` parameter is optional and will be fetched from the namespace
     - Storage options from describe_table() will be used automatically
-    - A `LanceNamespaceStorageOptionsProvider` will be created automatically for
-      storage options refresh
+    - Storage options provider will be created automatically for credential refresh
     - Initial storage options from describe_table() will be merged with
       any provided `storage_options`
     """
-    # Validate that user provides either uri OR (namespace + table_id), not both
+    # Validate that user provides either uri OR (namespace_client + table_id), not both
     has_uri = uri is not None
-    has_namespace = namespace is not None or table_id is not None
+    has_namespace = namespace_client is not None or table_id is not None
 
     if has_uri and has_namespace:
         raise ValueError(
-            "Cannot specify both 'uri' and 'namespace/table_id'. "
-            "Please provide either 'uri' or both 'namespace' and 'table_id'."
+            "Cannot specify both 'uri' and 'namespace_client/table_id'. "
+            "Please provide either 'uri' or both 'namespace_client' and 'table_id'."
         )
     elif not has_uri and not has_namespace:
         raise ValueError(
-            "Must specify either 'uri' or both 'namespace' and 'table_id'."
+            "Must specify either 'uri' or both 'namespace_client' and 'table_id'."
         )
 
     # Handle namespace-based dataset writing
-    if namespace is not None:
+    if namespace_client is not None:
         if table_id is None:
             raise ValueError(
-                "Both 'namespace' and 'table_id' must be provided together."
+                "Both 'namespace_client' and 'table_id' must be provided together."
             )
 
         # Implement write_into_namespace logic in Python
@@ -5853,16 +6514,15 @@ def write_dataset(
         from .namespace import (
             DeclareTableRequest,
             DescribeTableRequest,
-            LanceNamespaceStorageOptionsProvider,
         )
 
         # Determine which namespace method to call based on mode
         if mode == "create":
             declare_request = DeclareTableRequest(id=table_id, location=None)
-            response = namespace.declare_table(declare_request)
+            response = namespace_client.declare_table(declare_request)
         elif mode in ("append", "overwrite"):
             request = DescribeTableRequest(id=table_id, version=None)
-            response = namespace.describe_table(request)
+            response = namespace_client.describe_table(request)
         else:
             raise ValueError(f"Invalid mode: {mode}")
 
@@ -5874,33 +6534,29 @@ def write_dataset(
             )
 
         # Check if namespace manages versioning (commits go through namespace API)
-        managed_versioning = getattr(response, "managed_versioning", None) is True
+        namespace_client_managed_versioning = (
+            getattr(response, "managed_versioning", None) is True
+        )
 
         # Use namespace storage options
         namespace_storage_options = response.storage_options
 
-        # Set up storage options and provider
-        if namespace_storage_options:
-            # Create the storage options provider for automatic refresh
-            storage_options_provider = LanceNamespaceStorageOptionsProvider(
-                namespace=namespace, table_id=table_id
-            )
-
-            # Merge namespace storage options with any existing options
-            # Namespace options take precedence (same as Rust implementation)
+        # Merge namespace storage options with any existing options
+        # Namespace options take precedence (same as Rust implementation)
+        # Storage options provider will be created automatically in Rust
+        if namespace_storage_options is not None:
             if storage_options is None:
                 storage_options = dict(namespace_storage_options)
             else:
                 merged_options = dict(storage_options)
                 merged_options.update(namespace_storage_options)
                 storage_options = merged_options
-        else:
-            storage_options_provider = None
     elif table_id is not None:
-        raise ValueError("Both 'namespace' and 'table_id' must be provided together.")
+        raise ValueError(
+            "Both 'namespace_client' and 'table_id' must be provided together."
+        )
     else:
-        storage_options_provider = None
-        managed_versioning = False
+        namespace_client_managed_versioning = False
 
     if use_legacy_format is not None:
         warnings.warn(
@@ -5935,17 +6591,19 @@ def write_dataset(
         "transaction_properties": merged_properties,
         "initial_bases": initial_bases,
         "target_bases": target_bases,
+        "external_blob_mode": external_blob_mode,
         "allow_external_blob_outside_bases": allow_external_blob_outside_bases,
+        "blob_pack_file_size_threshold": blob_pack_file_size_threshold,
     }
 
-    # Add storage_options_provider if created from namespace
-    if storage_options_provider is not None:
-        params["storage_options_provider"] = storage_options_provider
-
-    # Add namespace and table_id for managed versioning (external manifest store)
-    if managed_versioning and namespace is not None and table_id is not None:
-        params["namespace"] = namespace
+    # Add namespace_client and table_id for storage options provider and managed
+    # versioning. The storage options provider will be created automatically in Rust.
+    if namespace_client is not None and table_id is not None:
+        params["namespace_client"] = namespace_client
         params["table_id"] = table_id
+        params["namespace_client_managed_versioning"] = (
+            namespace_client_managed_versioning
+        )
 
     if commit_lock:
         if not callable(commit_lock):
@@ -5963,7 +6621,9 @@ def write_dataset(
 
     ds = LanceDataset.__new__(LanceDataset)
     ds._storage_options = storage_options
-    ds._storage_options_provider = None
+    ds._namespace_client = namespace_client
+    ds._table_id = table_id
+    ds._namespace_client_managed_versioning = namespace_client_managed_versioning
     ds._ds = inner_ds
     ds._uri = inner_ds.uri
     ds._default_scan_options = None
diff --git a/python/python/lance/download.py b/python/python/lance/download.py
index 34b43eef1d0..9dc5ecbdac4 100644
--- a/python/python/lance/download.py
+++ b/python/python/lance/download.py
@@ -4,6 +4,7 @@
 import os
 import shutil
 import subprocess
+import sys
 import tarfile
 import traceback
 from io import BytesIO
@@ -13,6 +14,26 @@
 LANGUAGE_MODEL_HOME = language_model_home()
 
 
+def _safe_tar_extractall(tar, dest_dir):
+    """Extract tar safely, blocking path traversal attacks (CVE-2007-4559).
+
+    On Python >= 3.12 uses the built-in ``filter="data"`` safeguard.
+    On older versions, manually validates every member path.
+    """
+    if sys.version_info >= (3, 12):
+        tar.extractall(path=dest_dir, filter="data")
+    else:
+        abs_dest = os.path.realpath(dest_dir)
+        for member in tar.getmembers():
+            member_path = os.path.join(dest_dir, member.name)
+            abs_member = os.path.realpath(member_path)
+            if not abs_member.startswith(abs_dest + os.sep) and abs_member != abs_dest:
+                raise Exception(
+                    f"Tar member '{member.name}' would extract outside target directory"
+                )
+        tar.extractall(path=dest_dir)
+
+
 def check_lindera():
     if not shutil.which("lindera"):
         raise Exception(
@@ -69,7 +90,7 @@ def download_lindera(lm: str):
     try:
         os.chdir(src_dirname)
         with tarfile.open(fileobj=BytesIO(data)) as tar:
-            tar.extractall()
+            _safe_tar_extractall(tar, src_dirname)
             name = tar.getnames()[0]
         cmd = [
             "lindera",
diff --git a/python/python/lance/file.py b/python/python/lance/file.py
index 8a20e4aff2f..011fbe4a01d 100644
--- a/python/python/lance/file.py
+++ b/python/python/lance/file.py
@@ -2,11 +2,10 @@
 # SPDX-FileCopyrightText: Copyright The Lance Authors
 
 from pathlib import Path
-from typing import Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
 import pyarrow as pa
 
-from .io import StorageOptionsProvider
 from .lance import (
     LanceBufferDescriptor,
     LanceColumnMetadata,
@@ -25,6 +24,9 @@
     LanceFileWriter as _LanceFileWriter,
 )
 
+if TYPE_CHECKING:
+    from .namespace import LanceNamespace
+
 
 class ReaderResults:
     """
@@ -67,7 +69,8 @@ def __init__(
         storage_options: Optional[Dict[str, str]] = None,
         columns: Optional[List[str]] = None,
         *,
-        storage_options_provider: Optional[StorageOptionsProvider] = None,
+        namespace_client: Optional["LanceNamespace"] = None,
+        table_id: Optional[List[str]] = None,
         _inner_reader: Optional[_LanceFileReader] = None,
     ):
         """
@@ -82,9 +85,12 @@ def __init__(
         storage_options : optional, dict
             Extra options to be used for a particular storage connection. This is
             used to store connection parameters like credentials, endpoint, etc.
-        storage_options_provider : optional
-            A provider that can provide storage options dynamically. This is useful
-            for credentials that need to be refreshed or vended on-demand.
+        namespace_client : optional, LanceNamespace
+            A namespace client for automatic credential refresh.
+            Must be provided together with table_id.
+        table_id : optional, List[str]
+            The table identifier within the namespace.
+            Must be provided together with namespace_client.
         columns: list of str, default None
             List of column names to be fetched.
             All columns are fetched if None or unspecified.
@@ -97,7 +103,8 @@ def __init__(
             self._reader = _LanceFileReader(
                 path,
                 storage_options=storage_options,
-                storage_options_provider=storage_options_provider,
+                namespace_client=namespace_client,
+                table_id=table_id,
                 columns=columns,
             )
 
@@ -213,7 +220,8 @@ def __init__(
         self,
         base_path: str,
         storage_options: Optional[Dict[str, str]] = None,
-        storage_options_provider: Optional[StorageOptionsProvider] = None,
+        namespace_client: Optional["LanceNamespace"] = None,
+        table_id: Optional[List[str]] = None,
     ):
         """
         Creates a new file session
@@ -227,16 +235,20 @@ def __init__(
         storage_options : optional, dict
             Extra options to be used for a particular storage connection. This is
             used to store connection parameters like credentials, endpoint, etc.
-        storage_options_provider : optional
-            A provider that can provide storage options dynamically. This is useful
-            for credentials that need to be refreshed or vended on-demand.
+        namespace_client : optional, LanceNamespace
+            A namespace client for automatic credential refresh.
+            Must be provided together with table_id.
+        table_id : optional, List[str]
+            The table identifier within the namespace.
+            Must be provided together with namespace_client.
         """
         if isinstance(base_path, Path):
             base_path = str(base_path)
         self._session = _LanceFileSession(
             base_path,
             storage_options=storage_options,
-            storage_options_provider=storage_options_provider,
+            namespace_client=namespace_client,
+            table_id=table_id,
         )
 
     def open_reader(
@@ -380,7 +392,8 @@ def __init__(
         data_cache_bytes: Optional[int] = None,
         version: Optional[str] = None,
         storage_options: Optional[Dict[str, str]] = None,
-        storage_options_provider: Optional[StorageOptionsProvider] = None,
+        namespace_client: Optional["LanceNamespace"] = None,
+        table_id: Optional[List[str]] = None,
         max_page_bytes: Optional[int] = None,
         _inner_writer: Optional[_LanceFileWriter] = None,
         **kwargs,
@@ -407,10 +420,12 @@ def __init__(
         storage_options : optional, dict
             Extra options to be used for a particular storage connection. This is
             used to store connection parameters like credentials, endpoint, etc.
-        storage_options_provider : optional, StorageOptionsProvider
-            A storage options provider that can fetch and refresh storage options
-            dynamically. This is useful for credentials that expire and need to be
-            refreshed automatically.
+        namespace_client : optional, LanceNamespace
+            A namespace client for automatic credential refresh.
+            Must be provided together with table_id.
+        table_id : optional, List[str]
+            The table identifier within the namespace.
+            Must be provided together with namespace_client.
         max_page_bytes : optional, int
             The maximum size of a page in bytes, if a single array would create a
             page larger than this then it will be split into multiple pages. The
@@ -427,7 +442,8 @@ def __init__(
                 data_cache_bytes=data_cache_bytes,
                 version=version,
                 storage_options=storage_options,
-                storage_options_provider=storage_options_provider,
+                namespace_client=namespace_client,
+                table_id=table_id,
                 max_page_bytes=max_page_bytes,
                 **kwargs,
             )
diff --git a/python/python/lance/fragment.py b/python/python/lance/fragment.py
index f8cff450f06..8e762e24fdf 100644
--- a/python/python/lance/fragment.py
+++ b/python/python/lance/fragment.py
@@ -11,6 +11,7 @@
 from pathlib import Path
 from typing import (
     TYPE_CHECKING,
+    Any,
     Callable,
     Dict,
     Iterator,
@@ -48,6 +49,7 @@
         Transaction,
     )
     from .lance import LanceSchema
+    from .namespace import LanceNamespace
 
 
 DEFAULT_MAX_BYTES_PER_FILE = 90 * 1024 * 1024 * 1024
@@ -142,7 +144,7 @@ def from_json(json_data: str) -> FragmentMetadata:
 
         row_id_meta = json_data.get("row_id_meta")
         if row_id_meta is not None:
-            row_id_meta = RowIdMeta(**row_id_meta)
+            row_id_meta = RowIdMeta.from_dict(row_id_meta)
 
         created_at_version_meta = json_data.get("created_at_version_meta")
         if created_at_version_meta is not None:
@@ -249,6 +251,36 @@ def field_ids(self) -> List[int]:
         )
         return self.fields
 
+    @classmethod
+    def create(
+        cls,
+        dataset: "LanceDataset",
+        path: str,
+        *,
+        base_id: Optional[int] = None,
+    ) -> "DataFile":
+        """Create a DataFile by reading metadata from an existing lance file.
+
+        This is a convenience method for creating DataFile metadata needed
+        for operations like DataReplacement. It opens the file, reads its
+        schema and version information, matches columns to the dataset's
+        schema to determine field IDs, and calculates column indices.
+
+        Parameters
+        ----------
+        dataset : LanceDataset
+            The dataset this file will belong to.
+        path : str
+            The path to the data file, relative to the dataset's data directory.
+        base_id : int, optional
+            The base path ID if the file is outside the dataset directory.
+
+        Returns
+        -------
+        DataFile
+        """
+        return _Fragment.create_data_file(dataset._ds, path, base_id=base_id)
+
 
 class LanceFragment(pa.dataset.Fragment):
     def __init__(
@@ -314,6 +346,8 @@ def create(
         data_storage_version: Optional[str] = None,
         use_legacy_format: Optional[bool] = None,
         storage_options: Optional[Dict[str, str]] = None,
+        namespace_client: Optional["LanceNamespace"] = None,
+        table_id: Optional[List[str]] = None,
     ) -> FragmentMetadata:
         """Create a :class:`FragmentMetadata` from the given data.
 
@@ -353,6 +387,15 @@ def create(
         storage_options : optional, dict
             Extra options that make sense for a particular storage connection. This is
             used to store connection parameters like credentials, endpoint, etc.
+        namespace_client : optional, LanceNamespace
+            A namespace client for automatic credential refresh. When provided with
+            `table_id`, a storage options provider will be created automatically to
+            refresh credentials via the namespace. Must be provided together with
+            `table_id`. The caller should provide initial/merged storage options via
+            the `storage_options` parameter.
+        table_id : optional, List[str]
+            The table identifier when using a namespace (e.g., ["my_table"]).
+            Must be provided together with `namespace_client`.
 
         See Also
         --------
@@ -368,6 +411,16 @@ def create(
         -------
         FragmentMetadata
         """
+        # Validate namespace_client and table_id are provided together
+        if namespace_client is not None and table_id is None:
+            raise ValueError(
+                "Both 'namespace_client' and 'table_id' must be provided together."
+            )
+        elif table_id is not None and namespace_client is None:
+            raise ValueError(
+                "Both 'namespace_client' and 'table_id' must be provided together."
+            )
+
         if use_legacy_format is not None:
             warnings.warn(
                 "use_legacy_format is deprecated, use data_storage_version instead",
@@ -394,6 +447,8 @@ def create(
             mode=mode,
             data_storage_version=data_storage_version,
             storage_options=storage_options,
+            namespace_client=namespace_client,
+            table_id=table_id,
         )
 
     @property
@@ -477,7 +532,40 @@ def scanner(
         )
         from .dataset import LanceScanner
 
-        return LanceScanner(s, self._ds)
+        snapshot = {
+            "_limit": limit,
+            "_filter": filter_str,
+            "_search_filter": None,
+            "_substrait_filter": None,
+            "_prefilter": False,
+            "_late_materialization": None,
+            "_blob_handling": blob_handling,
+            "_offset": offset,
+            "_columns": tuple(columns) if isinstance(columns, list) else None,
+            "_columns_with_transform": (
+                tuple(columns.items()) if isinstance(columns, dict) else None
+            ),
+            "_nearest": None,
+            "_batch_size": batch_size,
+            "_io_buffer_size": None,
+            "_batch_readahead": batch_readahead,
+            "_fragment_readahead": None,
+            "_scan_in_order": True,
+            "_fragments": (self._fragment,),
+            "_with_row_id": with_row_id,
+            "_with_row_address": with_row_address,
+            "_use_stats": True,
+            "_fast_search": False,
+            "_full_text_query": None,
+            "_use_scalar_index": None,
+            "_include_deleted_rows": None,
+            "_scan_stats_callback": None,
+            "_strict_batch_size": False,
+            "_orderings": tuple(order_by) if order_by is not None else None,
+            "_disable_scoring_autoprojection": False,
+            "_substrait_aggregate": None,
+        }
+        return LanceScanner(s, self._ds, snapshot=snapshot)
 
     def open_session(
         self,
@@ -561,6 +649,44 @@ def to_table(
             order_by=order_by,
         ).to_table()
 
+    def to_pandas(
+        self,
+        columns: Optional[Union[List[str], Dict[str, str]]] = None,
+        filter: Optional[Union[str, pa.compute.Expression]] = None,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        with_row_id: bool = False,
+        with_row_address: bool = False,
+        blob_mode: str = "lazy",
+        order_by: Optional[List[ColumnOrdering]] = None,
+        **kwargs: Any,
+    ) -> Any:
+        """Read this fragment into a :py:class:`pandas.DataFrame`.
+
+        Parameters are the same as :meth:`to_table`, except pandas export uses
+        ``blob_mode`` instead of Arrow-facing ``blob_handling``.
+
+        Parameters
+        ----------
+        blob_mode: str, default "lazy"
+            Controls how blob columns are returned.
+
+            - ``"lazy"``: return :class:`lance.BlobFile` objects
+            - ``"bytes"``: return Python ``bytes``
+            - ``"descriptions"``: preserve ``to_table().to_pandas()`` behavior
+        **kwargs
+            Forwarded to :meth:`pyarrow.Table.to_pandas` for non-blob columns.
+        """
+        return self.scanner(
+            columns=columns,
+            filter=filter,
+            limit=limit,
+            offset=offset,
+            with_row_id=with_row_id,
+            with_row_address=with_row_address,
+            order_by=order_by,
+        ).to_pandas(blob_mode=blob_mode, **kwargs)
+
     def merge(
         self,
         data_obj: ReaderLike,
@@ -877,10 +1003,11 @@ def write_fragments(
         data_storage_version: Optional[str] = None,
         use_legacy_format: Optional[bool] = None,
         storage_options: Optional[Dict[str, str]] = None,
-        storage_options_provider=None,
         enable_stable_row_ids: bool = False,
         target_bases: Optional[List[str]] = None,
         initial_bases: Optional[List["DatasetBasePath"]] = None,
+        namespace_client: Optional[LanceNamespace] = None,
+        table_id: Optional[List[str]] = None,
     ) -> Transaction: ...
 
     @overload
@@ -898,10 +1025,11 @@ def write_fragments(
         data_storage_version: Optional[str] = None,
         use_legacy_format: Optional[bool] = None,
         storage_options: Optional[Dict[str, str]] = None,
-        storage_options_provider=None,
         enable_stable_row_ids: bool = False,
         target_bases: Optional[List[str]] = None,
         initial_bases: Optional[List["DatasetBasePath"]] = None,
+        namespace_client: Optional[LanceNamespace] = None,
+        table_id: Optional[List[str]] = None,
     ) -> List[FragmentMetadata]: ...
 
 
@@ -919,10 +1047,11 @@ def write_fragments(
     data_storage_version: Optional[str] = None,
     use_legacy_format: Optional[bool] = None,
     storage_options: Optional[Dict[str, str]] = None,
-    storage_options_provider=None,
     enable_stable_row_ids: bool = False,
     target_bases: Optional[List[str]] = None,
     initial_bases: Optional[List["DatasetBasePath"]] = None,
+    namespace_client: Optional[LanceNamespace] = None,
+    table_id: Optional[List[str]] = None,
 ) -> List[FragmentMetadata] | Transaction:
     """
     Write data into one or more fragments.
@@ -971,10 +1100,6 @@ def write_fragments(
     storage_options : Optional[Dict[str, str]]
         Extra options that make sense for a particular storage connection. This is
         used to store connection parameters like credentials, endpoint, etc.
-    storage_options_provider : Optional[StorageOptionsProvider]
-        A storage options provider that can fetch and refresh storage options
-        dynamically. This is useful for credentials that expire and need to be
-        refreshed automatically.
     enable_stable_row_ids: bool
         Experimental: if set to true, the writer will use stable row ids.
         These row ids are stable after compaction operations, but not after updates.
@@ -1002,6 +1127,15 @@ def write_fragments(
 
         **Only valid in CREATE mode**. Will raise an error if used with
         APPEND/OVERWRITE modes.
+    namespace_client : optional, LanceNamespace
+        A namespace client for automatic credential refresh. When provided with
+        `table_id`, a storage options provider will be created automatically to
+        refresh credentials via the namespace. Must be provided together with
+        `table_id`. The caller should provide initial/merged storage options via
+        the `storage_options` parameter.
+    table_id : optional, List[str]
+        The table identifier when using a namespace (e.g., ["my_table"]).
+        Must be provided together with `namespace_client`.
 
     Returns
     -------
@@ -1019,6 +1153,16 @@ def write_fragments(
     """
     from .dataset import LanceDataset
 
+    # Validate namespace_client and table_id are provided together
+    if namespace_client is not None and table_id is None:
+        raise ValueError(
+            "Both 'namespace_client' and 'table_id' must be provided together."
+        )
+    elif table_id is not None and namespace_client is None:
+        raise ValueError(
+            "Both 'namespace_client' and 'table_id' must be provided together."
+        )
+
     reader = _coerce_reader(data, schema)
 
     if isinstance(dataset_uri, Path):
@@ -1050,7 +1194,8 @@ def write_fragments(
         progress=progress,
         data_storage_version=data_storage_version,
         storage_options=storage_options,
-        storage_options_provider=storage_options_provider,
+        namespace_client=namespace_client,
+        table_id=table_id,
         enable_stable_row_ids=enable_stable_row_ids,
         target_bases=target_bases,
         initial_bases=initial_bases,
diff --git a/python/python/lance/indices/__init__.py b/python/python/lance/indices/__init__.py
index ac586876da0..edf9e5091ff 100644
--- a/python/python/lance/indices/__init__.py
+++ b/python/python/lance/indices/__init__.py
@@ -3,11 +3,21 @@
 
 from enum import Enum
 
-from lance.indices.builder import IndexConfig, IndicesBuilder
-from lance.indices.ivf import IvfModel
-from lance.indices.pq import PqModel
+from .. import lance as _lance
+from .builder import IndexConfig, IndicesBuilder
+from .ivf import IvfModel
+from .pq import PqModel
 
-__all__ = ["IndicesBuilder", "IndexConfig", "PqModel", "IvfModel", "IndexFileVersion"]
+IndexSegmentDescription = _lance.indices.IndexSegmentDescription
+
+__all__ = [
+    "IndicesBuilder",
+    "IndexConfig",
+    "PqModel",
+    "IvfModel",
+    "IndexFileVersion",
+    "IndexSegmentDescription",
+]
 
 
 class IndexFileVersion(str, Enum):
@@ -19,9 +29,11 @@ class SupportedDistributedIndices(str, Enum):
     # Scalar index types
     BTREE = "BTREE"
     INVERTED = "INVERTED"
+
     # Precise vector index types supported by distributed merge
     IVF_FLAT = "IVF_FLAT"
     IVF_PQ = "IVF_PQ"
     IVF_SQ = "IVF_SQ"
+
     # Deprecated generic placeholder (kept for backward compatibility)
     VECTOR = "VECTOR"
diff --git a/python/python/lance/indices/builder.py b/python/python/lance/indices/builder.py
index ca033780a0e..c31ea0a7a0c 100644
--- a/python/python/lance/indices/builder.py
+++ b/python/python/lance/indices/builder.py
@@ -51,7 +51,8 @@ def __init__(self, dataset, column: str):
             the dataset containing the data
         column: str
             The vector column to index, must be a fixed size list of floats
-            or 1-dimensional fixed-shape tensor column.
+            (or unsigned integers for hamming distance) or 1-dimensional
+            fixed-shape tensor column.
         """
         self.dataset = dataset
         self.column = self._normalize_column(column)
@@ -65,6 +66,7 @@ def train_ivf(
         accelerator: Optional[Union[str, "torch.Device"]] = None,
         sample_rate: int = 256,
         max_iters: int = 50,
+        fragment_ids: Optional[list[int]] = None,
     ) -> IvfModel:
         """
         Train IVF centroids for the given vector column.
@@ -89,7 +91,7 @@ def train_ivf(
             overtraining, reduced recall, and require large nprobes values.  If not
             specified the default will be the integer nearest the square root of the
             number of rows.
-        distance_type: "l2" | "dot" | "cosine"
+        distance_type: "l2" | "dot" | "cosine" | "hamming"
             The distance type to used.  This is defined in more detail in the LanceDB
             documentation on creating indices.
         accelerator: str | torch.Device
@@ -105,8 +107,10 @@ def train_ivf(
             some cases, k-means will not converge but will cycle between various
             possible minima.  In these cases we must terminate or run forever.  The
             max_iters parameter defines a cutoff at which we terminate training.
+        fragment_ids: list[int], optional
+            If provided, train using only the specified fragments from the dataset.
         """
-        num_rows = self.dataset.count_rows()
+        num_rows = self._count_rows(fragment_ids)
         num_partitions = self._determine_num_partitions(num_partitions, num_rows)
         self._verify_ivf_sample_rate(sample_rate, num_partitions, num_rows)
         distance_type = self._normalize_distance_type(distance_type)
@@ -123,9 +127,14 @@ def train_ivf(
                 distance_type,
                 sample_rate,
                 max_iters,
+                fragment_ids,
             )
             return IvfModel(ivf_centroids, distance_type)
         else:
+            if fragment_ids is not None:
+                raise NotImplementedError(
+                    "fragment_ids is not supported with accelerator IVF training"
+                )
             # Use accelerator to train ivf centroids
             from lance.vector import train_ivf_centroids_on_accelerator
 
@@ -153,6 +162,7 @@ def train_pq(
         *,
         sample_rate: int = 256,
         max_iters: int = 50,
+        fragment_ids: Optional[list[int]] = None,
     ) -> PqModel:
         """
         Train a PQ model for a given column.
@@ -183,10 +193,12 @@ def train_pq(
             This parameter is used in the same way as in the IVF model.
         max_iters: int
             This parameter is used in the same way as in the IVF model.
+        fragment_ids: list[int], optional
+            If provided, train using only the specified fragments from the dataset.
         """
         from lance.lance import indices
 
-        num_rows = self.dataset.count_rows()
+        num_rows = self._count_rows(fragment_ids)
         self.dataset.schema.field(self.column[0]).type.list_size
         num_subvectors = self._normalize_pq_params(num_subvectors, self.dimension)
         self._verify_pq_sample_rate(num_rows, sample_rate)
@@ -200,6 +212,7 @@ def train_pq(
             sample_rate,
             max_iters,
             ivf_model.centroids,
+            fragment_ids,
         )
         return PqModel(num_subvectors, pq_codebook)
 
@@ -212,11 +225,17 @@ def prepare_global_ivf_pq(
         accelerator: Optional[Union[str, "torch.Device"]] = None,
         sample_rate: int = 256,
         max_iters: int = 50,
+        fragment_ids: Optional[list[int]] = None,
     ) -> dict:
         """
         Perform global training for IVF+PQ using existing CPU training paths and
         return preprocessed artifacts for distributed builds.
 
+        Parameters
+        ----------
+        fragment_ids: list[int], optional
+            If provided, train using only the specified fragments from the dataset.
+
         Returns
         -------
         dict
@@ -238,6 +257,7 @@ def prepare_global_ivf_pq(
             accelerator=accelerator,  # None by default (CPU path)
             sample_rate=sample_rate,
             max_iters=max_iters,
+            fragment_ids=fragment_ids,
         )
 
         # Global PQ training using IVF residuals
@@ -246,6 +266,7 @@ def prepare_global_ivf_pq(
             num_subvectors,
             sample_rate=sample_rate,
             max_iters=max_iters,
+            fragment_ids=fragment_ids,
         )
 
         return {"ivf_centroids": ivf_model.centroids, "pq_codebook": pq_model.codebook}
@@ -458,6 +479,18 @@ def _determine_num_partitions(self, num_partitions: Optional[int], num_rows: int
             return round(math.sqrt(num_rows))
         return num_partitions
 
+    def _count_rows(self, fragment_ids: Optional[list[int]] = None) -> int:
+        if fragment_ids is None:
+            return self.dataset.count_rows()
+
+        num_rows = 0
+        for fragment_id in fragment_ids:
+            fragment = self.dataset.get_fragment(fragment_id)
+            if fragment is None:
+                raise ValueError(f"Fragment id does not exist: {fragment_id}")
+            num_rows += fragment.count_rows()
+        return num_rows
+
     def _normalize_pq_params(self, num_subvectors: int, dimension: int):
         if num_subvectors is None:
             if dimension % 16 == 0:
@@ -529,6 +562,7 @@ def _normalize_distance_type(self, distance_type):
             "cosine",
             "euclidean",
             "dot",
+            "hamming",
         ]:
             raise ValueError(f"Distance type {distance_type} not supported.")
         return distance_type.lower()
@@ -555,10 +589,13 @@ def _normalize_column(self, column):
                     f"Vector column {c} must be FixedSizeListArray "
                     f"1-dimensional FixedShapeTensorArray, got {field.type}"
                 )
-            if not pa.types.is_floating(field.type.value_type):
+            if not (
+                pa.types.is_floating(field.type.value_type)
+                or pa.types.is_unsigned_integer(field.type.value_type)
+            ):
                 raise TypeError(
-                    f"Vector column {c} must have floating value type, "
-                    f"got {field.type.value_type}"
+                    f"Vector column {c} must have floating or unsigned integer "
+                    f"value type, got {field.type.value_type}"
                 )
 
         return column
diff --git a/python/python/lance/indices/ivf.py b/python/python/lance/indices/ivf.py
index fa92f744d55..fef19dde73a 100644
--- a/python/python/lance/indices/ivf.py
+++ b/python/python/lance/indices/ivf.py
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The Lance Authors
 
+from typing import Dict, Optional
+
 import pyarrow as pa
 
 from lance.file import LanceFileReader, LanceFileWriter
@@ -24,7 +26,7 @@ def num_partitions(self) -> int:
         """
         return len(self.centroids)
 
-    def save(self, uri: str):
+    def save(self, uri: str, *, storage_options: Optional[Dict[str, str]] = None):
         """
         Save the IVF model to a lance file.
 
@@ -34,6 +36,8 @@ def save(self, uri: str):
         uri: str
             The URI to save the model to.  The URI can be a local file path or a
             cloud storage path.
+        storage_options : optional, dict
+            Extra options for the storage backend (e.g. S3 credentials).
         """
         with LanceFileWriter(
             uri,
@@ -41,12 +45,13 @@ def save(self, uri: str):
                 [pa.field("centroids", self.centroids.type)],
                 metadata={b"distance_type": self.distance_type.encode()},
             ),
+            storage_options=storage_options,
         ) as writer:
             batch = pa.table([self.centroids], names=["centroids"])
             writer.write_batch(batch)
 
     @classmethod
-    def load(cls, uri: str):
+    def load(cls, uri: str, *, storage_options: Optional[Dict[str, str]] = None):
         """
         Load an IVF model from a lance file.
 
@@ -56,8 +61,10 @@ def load(cls, uri: str):
         uri: str
             The URI to load the model from.  The URI can be a local file path or a
             cloud storage path.
+        storage_options : optional, dict
+            Extra options for the storage backend (e.g. S3 credentials).
         """
-        reader = LanceFileReader(uri)
+        reader = LanceFileReader(uri, storage_options=storage_options)
         num_rows = reader.metadata().num_rows
         metadata = reader.metadata().schema.metadata
         distance_type = metadata[b"distance_type"].decode()
diff --git a/python/python/lance/indices/pq.py b/python/python/lance/indices/pq.py
index 09f34f04dfe..b3aeb50bcbe 100644
--- a/python/python/lance/indices/pq.py
+++ b/python/python/lance/indices/pq.py
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The Lance Authors
 
+from typing import Dict, Optional
+
 import pyarrow as pa
 
 from lance.file import LanceFileReader, LanceFileWriter
@@ -23,7 +25,7 @@ def dimension(self):
         """The dimension of the vectors this model was trained on"""
         return self.codebook.type.list_size
 
-    def save(self, uri: str):
+    def save(self, uri: str, *, storage_options: Optional[Dict[str, str]] = None):
         """
         Save the PQ model to a lance file.
 
@@ -33,6 +35,8 @@ def save(self, uri: str):
         uri: str
             The URI to save the model to.  The URI can be a local file path or a
             cloud storage path.
+        storage_options : optional, dict
+            Extra options for the storage backend (e.g. S3 credentials).
         """
         with LanceFileWriter(
             uri,
@@ -40,12 +44,13 @@ def save(self, uri: str):
                 [pa.field("codebook", self.codebook.type)],
                 metadata={b"num_subvectors": str(self.num_subvectors).encode()},
             ),
+            storage_options=storage_options,
         ) as writer:
             batch = pa.table([self.codebook], names=["codebook"])
             writer.write_batch(batch)
 
     @classmethod
-    def load(cls, uri: str):
+    def load(cls, uri: str, *, storage_options: Optional[Dict[str, str]] = None):
         """
         Load a PQ model from a lance file.
 
@@ -55,8 +60,10 @@ def load(cls, uri: str):
         uri: str
             The URI to load the model from.  The URI can be a local file path or a
             cloud storage path.
+        storage_options : optional, dict
+            Extra options for the storage backend (e.g. S3 credentials).
         """
-        reader = LanceFileReader(uri)
+        reader = LanceFileReader(uri, storage_options=storage_options)
         num_rows = reader.metadata().num_rows
         metadata = reader.metadata().schema.metadata
         num_subvectors = int(metadata[b"num_subvectors"].decode())
diff --git a/python/python/lance/io.py b/python/python/lance/io.py
index b12d6dc106f..bfda99d22e7 100644
--- a/python/python/lance/io.py
+++ b/python/python/lance/io.py
@@ -4,137 +4,5 @@
 """I/O utilities for Lance datasets.
 
 This module provides utilities for customizing how Lance datasets interact with
-cloud storage, including credential management for long-running operations.
+cloud storage.
 """
-
-from abc import ABC, abstractmethod
-from typing import Dict
-
-
-class StorageOptionsProvider(ABC):
-    """Abstract base class for providing storage options to Lance datasets.
-
-    Storage options providers enable automatic refresh for long-running operations
-    on cloud storage (S3, Azure, GCS). This is currently only used for refreshing
-    AWS temporary access credentials. Implement this interface to integrate with
-    custom credential management systems such as AWS STS, GCP STS, or
-    proprietary credential services.
-
-    The provider is called automatically before storage options expire, ensuring
-    uninterrupted access during long-running queries, training jobs, or data processing.
-
-    Example
-    -------
-    >>> import lance
-    >>> class MyStorageOptionsProvider(StorageOptionsProvider):
-    ...     def fetch_storage_options(self):
-    ...         # Fetch from your credential service
-    ...         return {
-    ...             "aws_access_key_id": "ASIA...",
-    ...             "aws_secret_access_key": "secret",
-    ...             "aws_session_token": "token",
-    ...             "expires_at_millis": "1234567890000",
-    ...         }
-    ...
-    >>> provider = MyStorageOptionsProvider()
-    >>> dataset = lance.dataset(  # doctest: +SKIP
-    ...     "s3://bucket/table.lance", storage_options_provider=provider
-    ... )
-
-    Error Handling
-    --------------
-    If fetch_storage_options() raises an exception, operations requiring
-    credentials will fail. Implementations should handle recoverable errors
-    internally (e.g., retry token refresh) and only raise exceptions for
-    unrecoverable errors.
-    """
-
-    @abstractmethod
-    def fetch_storage_options(self) -> Dict[str, str]:
-        """Get fresh storage credentials.
-
-        This method is called automatically before each request and before existing
-        credentials expire. It must return credentials in the format below.
-
-        Returns
-        -------
-        Dict[str, str]
-            Dictionary of string key-value pairs containing cloud storage credentials
-            and expiration time. Required keys:
-
-            - "expires_at_millis" (str): Unix timestamp in milliseconds (as string)
-              when credentials expire. Lance will automatically call
-              fetch_storage_options() again before this time.
-
-            Plus provider-specific credential keys:
-
-            AWS S3:
-              - "aws_access_key_id" (str): AWS access key
-              - "aws_secret_access_key" (str): AWS secret key
-              - "aws_session_token" (str, optional): Session token for temporary
-                credentials
-
-            Azure Blob Storage:
-              - "account_name" (str): Storage account name
-              - "account_key" (str): Storage account key
-              - Or "sas_token" (str): SAS token
-
-            Google Cloud Storage:
-              - "service_account_key" (str): Service account JSON key
-              - Or "token" (str): OAuth token
-
-        Raises
-        ------
-        Exception
-            If unable to fetch credentials, the exception will be propagated
-            and operations requiring credentials will fail.
-
-        Example
-        -------
-        >>> def fetch_storage_options(self):
-        ...     # Example: AWS temporary credentials
-        ...     response = sts_client.assume_role(
-        ...         RoleArn='arn:aws:iam::123456789012:role/DataReader',
-        ...         RoleSessionName='lance-session'
-        ...     )
-        ...     creds = response['Credentials']
-        ...     expires_at_millis = int(creds['Expiration'].timestamp() * 1000)
-        ...     return {
-        ...         "aws_access_key_id": creds['AccessKeyId'],
-        ...         "aws_secret_access_key": creds['SecretAccessKey'],
-        ...         "aws_session_token": creds['SessionToken'],
-        ...         "expires_at_millis": str(expires_at_millis),
-        ...     }
-        """
-        pass
-
-    def provider_id(self) -> str:
-        """Return a human-readable unique identifier for this provider instance.
-
-        This is used for equality comparison and hashing in the object store
-        registry. Two providers with the same ID will be treated as equal and
-        share the same cached ObjectStore instance.
-
-        The default implementation uses the class name and object's string
-        representation. Override this method to provide semantic equality based
-        on configuration.
-
-        Returns
-        -------
-        str
-            A human-readable unique identifier string.
-            For example: "MyProvider { endpoint: 'https://api.example.com' }"
-
-        Example
-        -------
-        >>> class MyProvider(StorageOptionsProvider):
-        ...     def __init__(self, endpoint):
-        ...         self.endpoint = endpoint
-        ...
-        ...     def fetch_storage_options(self):
-        ...         return {"expires_at_millis": "1234567890000"}
-        ...
-        ...     def provider_id(self):
-        ...         return f"MyProvider {{ endpoint: {self.endpoint!r} }}"
-        """
-        return f"{self.__class__.__name__} {{ repr: {str(self)!r} }}"
diff --git a/python/python/lance/lance/__init__.pyi b/python/python/lance/lance/__init__.pyi
index 41db5678b0b..82eaae3c97c 100644
--- a/python/python/lance/lance/__init__.pyi
+++ b/python/python/lance/lance/__init__.pyi
@@ -28,6 +28,7 @@ from typing import (
 )
 
 import pyarrow as pa
+from lance_namespace import LanceNamespace
 
 from .._arrow.bf16 import BFloat16Array
 from ..commit import CommitLock
@@ -46,8 +47,8 @@ from ..fragment import (
     DataFile,
     FragmentMetadata,
 )
-from ..io import StorageOptionsProvider
 from ..progress import FragmentWriteProgress as FragmentWriteProgress
+from ..progress import IndexProgress as IndexProgress
 from ..types import ReaderLike as ReaderLike
 from ..udf import BatchUDF as BatchUDF
 from .debug import format_fragment as format_fragment
@@ -61,6 +62,7 @@ from .fragment import (
     RowIdMeta as RowIdMeta,
 )
 from .indices import IndexDescription as IndexDescription
+from .indices import IndexSegmentDescription as IndexSegmentDescription
 from .lance import PySearchFilter
 from .optimize import (
     Compaction as Compaction,
@@ -99,7 +101,8 @@ class LanceFileWriter:
         data_cache_bytes: Optional[int],
         version: Optional[str],
         storage_options: Optional[Dict[str, str]],
-        storage_options_provider: Optional[StorageOptionsProvider],
+        namespace_client: Optional[LanceNamespace],
+        table_id: Optional[List[str]],
         keep_original_array: Optional[bool],
         max_page_bytes: Optional[int],
     ): ...
@@ -113,7 +116,8 @@ class LanceFileSession:
         self,
         base_path: str,
         storage_options: Optional[Dict[str, str]] = None,
-        storage_options_provider: Optional[StorageOptionsProvider] = None,
+        namespace_client: Optional[LanceNamespace] = None,
+        table_id: Optional[List[str]] = None,
     ): ...
     def open_reader(
         self, path: str, columns: Optional[List[str]] = None
@@ -137,7 +141,8 @@ class LanceFileReader:
         self,
         path: str,
         storage_options: Optional[Dict[str, str]],
-        storage_options_provider: Optional[StorageOptionsProvider],
+        namespace_client: Optional[LanceNamespace],
+        table_id: Optional[List[str]],
         columns: Optional[List[str]] = None,
     ): ...
     def read_all(
@@ -283,6 +288,27 @@ class _Dataset:
         row_indices: List[int],
         blob_column: str,
     ) -> List[LanceBlobFile]: ...
+    def read_blobs(
+        self,
+        row_ids: List[int],
+        blob_column: str,
+        io_buffer_size: Optional[int] = None,
+        preserve_order: Optional[bool] = None,
+    ) -> List[Tuple[int, bytes]]: ...
+    def read_blobs_by_addresses(
+        self,
+        row_addresses: List[int],
+        blob_column: str,
+        io_buffer_size: Optional[int] = None,
+        preserve_order: Optional[bool] = None,
+    ) -> List[Tuple[int, bytes]]: ...
+    def read_blobs_by_indices(
+        self,
+        row_indices: List[int],
+        blob_column: str,
+        io_buffer_size: Optional[int] = None,
+        preserve_order: Optional[bool] = None,
+    ) -> List[Tuple[int, bytes]]: ...
     def take_scan(
         self,
         row_slices: Iterable[Tuple[int, int]],
@@ -356,10 +382,18 @@ class _Dataset:
         kwargs: Optional[Dict[str, Any]] = None,
     ): ...
     def drop_index(self, name: str): ...
-    def prewarm_index(self, name: str): ...
+    def prewarm_index(self, name: str, *, with_position: bool = False): ...
     def merge_index_metadata(
-        self, index_uuid: str, index_type: str, batch_readhead: Optional[int] = None
+        self,
+        index_uuid: str,
+        index_type: str,
+        batch_readhead: Optional[int] = None,
+        progress_callback: Optional[Callable[[IndexProgress], None]] = None,
     ): ...
+    def merge_existing_index_segments(self, segments: List[Index]) -> Index: ...
+    def commit_existing_index_segments(
+        self, index_name: str, column: str, segments: List[Index]
+    ) -> None: ...
     def count_fragments(self) -> int: ...
     def num_small_files(self, max_rows_per_group: int) -> int: ...
     def get_fragments(self) -> List[_Fragment]: ...
@@ -380,7 +414,6 @@ class _Dataset:
         read_version: Optional[int] = None,
         commit_lock: Optional[CommitLock] = None,
         storage_options: Optional[Dict[str, str]] = None,
-        storage_options_provider: Optional[StorageOptionsProvider] = None,
         enable_v2_manifest_paths: Optional[bool] = None,
         detached: Optional[bool] = None,
         max_retries: Optional[int] = None,
@@ -393,7 +426,6 @@ class _Dataset:
         transactions: Sequence[Transaction],
         commit_lock: Optional[CommitLock] = None,
         storage_options: Optional[Dict[str, str]] = None,
-        storage_options_provider: Optional[StorageOptionsProvider] = None,
         enable_v2_manifest_paths: Optional[bool] = None,
         detached: Optional[bool] = None,
         max_retries: Optional[int] = None,
@@ -504,6 +536,8 @@ def _write_fragments(
     progress: Optional[FragmentWriteProgress],
     data_storage_version: Optional[str],
     storage_options: Optional[Dict[str, str]],
+    namespace_client: Optional[LanceNamespace],
+    table_id: Optional[List[str]],
     enable_stable_row_ids: bool,
 ): ...
 def _write_fragments_transaction(
@@ -516,6 +550,8 @@ def _write_fragments_transaction(
     progress: Optional[FragmentWriteProgress],
     data_storage_version: Optional[str],
     storage_options: Optional[Dict[str, str]],
+    namespace_client: Optional[LanceNamespace],
+    table_id: Optional[List[str]],
     enable_stable_row_ids: bool,
 ) -> Transaction: ...
 def _json_to_schema(schema_json: str) -> pa.Schema: ...
diff --git a/python/python/lance/lance/indices/__init__.pyi b/python/python/lance/lance/indices/__init__.pyi
index feed1f934fe..152ea1d10b2 100644
--- a/python/python/lance/lance/indices/__init__.pyi
+++ b/python/python/lance/lance/indices/__init__.pyi
@@ -29,6 +29,7 @@ def train_ivf_model(
     distance_type: str,
     sample_rate: int,
     max_iters: int,
+    fragment_ids: Optional[list[int]] = None,
 ) -> pa.Array: ...
 def train_pq_model(
     dataset,
@@ -39,6 +40,7 @@ def train_pq_model(
     sample_rate: int,
     max_iters: int,
     ivf_model: pa.Array,
+    fragment_ids: Optional[list[int]] = None,
 ) -> pa.Array: ...
 def transform_vectors(
     dataset,
@@ -57,6 +59,7 @@ class IndexSegmentDescription:
     fragment_ids: set[int]
     index_version: int
     created_at: Optional[datetime]
+    size_bytes: Optional[int]
 
     def __repr__(self) -> str: ...
 
@@ -69,5 +72,6 @@ class IndexDescription:
     field_names: list[str]
     segments: list[IndexSegmentDescription]
     details: dict
+    total_size_bytes: Optional[int]
 
     def __repr__(self) -> str: ...
diff --git a/python/python/lance/namespace.py b/python/python/lance/namespace.py
index 63ad2abd007..7d283b69579 100644
--- a/python/python/lance/namespace.py
+++ b/python/python/lance/namespace.py
@@ -13,41 +13,85 @@
 """
 
 from abc import ABC, abstractmethod
-from typing import Dict, List, Optional
+from typing import Dict, Optional
 
 from lance_namespace import (
+    AlterTableAddColumnsRequest,
+    AlterTableAddColumnsResponse,
+    AlterTableAlterColumnsRequest,
+    AlterTableAlterColumnsResponse,
+    AlterTableDropColumnsRequest,
+    AlterTableDropColumnsResponse,
+    AlterTransactionRequest,
+    AlterTransactionResponse,
+    AnalyzeTableQueryPlanRequest,
+    CountTableRowsRequest,
     CreateNamespaceRequest,
     CreateNamespaceResponse,
+    CreateTableIndexRequest,
+    CreateTableIndexResponse,
     CreateTableRequest,
     CreateTableResponse,
+    CreateTableTagRequest,
+    CreateTableTagResponse,
     DeclareTableRequest,
     DeclareTableResponse,
+    DeleteFromTableRequest,
+    DeleteFromTableResponse,
+    DeleteTableTagRequest,
+    DeleteTableTagResponse,
     DeregisterTableRequest,
     DeregisterTableResponse,
     DescribeNamespaceRequest,
     DescribeNamespaceResponse,
+    DescribeTableIndexStatsRequest,
+    DescribeTableIndexStatsResponse,
     DescribeTableRequest,
     DescribeTableResponse,
+    DescribeTransactionRequest,
+    DescribeTransactionResponse,
     DropNamespaceRequest,
     DropNamespaceResponse,
+    DropTableIndexRequest,
+    DropTableIndexResponse,
     DropTableRequest,
     DropTableResponse,
+    ExplainTableQueryPlanRequest,
+    GetTableStatsRequest,
+    GetTableStatsResponse,
+    GetTableTagVersionRequest,
+    GetTableTagVersionResponse,
+    InsertIntoTableRequest,
+    InsertIntoTableResponse,
     LanceNamespace,
     ListNamespacesRequest,
     ListNamespacesResponse,
+    ListTableIndicesRequest,
+    ListTableIndicesResponse,
     ListTablesRequest,
     ListTablesResponse,
+    ListTableTagsRequest,
+    ListTableTagsResponse,
     ListTableVersionsRequest,
     ListTableVersionsResponse,
+    MergeInsertIntoTableRequest,
+    MergeInsertIntoTableResponse,
     NamespaceExistsRequest,
     RegisterTableRequest,
     RegisterTableResponse,
     RenameTableRequest,
     RenameTableResponse,
+    RestoreTableRequest,
+    RestoreTableResponse,
     TableExistsRequest,
+    UpdateTableRequest,
+    UpdateTableResponse,
+    UpdateTableSchemaMetadataRequest,
+    UpdateTableSchemaMetadataResponse,
+    UpdateTableTagRequest,
+    UpdateTableTagResponse,
 )
 
-from .io import StorageOptionsProvider
 from .lance import PyDirectoryNamespace  # Low-level Rust binding
 
 try:
@@ -64,7 +108,6 @@
     "DirectoryNamespace",
     "RestNamespace",
     "RestAdapter",
-    "LanceNamespaceStorageOptionsProvider",
     "DynamicContextProvider",
 ]
 
@@ -265,6 +308,16 @@ class DirectoryNamespace(LanceNamespace):
                 - credential_vendor.azure_duration_millis (optional): Duration in ms
                   (default: 3600000, up to 7 days)
 
+        Testing properties:
+            - ops_metrics_enabled (optional): "true" to enable operation metrics
+              tracking. Use `retrieve_ops_metrics()` to get call counts.
+            - vend_input_storage_options (optional): "true" to return input storage
+              options in describe_table() when no credential vendor is configured.
+              Useful for testing credential refresh.
+            - vend_input_storage_options_refresh_interval_millis (optional): When set
+              with vend_input_storage_options, adds expires_at_millis to storage
+              options. Value is current_time_millis + this interval.
+
     Examples
     --------
     >>> import lance.namespace
@@ -387,6 +440,10 @@ def declare_table(self, request: DeclareTableRequest) -> DeclareTableResponse:
         response_dict = self._inner.declare_table(request.model_dump())
         return DeclareTableResponse.from_dict(response_dict)
 
+    def rename_table(self, request: RenameTableRequest) -> RenameTableResponse:
+        response_dict = self._inner.rename_table(request.model_dump())
+        return RenameTableResponse.from_dict(response_dict)
+
     # Table version operations
 
     def list_table_versions(
@@ -451,6 +508,348 @@ def batch_delete_table_versions(self, request: dict) -> dict:
         """
         return self._inner.batch_delete_table_versions(request)
 
+    # Data manipulation operations
+
+    def count_table_rows(self, request: CountTableRowsRequest) -> int:
+        """Count the number of rows in a table, optionally filtered by a predicate.
+
+        Parameters
+        ----------
+        request : CountTableRowsRequest
+            Request with table id, optional version, and optional predicate filter
+
+        Returns
+        -------
+        int
+            The number of rows matching the criteria
+        """
+        return self._inner.count_table_rows(request.model_dump())
+
+    def insert_into_table(
+        self, request: InsertIntoTableRequest, request_data: bytes
+    ) -> InsertIntoTableResponse:
+        """Insert data into a table.
+
+        Parameters
+        ----------
+        request : InsertIntoTableRequest
+            Request with table id and write mode (append/overwrite)
+        request_data : bytes
+            Arrow IPC stream containing the data to insert
+
+        Returns
+        -------
+        InsertIntoTableResponse
+            Response with optional transaction_id
+        """
+        response_dict = self._inner.insert_into_table(
+            request.model_dump(), request_data
+        )
+        return InsertIntoTableResponse.from_dict(response_dict)
+
+    def merge_insert_into_table(
+        self, request: MergeInsertIntoTableRequest, request_data: bytes
+    ) -> MergeInsertIntoTableResponse:
+        """Merge-insert data into a table (upsert operation).
+
+        Parameters
+        ----------
+        request : MergeInsertIntoTableRequest
+            Request with table id, merge keys, and merge behavior
+        request_data : bytes
+            Arrow IPC stream containing the data to merge
+
+        Returns
+        -------
+        MergeInsertIntoTableResponse
+            Response with optional transaction_id
+        """
+        response_dict = self._inner.merge_insert_into_table(
+            request.model_dump(), request_data
+        )
+        return MergeInsertIntoTableResponse.from_dict(response_dict)
+
+    def update_table(self, request: UpdateTableRequest) -> UpdateTableResponse:
+        """Update rows in a table matching a filter.
+
+        Parameters
+        ----------
+        request : UpdateTableRequest
+            Request with table id, filter predicate, and column updates
+
+        Returns
+        -------
+        UpdateTableResponse
+            Response with optional transaction_id
+        """
+        response_dict = self._inner.update_table(request.model_dump())
+        return UpdateTableResponse.from_dict(response_dict)
+
+    def delete_from_table(
+        self, request: DeleteFromTableRequest
+    ) -> DeleteFromTableResponse:
+        """Delete rows from a table matching a filter.
+
+        Parameters
+        ----------
+        request : DeleteFromTableRequest
+            Request with table id and filter predicate
+
+        Returns
+        -------
+        DeleteFromTableResponse
+            Response with optional transaction_id
+        """
+        response_dict = self._inner.delete_from_table(request.model_dump())
+        return DeleteFromTableResponse.from_dict(response_dict)
+
+    def query_table(self, request) -> bytes:
+        """Query a table and return results as Arrow IPC.
+
+        Parameters
+        ----------
+        request : QueryTableRequest or dict
+            Query request with table id, optional filter, columns, vector search
+            parameters, limit/offset, etc.
+
+        Returns
+        -------
+        bytes
+            Arrow IPC file format containing the query results
+        """
+        if hasattr(request, "model_dump"):
+            request = request.model_dump()
+        return self._inner.query_table(request)
+
+    # Index operations
+
+    def create_table_index(
+        self, request: CreateTableIndexRequest
+    ) -> CreateTableIndexResponse:
+        """Create an index on a table.
+
+        Parameters
+        ----------
+        request : CreateTableIndexRequest
+            Request with table id, index name, column, and index configuration
+
+        Returns
+        -------
+        CreateTableIndexResponse
+            Response with optional transaction_id
+        """
+        response_dict = self._inner.create_table_index(request.model_dump())
+        return CreateTableIndexResponse.from_dict(response_dict)
+
+    def list_table_indices(
+        self, request: ListTableIndicesRequest
+    ) -> ListTableIndicesResponse:
+        """List all indices on a table.
+
+        Parameters
+        ----------
+        request : ListTableIndicesRequest
+            Request with table id
+
+        Returns
+        -------
+        ListTableIndicesResponse
+            Response with list of index metadata
+        """
+        response_dict = self._inner.list_table_indices(request.model_dump())
+        return ListTableIndicesResponse.from_dict(response_dict)
+
+    def describe_table_index_stats(
+        self, request: DescribeTableIndexStatsRequest
+    ) -> DescribeTableIndexStatsResponse:
+        """Get statistics for a specific index.
+
+        Parameters
+        ----------
+        request : DescribeTableIndexStatsRequest
+            Request with table id and index name
+
+        Returns
+        -------
+        DescribeTableIndexStatsResponse
+            Response with index statistics
+        """
+        response_dict = self._inner.describe_table_index_stats(request.model_dump())
+        return DescribeTableIndexStatsResponse.from_dict(response_dict)
+
+    # Transaction operations
+
+    def describe_transaction(
+        self, request: DescribeTransactionRequest
+    ) -> DescribeTransactionResponse:
+        """Describe a transaction.
+
+        Parameters
+        ----------
+        request : DescribeTransactionRequest
+            Request with transaction id
+
+        Returns
+        -------
+        DescribeTransactionResponse
+            Response with transaction details
+        """
+        response_dict = self._inner.describe_transaction(request.model_dump())
+        return DescribeTransactionResponse.from_dict(response_dict)
+
+    def alter_transaction(
+        self, request: AlterTransactionRequest
+    ) -> AlterTransactionResponse:
+        """Alter a transaction (commit or abort).
+
+        Parameters
+        ----------
+        request : AlterTransactionRequest
+            Request with transaction id and action
+
+        Returns
+        -------
+        AlterTransactionResponse
+            Response confirming the action
+        """
+        response_dict = self._inner.alter_transaction(request.model_dump())
+        return AlterTransactionResponse.from_dict(response_dict)
+
+    # Additional index operations
+
+    def create_table_scalar_index(
+        self, request: CreateTableIndexRequest
+    ) -> CreateTableIndexResponse:
+        """Create a scalar index on a table column."""
+        response_dict = self._inner.create_table_scalar_index(request.model_dump())
+        return CreateTableIndexResponse.from_dict(response_dict)
+
+    def drop_table_index(
+        self, request: DropTableIndexRequest
+    ) -> DropTableIndexResponse:
+        """Drop an index from a table."""
+        response_dict = self._inner.drop_table_index(request.model_dump())
+        return DropTableIndexResponse.from_dict(response_dict)
+
+    # Additional table operations
+
+    def list_all_tables(self, request: ListTablesRequest) -> ListTablesResponse:
+        """List all tables recursively across all namespaces."""
+        response_dict = self._inner.list_all_tables(request.model_dump())
+        return ListTablesResponse.from_dict(response_dict)
+
+    def restore_table(self, request: RestoreTableRequest) -> RestoreTableResponse:
+        """Restore a previously dropped table."""
+        response_dict = self._inner.restore_table(request.model_dump())
+        return RestoreTableResponse.from_dict(response_dict)
+
+    def update_table_schema_metadata(
+        self, request: UpdateTableSchemaMetadataRequest
+    ) -> UpdateTableSchemaMetadataResponse:
+        """Update the schema metadata of a table."""
+        response_dict = self._inner.update_table_schema_metadata(request.model_dump())
+        return UpdateTableSchemaMetadataResponse.from_dict(response_dict)
+
+    def get_table_stats(self, request: GetTableStatsRequest) -> GetTableStatsResponse:
+        """Get statistics for a table."""
+        response_dict = self._inner.get_table_stats(request.model_dump())
+        return GetTableStatsResponse.from_dict(response_dict)
+
+    # Query plan operations
+
+    def explain_table_query_plan(self, request: ExplainTableQueryPlanRequest) -> str:
+        """Explain the query plan for a table query."""
+        return self._inner.explain_table_query_plan(request.model_dump())
+
+    def analyze_table_query_plan(self, request: AnalyzeTableQueryPlanRequest) -> str:
+        """Analyze the query plan for a table query with execution statistics."""
+        return self._inner.analyze_table_query_plan(request.model_dump())
+
+    # Column alteration operations
+
+    def alter_table_add_columns(
+        self, request: AlterTableAddColumnsRequest
+    ) -> AlterTableAddColumnsResponse:
+        """Add columns to a table."""
+        response_dict = self._inner.alter_table_add_columns(request.model_dump())
+        return AlterTableAddColumnsResponse.from_dict(response_dict)
+
+    def alter_table_alter_columns(
+        self, request: AlterTableAlterColumnsRequest
+    ) -> AlterTableAlterColumnsResponse:
+        """Alter existing columns in a table."""
+        response_dict = self._inner.alter_table_alter_columns(request.model_dump())
+        return AlterTableAlterColumnsResponse.from_dict(response_dict)
+
+    def alter_table_drop_columns(
+        self, request: AlterTableDropColumnsRequest
+    ) -> AlterTableDropColumnsResponse:
+        """Drop columns from a table."""
+        response_dict = self._inner.alter_table_drop_columns(request.model_dump())
+        return AlterTableDropColumnsResponse.from_dict(response_dict)
+
+    # Table tag operations
+
+    def list_table_tags(self, request: ListTableTagsRequest) -> ListTableTagsResponse:
+        """List all tags on a table."""
+        response_dict = self._inner.list_table_tags(request.model_dump())
+        return ListTableTagsResponse.from_dict(response_dict)
+
+    def get_table_tag_version(
+        self, request: GetTableTagVersionRequest
+    ) -> GetTableTagVersionResponse:
+        """Get the version associated with a table tag."""
+        response_dict = self._inner.get_table_tag_version(request.model_dump())
+        return GetTableTagVersionResponse.from_dict(response_dict)
+
+    def create_table_tag(
+        self, request: CreateTableTagRequest
+    ) -> CreateTableTagResponse:
+        """Create a new tag pointing to a table version."""
+        response_dict = self._inner.create_table_tag(request.model_dump())
+        return CreateTableTagResponse.from_dict(response_dict)
+
+    def delete_table_tag(
+        self, request: DeleteTableTagRequest
+    ) -> DeleteTableTagResponse:
+        """Delete a tag from a table."""
+        response_dict = self._inner.delete_table_tag(request.model_dump())
+        return DeleteTableTagResponse.from_dict(response_dict)
+
+    def update_table_tag(
+        self, request: UpdateTableTagRequest
+    ) -> UpdateTableTagResponse:
+        """Update a tag to point to a different version."""
+        response_dict = self._inner.update_table_tag(request.model_dump())
+        return UpdateTableTagResponse.from_dict(response_dict)
+
+    # Operation metrics methods
+
+    def retrieve_ops_metrics(self) -> Dict[str, int]:
+        """Retrieve operation metrics as a dictionary.
+
+        Returns a dict where keys are operation names (e.g., "list_tables",
+        "describe_table") and values are the number of times each operation
+        was called.
+
+        Returns an empty dict if `ops_metrics_enabled` was false when creating
+        the namespace.
+
+        Returns
+        -------
+        Dict[str, int]
+            Operation name to call count mapping
+        """
+        return self._inner.retrieve_ops_metrics()
+
+    def reset_ops_metrics(self) -> None:
+        """Reset all operation metrics counters to zero.
+
+        Does nothing if `ops_metrics_enabled` was false when creating the
+        namespace.
+        """
+        self._inner.reset_ops_metrics()
+
 
 class RestNamespace(LanceNamespace):
     """REST-based Lance Namespace implementation backed by Rust.
@@ -658,6 +1057,348 @@ def batch_delete_table_versions(self, request: dict) -> dict:
         """
         return self._inner.batch_delete_table_versions(request)
 
+    # Data manipulation operations
+
+    def count_table_rows(self, request: CountTableRowsRequest) -> int:
+        """Count the number of rows in a table, optionally filtered by a predicate.
+
+        Parameters
+        ----------
+        request : CountTableRowsRequest
+            Request with table id, optional version, and optional predicate filter
+
+        Returns
+        -------
+        int
+            The number of rows matching the criteria
+        """
+        return self._inner.count_table_rows(request.model_dump())
+
+    def insert_into_table(
+        self, request: InsertIntoTableRequest, request_data: bytes
+    ) -> InsertIntoTableResponse:
+        """Insert data into a table.
+
+        Parameters
+        ----------
+        request : InsertIntoTableRequest
+            Request with table id and write mode (append/overwrite)
+        request_data : bytes
+            Arrow IPC stream containing the data to insert
+
+        Returns
+        -------
+        InsertIntoTableResponse
+            Response with optional transaction_id
+        """
+        response_dict = self._inner.insert_into_table(
+            request.model_dump(), request_data
+        )
+        return InsertIntoTableResponse.from_dict(response_dict)
+
+    def merge_insert_into_table(
+        self, request: MergeInsertIntoTableRequest, request_data: bytes
+    ) -> MergeInsertIntoTableResponse:
+        """Merge-insert data into a table (upsert operation).
+
+        Parameters
+        ----------
+        request : MergeInsertIntoTableRequest
+            Request with table id, merge keys, and merge behavior
+        request_data : bytes
+            Arrow IPC stream containing the data to merge
+
+        Returns
+        -------
+        MergeInsertIntoTableResponse
+            Response with optional transaction_id
+        """
+        response_dict = self._inner.merge_insert_into_table(
+            request.model_dump(), request_data
+        )
+        return MergeInsertIntoTableResponse.from_dict(response_dict)
+
+    def update_table(self, request: UpdateTableRequest) -> UpdateTableResponse:
+        """Update rows in a table matching a filter.
+
+        Parameters
+        ----------
+        request : UpdateTableRequest
+            Request with table id, filter predicate, and column updates
+
+        Returns
+        -------
+        UpdateTableResponse
+            Response with optional transaction_id
+        """
+        response_dict = self._inner.update_table(request.model_dump())
+        return UpdateTableResponse.from_dict(response_dict)
+
+    def delete_from_table(
+        self, request: DeleteFromTableRequest
+    ) -> DeleteFromTableResponse:
+        """Delete rows from a table matching a filter.
+
+        Parameters
+        ----------
+        request : DeleteFromTableRequest
+            Request with table id and filter predicate
+
+        Returns
+        -------
+        DeleteFromTableResponse
+            Response confirming the deletion
+        """
+        response_dict = self._inner.delete_from_table(request.model_dump())
+        return DeleteFromTableResponse.from_dict(response_dict)
+
+    def query_table(self, request) -> bytes:
+        """Query a table and return results as Arrow IPC.
+
+        Parameters
+        ----------
+        request : QueryTableRequest or dict
+            Query request with table id, optional filter, columns, vector search
+            parameters, limit/offset, etc.
+
+        Returns
+        -------
+        bytes
+            Arrow IPC file format containing the query results
+        """
+        if hasattr(request, "model_dump"):
+            request = request.model_dump()
+        return self._inner.query_table(request)
+
+    # Index operations
+
+    def create_table_index(
+        self, request: CreateTableIndexRequest
+    ) -> CreateTableIndexResponse:
+        """Create an index on a table.
+
+        Parameters
+        ----------
+        request : CreateTableIndexRequest
+            Request with table id, index name, column, and index configuration
+
+        Returns
+        -------
+        CreateTableIndexResponse
+            Response with optional transaction_id
+        """
+        response_dict = self._inner.create_table_index(request.model_dump())
+        return CreateTableIndexResponse.from_dict(response_dict)
+
+    def list_table_indices(
+        self, request: ListTableIndicesRequest
+    ) -> ListTableIndicesResponse:
+        """List all indices on a table.
+
+        Parameters
+        ----------
+        request : ListTableIndicesRequest
+            Request with table id
+
+        Returns
+        -------
+        ListTableIndicesResponse
+            Response with list of index metadata
+        """
+        response_dict = self._inner.list_table_indices(request.model_dump())
+        return ListTableIndicesResponse.from_dict(response_dict)
+
+    def describe_table_index_stats(
+        self, request: DescribeTableIndexStatsRequest
+    ) -> DescribeTableIndexStatsResponse:
+        """Get statistics for a specific index.
+
+        Parameters
+        ----------
+        request : DescribeTableIndexStatsRequest
+            Request with table id and index name
+
+        Returns
+        -------
+        DescribeTableIndexStatsResponse
+            Response with index statistics
+        """
+        response_dict = self._inner.describe_table_index_stats(request.model_dump())
+        return DescribeTableIndexStatsResponse.from_dict(response_dict)
+
+    # Transaction operations
+
+    def describe_transaction(
+        self, request: DescribeTransactionRequest
+    ) -> DescribeTransactionResponse:
+        """Describe a transaction.
+
+        Parameters
+        ----------
+        request : DescribeTransactionRequest
+            Request with transaction id
+
+        Returns
+        -------
+        DescribeTransactionResponse
+            Response with transaction details
+        """
+        response_dict = self._inner.describe_transaction(request.model_dump())
+        return DescribeTransactionResponse.from_dict(response_dict)
+
+    def alter_transaction(
+        self, request: AlterTransactionRequest
+    ) -> AlterTransactionResponse:
+        """Alter a transaction (commit or abort).
+
+        Parameters
+        ----------
+        request : AlterTransactionRequest
+            Request with transaction id and action
+
+        Returns
+        -------
+        AlterTransactionResponse
+            Response confirming the action
+        """
+        response_dict = self._inner.alter_transaction(request.model_dump())
+        return AlterTransactionResponse.from_dict(response_dict)
+
+    # Additional index operations
+
+    def create_table_scalar_index(
+        self, request: CreateTableIndexRequest
+    ) -> CreateTableIndexResponse:
+        """Create a scalar index on a table column."""
+        response_dict = self._inner.create_table_scalar_index(request.model_dump())
+        return CreateTableIndexResponse.from_dict(response_dict)
+
+    def drop_table_index(
+        self, request: DropTableIndexRequest
+    ) -> DropTableIndexResponse:
+        """Drop an index from a table."""
+        response_dict = self._inner.drop_table_index(request.model_dump())
+        return DropTableIndexResponse.from_dict(response_dict)
+
+    # Additional table operations
+
+    def list_all_tables(self, request: ListTablesRequest) -> ListTablesResponse:
+        """List all tables recursively across all namespaces."""
+        response_dict = self._inner.list_all_tables(request.model_dump())
+        return ListTablesResponse.from_dict(response_dict)
+
+    def restore_table(self, request: RestoreTableRequest) -> RestoreTableResponse:
+        """Restore a previously dropped table."""
+        response_dict = self._inner.restore_table(request.model_dump())
+        return RestoreTableResponse.from_dict(response_dict)
+
+    def update_table_schema_metadata(
+        self, request: UpdateTableSchemaMetadataRequest
+    ) -> UpdateTableSchemaMetadataResponse:
+        """Update the schema metadata of a table."""
+        response_dict = self._inner.update_table_schema_metadata(request.model_dump())
+        return UpdateTableSchemaMetadataResponse.from_dict(response_dict)
+
+    def get_table_stats(self, request: GetTableStatsRequest) -> GetTableStatsResponse:
+        """Get statistics for a table."""
+        response_dict = self._inner.get_table_stats(request.model_dump())
+        return GetTableStatsResponse.from_dict(response_dict)
+
+    # Query plan operations
+
+    def explain_table_query_plan(self, request: ExplainTableQueryPlanRequest) -> str:
+        """Explain the query plan for a table query."""
+        return self._inner.explain_table_query_plan(request.model_dump())
+
+    def analyze_table_query_plan(self, request: AnalyzeTableQueryPlanRequest) -> str:
+        """Analyze the query plan for a table query with execution statistics."""
+        return self._inner.analyze_table_query_plan(request.model_dump())
+
+    # Column alteration operations
+
+    def alter_table_add_columns(
+        self, request: AlterTableAddColumnsRequest
+    ) -> AlterTableAddColumnsResponse:
+        """Add columns to a table."""
+        response_dict = self._inner.alter_table_add_columns(request.model_dump())
+        return AlterTableAddColumnsResponse.from_dict(response_dict)
+
+    def alter_table_alter_columns(
+        self, request: AlterTableAlterColumnsRequest
+    ) -> AlterTableAlterColumnsResponse:
+        """Alter existing columns in a table."""
+        response_dict = self._inner.alter_table_alter_columns(request.model_dump())
+        return AlterTableAlterColumnsResponse.from_dict(response_dict)
+
+    def alter_table_drop_columns(
+        self, request: AlterTableDropColumnsRequest
+    ) -> AlterTableDropColumnsResponse:
+        """Drop columns from a table."""
+        response_dict = self._inner.alter_table_drop_columns(request.model_dump())
+        return AlterTableDropColumnsResponse.from_dict(response_dict)
+
+    # Table tag operations
+
+    def list_table_tags(self, request: ListTableTagsRequest) -> ListTableTagsResponse:
+        """List all tags on a table."""
+        response_dict = self._inner.list_table_tags(request.model_dump())
+        return ListTableTagsResponse.from_dict(response_dict)
+
+    def get_table_tag_version(
+        self, request: GetTableTagVersionRequest
+    ) -> GetTableTagVersionResponse:
+        """Get the version associated with a table tag."""
+        response_dict = self._inner.get_table_tag_version(request.model_dump())
+        return GetTableTagVersionResponse.from_dict(response_dict)
+
+    def create_table_tag(
+        self, request: CreateTableTagRequest
+    ) -> CreateTableTagResponse:
+        """Create a new tag pointing to a table version."""
+        response_dict = self._inner.create_table_tag(request.model_dump())
+        return CreateTableTagResponse.from_dict(response_dict)
+
+    def delete_table_tag(
+        self, request: DeleteTableTagRequest
+    ) -> DeleteTableTagResponse:
+        """Delete a tag from a table."""
+        response_dict = self._inner.delete_table_tag(request.model_dump())
+        return DeleteTableTagResponse.from_dict(response_dict)
+
+    def update_table_tag(
+        self, request: UpdateTableTagRequest
+    ) -> UpdateTableTagResponse:
+        """Update a tag to point to a different version."""
+        response_dict = self._inner.update_table_tag(request.model_dump())
+        return UpdateTableTagResponse.from_dict(response_dict)
+
+    # Operation metrics methods
+
+    def retrieve_ops_metrics(self) -> Dict[str, int]:
+        """Retrieve operation metrics as a dictionary.
+
+        Returns a dict where keys are operation names (e.g., "list_tables",
+        "describe_table") and values are the number of times each operation
+        was called.
+
+        Returns an empty dict if `ops_metrics_enabled` was false when creating
+        the namespace.
+
+        Returns
+        -------
+        Dict[str, int]
+            Operation name to call count mapping
+        """
+        return self._inner.retrieve_ops_metrics()
+
+    def reset_ops_metrics(self) -> None:
+        """Reset all operation metrics counters to zero.
+
+        Does nothing if `ops_metrics_enabled` was false when creating the
+        namespace.
+        """
+        self._inner.reset_ops_metrics()
+
 
 class RestAdapter:
     """REST adapter server that creates a namespace backend and exposes it via REST.
@@ -669,11 +1410,12 @@ class RestAdapter:
 
     Parameters
     ----------
-    namespace_impl : str
-        Namespace implementation type ("dir", "rest", etc.)
-    namespace_properties : dict, optional
-        Configuration properties for the backend namespace.
+    namespace_client_impl : str
+        Namespace client implementation type ("dir", "rest", etc.)
+    namespace_client_properties : dict, optional
+        Configuration properties for the backend namespace client.
         For DirectoryNamespace ("dir"):
+
         - root (required): Root directory path or URI
         - manifest_enabled (optional): Enable manifest tracking (default: "true")
         - dir_listing_enabled (optional): Enable directory listing fallback
@@ -701,8 +1443,8 @@ class RestAdapter:
 
     def __init__(
         self,
-        namespace_impl: str,
-        namespace_properties: Dict[str, str] = None,
+        namespace_client_impl: str,
+        namespace_client_properties: Dict[str, str] = None,
         session=None,
         host: str = None,
         port: int = None,
@@ -715,14 +1457,18 @@ def __init__(
             )
 
         # Convert to string properties
-        if namespace_properties is None:
-            namespace_properties = {}
-        str_properties = {str(k): str(v) for k, v in namespace_properties.items()}
+        if namespace_client_properties is None:
+            namespace_client_properties = {}
+        str_properties = {
+            str(k): str(v) for k, v in namespace_client_properties.items()
+        }
 
         # Create the underlying Rust adapter
-        self._inner = PyRestAdapter(namespace_impl, str_properties, session, host, port)
+        self._inner = PyRestAdapter(
+            namespace_client_impl, str_properties, session, host, port
+        )
         self.host = host
-        self.namespace_impl = namespace_impl
+        self.namespace_client_impl = namespace_client_impl
 
     @property
     def port(self) -> int:
@@ -752,117 +1498,3 @@ def __exit__(self, exc_type, exc_value, traceback):
 
     def __repr__(self) -> str:
         return f"RestAdapter(host='{self.host}', port={self.port})"
-
-
-class LanceNamespaceStorageOptionsProvider(StorageOptionsProvider):
-    """Storage options provider that fetches storage options from a LanceNamespace.
-
-    This provider automatically fetches fresh storage options by calling the
-    namespace's describe_table() method, which returns both the table location
-    and time-limited storage options. This is currently only used for refreshing
-    AWS temporary access credentials.
-
-    This is the recommended approach for LanceDB Cloud and other namespace-based
-    deployments, as it handles storage options refresh automatically.
-
-    Parameters
-    ----------
-    namespace : LanceNamespace
-        The namespace instance to fetch storage options from. Use
-        lance.namespace.connect() to create a namespace instance.
-    table_id : List[str]
-        The table identifier (e.g., ["workspace", "table_name"])
-
-    Example
-    -------
-    This example shows how to use the storage options provider with a namespace.
-
-    .. code-block:: python
-
-        import lance
-        import lance.namespace
-
-        # Connect to a namespace
-        namespace = lance.namespace.connect("rest", {"uri": "http://localhost:4099"})
-
-        # Create storage options provider
-        provider = lance.LanceNamespaceStorageOptionsProvider(
-            namespace=namespace,
-            table_id=["workspace", "table_name"]
-        )
-
-        # Use with dataset - storage options auto-refresh!
-        dataset = lance.dataset(
-            "s3://bucket/table.lance",
-            storage_options_provider=provider
-        )
-    """
-
-    def __init__(self, namespace: LanceNamespace, table_id: List[str]):
-        """Initialize with namespace and table ID.
-
-        Parameters
-        ----------
-        namespace : LanceNamespace
-            The namespace instance with a describe_table() method
-        table_id : List[str]
-            The table identifier
-        """
-        self._namespace = namespace
-        self._table_id = table_id
-
-    def fetch_storage_options(self) -> Dict[str, str]:
-        """Fetch storage options from the namespace.
-
-        This calls namespace.describe_table() to get the latest storage options
-        and optionally their expiration time.
-
-        Returns
-        -------
-        Dict[str, str]
-            Flat dictionary of string key-value pairs containing storage options.
-            May optionally include expires_at_millis. If expires_at_millis is not
-            provided, credentials are treated as non-expiring and will not be
-            automatically refreshed.
-
-        Raises
-        ------
-        RuntimeError
-            If the namespace doesn't return storage options
-        """
-        request = DescribeTableRequest(id=self._table_id, version=None)
-        response = self._namespace.describe_table(request)
-        storage_options = response.storage_options
-        if storage_options is None:
-            raise RuntimeError(
-                "Namespace did not return storage_options. "
-                "Ensure the namespace supports storage options providing."
-            )
-
-        # Return the storage_options directly - it's already a flat Map<String, String>
-        # Note: expires_at_millis is optional. If not provided, credentials are treated
-        # as non-expiring and will not be automatically refreshed.
-        return storage_options
-
-    def provider_id(self) -> str:
-        """Return a human-readable unique identifier for this provider instance.
-
-        This creates a semantic ID based on the namespace's ID and the table ID,
-        enabling proper equality comparison and caching.
-
-        Returns
-        -------
-        str
-            A human-readable unique identifier string combining namespace and table info
-        """
-        # Try to call namespace_id() if available (lance-namespace >= 0.0.20)
-        if hasattr(self._namespace, "namespace_id"):
-            namespace_id = self._namespace.namespace_id()
-        else:
-            # Fallback for older namespace versions
-            namespace_id = str(self._namespace)
-
-        return (
-            f"LanceNamespaceStorageOptionsProvider {{ "
-            f"namespace: {namespace_id}, table_id: {self._table_id!r} }}"
-        )
diff --git a/python/python/lance/optimize.py b/python/python/lance/optimize.py
index 5e4da3b1b0a..8b98308d442 100644
--- a/python/python/lance/optimize.py
+++ b/python/python/lance/optimize.py
@@ -81,3 +81,11 @@ class CompactionOptions(TypedDict):
     """
     Whether to defer index remapping during compaction (default: False).
     """
+    max_source_fragments: Optional[int]
+    """
+    Maximum number of source fragments to compact in a single run. Tasks
+    are included until adding the next task would exceed this limit,
+    allowing for incremental compaction (e.g., compact 20 fragments at a
+    time). Fragments are processed oldest first.
+    (default: None, no limit)
+    """
diff --git a/python/python/lance/progress.py b/python/python/lance/progress.py
index 0c428638648..e5c1d934d0b 100644
--- a/python/python/lance/progress.py
+++ b/python/python/lance/progress.py
@@ -8,13 +8,49 @@
 import json
 import os
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Dict, Optional
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Dict, Literal, Optional
 
 if TYPE_CHECKING:
     # We don't import directly because of circular import
     from .fragment import FragmentMetadata
 
 
+@dataclass(frozen=True)
+class IndexProgress:
+    """Progress event emitted while creating or merging an index.
+
+    Parameters
+    ----------
+    event : str
+        The type of event. One of ``"start"``, ``"progress"``, or ``"complete"``.
+    stage : str
+        The stage name. Stage names are index-type-specific.
+    completed : int, optional
+        The amount of work completed so far, if known.
+    total : int, optional
+        The total amount of work for the stage, if known.
+    unit : str
+        The unit of work for ``completed`` / ``total``.
+    """
+
+    event: Literal["start", "progress", "complete"]
+    stage: str
+    completed: Optional[int] = None
+    total: Optional[int] = None
+    unit: str = ""
+
+    @property
+    def fraction(self) -> Optional[float]:
+        """Return fractional progress if both ``completed`` and ``total`` are known.
+
+        Returns ``None`` when ``total`` is ``None`` or ``0`` (treated as unknown).
+        """
+        if self.completed is None or self.total in (None, 0):
+            return None
+        return min(self.completed / self.total, 1.0)
+
+
 class FragmentWriteProgress(ABC):
     """Progress tracking for Writing a Dataset or Fragment.
 
diff --git a/python/python/lance/tf/data.py b/python/python/lance/tf/data.py
index 7b6ff8e51a4..68280f9211f 100644
--- a/python/python/lance/tf/data.py
+++ b/python/python/lance/tf/data.py
@@ -141,7 +141,7 @@ def from_lance(
     filter: Optional[str] = None,
     fragments: Union[Iterable[int], Iterable[LanceFragment], tf.data.Dataset] = None,
     output_signature: Optional[Dict[str, tf.TypeSpec]] = None,
-    namespace: Optional["LanceNamespace"] = None,
+    namespace_client: Optional["LanceNamespace"] = None,
     table_id: Optional[List[str]] = None,
     ignore_namespace_table_storage_options: bool = False,
 ) -> tf.data.Dataset:
@@ -151,7 +151,7 @@ def from_lance(
     ----------
     dataset : Union[str, Path, LanceDataset], optional
         Lance dataset or dataset URI/path. Either ``dataset`` or both
-        ``namespace`` and ``table_id`` must be provided.
+        ``namespace_client`` and ``table_id`` must be provided.
     columns : Optional[List[str]], optional
         List of columns to include in the output dataset.
         If not set, all columns will be read.
@@ -165,13 +165,15 @@ def from_lance(
     output_signature : Optional[tf.TypeSpec], optional
         Override output signature of the returned tensors. If not provided,
         the output signature is inferred from the projection Schema.
-    namespace : Optional[LanceNamespace], optional
-        Namespace to resolve the table location when ``table_id`` is provided.
+    namespace_client : Optional[LanceNamespace], optional
+        Namespace client to resolve the table location when ``table_id`` is
+        provided.
     table_id : Optional[List[str]], optional
-        Table identifier used together with ``namespace`` to locate the table.
+        Table identifier used together with ``namespace_client`` to locate
+        the table.
     ignore_namespace_table_storage_options : bool, default False
-        When using ``namespace``/``table_id``, ignore storage options returned
-        by the namespace.
+        When using ``namespace_client``/``table_id``, ignore storage options
+        returned by the namespace.
 
     Examples
     --------
@@ -212,15 +214,15 @@ def from_lance(
 
     """
     if isinstance(dataset, LanceDataset):
-        if namespace is not None or table_id is not None:
+        if namespace_client is not None or table_id is not None:
             raise ValueError(
-                "Cannot specify 'namespace' or 'table_id' when passing "
+                "Cannot specify 'namespace_client' or 'table_id' when passing "
                 "a LanceDataset instance"
             )
     else:
         dataset = lance.dataset(
             dataset,
-            namespace=namespace,
+            namespace_client=namespace_client,
             table_id=table_id,
             ignore_namespace_table_storage_options=ignore_namespace_table_storage_options,
         )
diff --git a/python/python/lance/torch/data.py b/python/python/lance/torch/data.py
index d5adcbbfe19..759b5b2f50a 100644
--- a/python/python/lance/torch/data.py
+++ b/python/python/lance/torch/data.py
@@ -52,6 +52,52 @@ def __call__(
     ) -> Union[dict[str, torch.Tensor], torch.Tensor]: ...
 
 
+def _is_bfloat16_type(t: pa.DataType) -> bool:
+    """Check if a PyArrow type is the lance bfloat16 extension type."""
+    return isinstance(t, pa.ExtensionType) and t.extension_name == "lance.bfloat16"
+
+
+def _bf16_to_tensor(arr: pa.Array) -> torch.Tensor:
+    """Convert a bfloat16 extension array to a torch.bfloat16 tensor.
+
+    Reinterprets the raw bytes as uint16 and views as bfloat16,
+    since they share the same 2-byte memory layout.
+    Null values are replaced with NaN.
+    """
+    storage = arr.storage if isinstance(arr.type, pa.ExtensionType) else arr
+    buf = storage.buffers()[1]
+    offset = storage.offset * 2  # 2 bytes per bf16 value
+    try:
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore",
+                message="The given buffer is not writable",
+                category=UserWarning,
+            )
+            tensor = torch.frombuffer(
+                memoryview(buf),
+                dtype=torch.uint16,
+                count=len(storage),
+                offset=offset,
+            ).view(torch.bfloat16)
+    except (AttributeError, RuntimeError, TypeError):
+        np_uint16 = np.frombuffer(
+            buf, dtype=np.uint16, count=len(storage), offset=offset
+        )
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore",
+                message="The given NumPy array is not writable",
+                category=UserWarning,
+            )
+            tensor = torch.from_numpy(np_uint16).view(torch.bfloat16)
+    if arr.null_count > 0:
+        tensor = tensor.clone()
+        null_mask = torch.from_numpy(arr.is_null().to_numpy(zero_copy_only=False))
+        tensor[null_mask] = float("nan")
+    return tensor
+
+
 # Convert an Arrow FSL array into a 2D torch tensor
 def _fsl_to_tensor(arr: pa.FixedSizeListArray, dimension: int) -> torch.Tensor:
     # Note: FixedSizeListArray.values does not take offset/len into account and
@@ -104,6 +150,14 @@ def _to_tensor(
             or pa.types.is_integer(arr.type.value_type)
         ):
             tensor = _fsl_to_tensor(arr, arr.type.list_size)
+        elif pa.types.is_fixed_size_list(arr.type) and _is_bfloat16_type(
+            arr.type.value_type
+        ):
+            values = arr.values
+            start = arr.offset * arr.type.list_size
+            num_vals = len(arr) * arr.type.list_size
+            values = values.slice(start, num_vals)
+            tensor = _bf16_to_tensor(values).view(-1, arr.type.list_size)
         elif (
             pa.types.is_integer(arr.type)
             or pa.types.is_floating(arr.type)
@@ -113,13 +167,15 @@ def _to_tensor(
 
             if uint64_as_int64 and tensor.dtype == torch.uint64:
                 tensor = tensor.to(torch.int64)
+        elif _is_bfloat16_type(arr.type):
+            tensor = _bf16_to_tensor(arr)
         elif hf_converter is not None:
             tensor = hf_converter.to_pytorch(col, arr)
 
         if tensor is None:
             raise ValueError(
-                "Only support FixedSizeList<f16/f32/f64> or "
-                + f"numeric values, got: {arr.type}"
+                "Only support FixedSizeList<f16/bf16/f32/f64> or "
+                + f"numeric/bfloat16 values, got: {arr.type}"
             )
 
         del arr
@@ -443,10 +499,7 @@ def __getitems__(self, indices):
         """
         if self._ds is None:
             # Worker-process initialization
-            import os
-
-            self._ds = lance.dataset(self.uri)
-            print(f"Worker {os.getpid()} initialized dataset")
+            self._ds = lance.dataset(self.uri, **self.dataset_options)
 
         # Leverage native batch reading
         batch = self._ds.take(indices)
diff --git a/python/python/lance/vector.py b/python/python/lance/vector.py
index 73f7981fe28..34a6154a321 100644
--- a/python/python/lance/vector.py
+++ b/python/python/lance/vector.py
@@ -270,10 +270,6 @@ def train_ivf_centroids_on_accelerator(
         kmeans.centroids.cpu().numpy().astype(vector_value_type.to_pandas_dtype())
     )
 
-    with tempfile.NamedTemporaryFile(delete=False) as f:
-        np.save(f, centroids)
-    LOGGER.info("Saved centroids to %s", f.name)
-
     return centroids, kmeans
 
 
diff --git a/python/python/tests/compat/compat_decorator.py b/python/python/tests/compat/compat_decorator.py
index cb9aa10c573..0ab35672410 100644
--- a/python/python/tests/compat/compat_decorator.py
+++ b/python/python/tests/compat/compat_decorator.py
@@ -10,11 +10,13 @@
 
 import inspect
 import json
+import os
 import subprocess
 import sys
 import urllib.request
+from contextlib import contextmanager
 from functools import lru_cache
-from typing import List
+from typing import Dict, List, Optional
 
 import pytest
 from packaging.version import Version
@@ -147,6 +149,42 @@ def check_read(self):
     def check_write(self):
         pass
 
+    def skip_read_after_current_write(self, version: str) -> bool:
+        """Return True to skip the old-version read after current-version writes."""
+        return False
+
+    def skip_downgrade(self, version: str) -> bool:
+        """Return True to skip the current-write -> old-read downgrade test."""
+        return False
+
+    def current_env(self, method_name: str) -> Dict[str, str]:
+        """Return environment overrides for methods executed in the current runtime."""
+        return {}
+
+    def compat_env(self, version: str, method_name: str) -> Dict[str, str]:
+        """Return environment overrides for methods executed in a compat venv."""
+        return {}
+
+
+@contextmanager
+def _temporary_env(overrides: Optional[Dict[str, str]]):
+    if not overrides:
+        yield
+        return
+
+    sentinel = object()
+    old_values = {key: os.environ.get(key, sentinel) for key in overrides}
+    try:
+        for key, value in overrides.items():
+            os.environ[key] = value
+        yield
+    finally:
+        for key, value in old_values.items():
+            if value is sentinel:
+                os.environ.pop(key, None)
+            else:
+                os.environ[key] = value
+
 
 def compat_test(min_version: str = "0.16.0"):
     """Decorator to generate upgrade/downgrade compatibility tests.
@@ -285,12 +323,18 @@ def test_func({sig_params}):
     """Test that old Lance version can read data written by current version."""
     from pathlib import Path
     obj = cls(tmp_path / "data.lance", {init_params})
+    obj.compat_version = version
+    if obj.skip_downgrade(version):
+        pytest.skip(
+            "downgrade compatibility is intentionally unsupported for this test"
+        )
     # Current version: create data
-    obj.create()
+    with _temporary_env(obj.current_env("create")):
+        obj.create()
     # Old version: verify can read
     venv = venv_factory.get_venv(version)
-    venv.execute_method(obj, "check_read")
-    venv.execute_method(obj, "check_write")
+    venv.execute_method(obj, "check_read", obj.compat_env(version, "check_read"))
+    venv.execute_method(obj, "check_write", obj.compat_env(version, "check_write"))
 '''
     else:  # upgrade_downgrade
         func_body = f'''
@@ -298,18 +342,21 @@ def test_func({sig_params}):
     """Test round-trip compatibility: old -> current -> old."""
     from pathlib import Path
     obj = cls(tmp_path / "data.lance", {init_params})
+    obj.compat_version = version
     venv = venv_factory.get_venv(version)
     # Old version: create data
-    venv.execute_method(obj, "create")
+    venv.execute_method(obj, "create", obj.compat_env(version, "create"))
     # Current version: read and write
-    obj.check_read()
-    obj.check_write()
+    with _temporary_env(obj.current_env("check_read")):
+        obj.check_read()
+    with _temporary_env(obj.current_env("check_write")):
+        obj.check_write()
     # Old version: verify can still read
-    venv.execute_method(obj, "check_read")
-    venv.execute_method(obj, "check_write")
+    venv.execute_method(obj, "check_read", obj.compat_env(version, "check_read"))
+    venv.execute_method(obj, "check_write", obj.compat_env(version, "check_write"))
 '''
 
     # Execute to create the function
-    namespace = {"cls": cls}
+    namespace = {"cls": cls, "_temporary_env": _temporary_env, "pytest": pytest}
     exec(func_body, namespace)
     return namespace["test_func"]
diff --git a/python/python/tests/compat/test_scalar_indices.py b/python/python/tests/compat/test_scalar_indices.py
index 5d42a837bdc..35022df3b12 100644
--- a/python/python/tests/compat/test_scalar_indices.py
+++ b/python/python/tests/compat/test_scalar_indices.py
@@ -19,6 +19,7 @@
     UpgradeDowngradeTest,
     compat_test,
 )
+from .util import safe_data_storage_version
 
 
 @compat_test(min_version="0.30.0")
@@ -40,7 +41,12 @@ def create(self):
                 "btree": pa.array(range(1000)),
             }
         )
-        dataset = lance.write_dataset(data, self.path, max_rows_per_file=100)
+        dataset = lance.write_dataset(
+            data,
+            self.path,
+            max_rows_per_file=100,
+            data_storage_version=safe_data_storage_version(self.compat_version),
+        )
         dataset.create_scalar_index("btree", "BTREE")
 
     def check_read(self):
@@ -92,7 +98,12 @@ def create(self):
                 "label_list": pa.array([[f"label{i}"] for i in range(1000)]),
             }
         )
-        dataset = lance.write_dataset(data, self.path, max_rows_per_file=100)
+        dataset = lance.write_dataset(
+            data,
+            self.path,
+            max_rows_per_file=100,
+            data_storage_version=safe_data_storage_version(self.compat_version),
+        )
         dataset.create_scalar_index("bitmap", "BITMAP")
         dataset.create_scalar_index("label_list", "LABEL_LIST")
 
@@ -141,7 +152,12 @@ def create(self):
                 "ngram": pa.array([f"word{i}" for i in range(1000)]),
             }
         )
-        dataset = lance.write_dataset(data, self.path, max_rows_per_file=100)
+        dataset = lance.write_dataset(
+            data,
+            self.path,
+            max_rows_per_file=100,
+            data_storage_version=safe_data_storage_version(self.compat_version),
+        )
         dataset.create_scalar_index("ngram", "NGRAM")
 
     def check_read(self):
@@ -186,7 +202,12 @@ def create(self):
                 "bloomfilter": pa.array(range(1000)),
             }
         )
-        dataset = lance.write_dataset(data, self.path, max_rows_per_file=100)
+        dataset = lance.write_dataset(
+            data,
+            self.path,
+            max_rows_per_file=100,
+            data_storage_version=safe_data_storage_version(self.compat_version),
+        )
         dataset.create_scalar_index("zonemap", "ZONEMAP")
         dataset.create_scalar_index("bloomfilter", "BLOOMFILTER")
 
@@ -237,7 +258,12 @@ def create(self):
                 "json": pa.array([f'{{"val": {i}}}' for i in range(1000)], pa.json_()),
             }
         )
-        dataset = lance.write_dataset(data, self.path, max_rows_per_file=100)
+        dataset = lance.write_dataset(
+            data,
+            self.path,
+            max_rows_per_file=100,
+            data_storage_version=safe_data_storage_version(self.compat_version),
+        )
         dataset.create_scalar_index(
             "json",
             IndexConfig(
@@ -288,16 +314,22 @@ def create(self):
                 ),
             }
         )
-        dataset = lance.write_dataset(data, self.path, max_rows_per_file=100)
-        dataset.create_scalar_index("text", "INVERTED")
+        dataset = lance.write_dataset(
+            data,
+            self.path,
+            max_rows_per_file=100,
+            data_storage_version=safe_data_storage_version(self.compat_version),
+        )
+        dataset.create_scalar_index("text", "INVERTED", with_position=True)
 
     def check_read(self):
         """Verify FTS index can be queried."""
         ds = lance.dataset(self.path)
-        # Search for documents containing "words" and "7"
-        # Note: Actual FTS query syntax may vary
-        table = ds.to_table(filter="text LIKE '%words 7 %'")
-        assert table.num_rows > 0
+        match_table = ds.to_table(
+            full_text_query={"query": "words 7", "columns": ["text"]}
+        )
+        assert match_table.num_rows > 0
+        assert 7 in match_table.column("idx").to_pylist()
 
     def check_write(self):
         """Verify can insert data with FTS index."""
@@ -313,3 +345,13 @@ def check_write(self):
         )
         ds.insert(data)
         ds.optimize.compact_files()
+
+    def skip_downgrade(self, version: str) -> bool:
+        return version.startswith("0.")
+
+    def current_env(self, method_name: str) -> dict[str, str]:
+        if method_name == "create":
+            return {"LANCE_FTS_FORMAT_VERSION": "1"}
+        if method_name == "check_write":
+            return {"LANCE_FTS_FORMAT_VERSION": "2"}
+        return {}
diff --git a/python/python/tests/compat/test_vector_indices.py b/python/python/tests/compat/test_vector_indices.py
index 25d43c2f17b..194435c095a 100644
--- a/python/python/tests/compat/test_vector_indices.py
+++ b/python/python/tests/compat/test_vector_indices.py
@@ -20,6 +20,7 @@
     UpgradeDowngradeTest,
     compat_test,
 )
+from .util import safe_data_storage_version
 
 
 @compat_test(min_version="0.29.1.beta2")
@@ -44,7 +45,11 @@ def create(self):
             }
         )
 
-        dataset = lance.write_dataset(data, self.path)
+        dataset = lance.write_dataset(
+            data,
+            self.path,
+            data_storage_version=safe_data_storage_version(self.compat_version),
+        )
         dataset.create_index(
             "vec",
             "IVF_PQ",
@@ -109,7 +114,11 @@ def create(self):
             }
         )
 
-        dataset = lance.write_dataset(data, self.path)
+        dataset = lance.write_dataset(
+            data,
+            self.path,
+            data_storage_version=safe_data_storage_version(self.compat_version),
+        )
         dataset.create_index(
             "vec",
             "IVF_HNSW_PQ",
@@ -174,7 +183,11 @@ def create(self):
             }
         )
 
-        dataset = lance.write_dataset(data, self.path)
+        dataset = lance.write_dataset(
+            data,
+            self.path,
+            data_storage_version=safe_data_storage_version(self.compat_version),
+        )
         dataset.create_index(
             "vec",
             "IVF_HNSW_SQ",
@@ -235,7 +248,11 @@ def create(self):
             }
         )
 
-        dataset = lance.write_dataset(data, self.path)
+        dataset = lance.write_dataset(
+            data,
+            self.path,
+            data_storage_version=safe_data_storage_version(self.compat_version),
+        )
         dataset.create_index(
             "vec",
             "IVF_RQ",
diff --git a/python/python/tests/compat/util.py b/python/python/tests/compat/util.py
index f20bc93f23f..f587393be10 100644
--- a/python/python/tests/compat/util.py
+++ b/python/python/tests/compat/util.py
@@ -9,6 +9,21 @@
 import pyarrow as pa
 
 
+def safe_data_storage_version(version_str):
+    """Return a data_storage_version safe for the given lance version.
+
+    Versions 0.30 and older use "2.0", newer versions use "stable".
+    """
+    parts = version_str.split(".")
+    major = int(parts[0])
+    if major > 0:
+        return "stable"
+    minor = int(parts[1]) if len(parts) > 1 else 0
+    if minor <= 30:
+        return "2.0"
+    return "stable"
+
+
 def build_basic_types():
     schema = pa.schema(
         [
diff --git a/python/python/tests/compat/venv_manager.py b/python/python/tests/compat/venv_manager.py
index 996422aea34..a5e52bbc3fd 100644
--- a/python/python/tests/compat/venv_manager.py
+++ b/python/python/tests/compat/venv_manager.py
@@ -162,7 +162,12 @@ def _receive_message(self) -> Any:
 
         return pickle.loads(data)
 
-    def execute_method(self, obj: Any, method_name: str) -> Any:
+    def execute_method(
+        self,
+        obj: Any,
+        method_name: str,
+        env_overrides: Optional[dict[str, str]] = None,
+    ) -> Any:
         """
         Execute a method on a pickled object in the virtual environment.
 
@@ -192,8 +197,8 @@ def execute_method(self, obj: Any, method_name: str) -> Any:
         # Ensure subprocess is running
         self._ensure_subprocess()
         try:
-            # Send request: (obj, method_name)
-            self._send_message((obj, method_name))
+            # Send request: (obj, method_name, env_overrides)
+            self._send_message((obj, method_name, env_overrides or {}))
 
             # Receive response
             response = self._receive_message()
diff --git a/python/python/tests/compat/venv_runner.py b/python/python/tests/compat/venv_runner.py
index ce7b3c3de77..8ad7f06ba21 100644
--- a/python/python/tests/compat/venv_runner.py
+++ b/python/python/tests/compat/venv_runner.py
@@ -22,6 +22,7 @@
 import sys
 import time
 import traceback
+from contextlib import contextmanager
 
 # Enable detailed timing output with DEBUG=1
 DEBUG = os.environ.get("DEBUG", "").lower() in ("1", "true", "yes")
@@ -55,17 +56,41 @@ def write_message(stream, obj):
     stream.buffer.flush()
 
 
+@contextmanager
+def temporary_env(overrides):
+    if not overrides:
+        yield
+        return
+
+    sentinel = object()
+    old_values = {key: os.environ.get(key, sentinel) for key in overrides}
+    try:
+        for key, value in overrides.items():
+            os.environ[key] = value
+        yield
+    finally:
+        for key, value in old_values.items():
+            if value is sentinel:
+                os.environ.pop(key, None)
+            else:
+                os.environ[key] = value
+
+
 def main():
     """Main loop that processes method calls until EOF."""
     while True:
         try:
-            # Read request (obj, method_name)
+            # Read request (obj, method_name, env_overrides)
             request = read_message(sys.stdin)
             if request is None:
                 # EOF - parent closed connection
                 break
 
-            obj, method_name = request
+            if len(request) == 2:
+                obj, method_name = request
+                env_overrides = {}
+            else:
+                obj, method_name, env_overrides = request
 
             # Execute method with timing
             start_time = time.time()
@@ -77,7 +102,8 @@ def main():
                 )
 
             method = getattr(obj, method_name)
-            result = method()
+            with temporary_env(env_overrides):
+                result = method()
 
             if DEBUG:
                 exec_time = time.time() - start_time
diff --git a/python/python/tests/conftest.py b/python/python/tests/conftest.py
index 49a6eeaa490..3790535efc7 100644
--- a/python/python/tests/conftest.py
+++ b/python/python/tests/conftest.py
@@ -1,10 +1,46 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The Lance Authors
 import sys
+from typing import Optional
 
 import pytest
 
 
+class ProgressRecorder:
+    """Reusable progress callback recorder for index build tests."""
+
+    def __init__(
+        self,
+        fail_after: Optional[int] = None,
+        fail_on_tag: Optional[str] = None,
+    ):
+        self.events = []
+        self.fail_after = fail_after
+        self.fail_on_tag = fail_on_tag
+
+    def __call__(self, event):
+        self.events.append(event)
+        event_tag = f"{event.event}:{event.stage}"
+        if self.fail_on_tag is not None and event_tag == self.fail_on_tag:
+            raise RuntimeError("progress callback failure")
+        if self.fail_after is not None and len(self.events) >= self.fail_after:
+            raise RuntimeError("progress callback failure")
+
+
+def progress_event_tags(events):
+    return [f"{event.event}:{event.stage}" for event in events]
+
+
+def stage_progress_values(events, stage):
+    return [
+        event.completed
+        for event in events
+        if event.event == "progress"
+        and event.stage == stage
+        and event.completed is not None
+    ]
+
+
 @pytest.fixture(params=(True, False))
 def provide_pandas(request, monkeypatch):
     if not request.param:
diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py
index 782a83d814c..c2baae79ce3 100644
--- a/python/python/tests/test_blob.py
+++ b/python/python/tests/test_blob.py
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The Lance Authors
 
+import importlib
 import io
 import subprocess
 import sys
@@ -10,7 +11,35 @@
 import lance
 import pyarrow as pa
 import pytest
-from lance import Blob, BlobColumn, DatasetBasePath
+from lance import Blob, BlobColumn, BlobFile, DatasetBasePath
+
+lance_dataset_module = importlib.import_module("lance.dataset")
+
+
+def _blob_row_ids(dataset):
+    return dataset.to_table(columns=[], with_row_id=True).column("_rowid").to_pylist()
+
+
+def _blob_row_addresses(dataset):
+    return (
+        dataset.to_table(columns=["idx"], with_row_address=True)
+        .column("_rowaddr")
+        .to_pylist()
+    )
+
+
+def _out_of_order_blob_selection(dataset_with_blobs, selection_kind):
+    addresses = _blob_row_addresses(dataset_with_blobs)
+    expected = [(addresses[4], b"quux"), (addresses[0], b"foo")]
+
+    if selection_kind == "ids":
+        return [
+            _blob_row_ids(dataset_with_blobs)[4],
+            _blob_row_ids(dataset_with_blobs)[0],
+        ], expected
+    if selection_kind == "addresses":
+        return [addresses[4], addresses[0]], expected
+    return [4, 0], expected
 
 
 def test_blob_read_from_binary():
@@ -248,6 +277,134 @@ def test_blob_by_indices(tmp_path, dataset_with_blobs):
             assert f1.read() == f2.read()
 
 
+@pytest.mark.parametrize(
+    ("selection_kind", "selection_values", "expected"),
+    [
+        ("ids", [0, (1 << 32) + 1], [(0, b"foo"), ((1 << 32) + 1, b"quux")]),
+        ("addresses", [0, (1 << 32) + 1], [(0, b"foo"), ((1 << 32) + 1, b"quux")]),
+        ("indices", [0, 4], [(0, b"foo"), ((1 << 32) + 1, b"quux")]),
+    ],
+)
+def test_read_blobs(dataset_with_blobs, selection_kind, selection_values, expected):
+    kwargs = {selection_kind: selection_values}
+
+    blobs = dataset_with_blobs.read_blobs(
+        "blobs",
+        **kwargs,
+        io_buffer_size=1024,
+        preserve_order=True,
+    )
+
+    assert blobs == expected
+
+
+def test_read_blobs_requires_single_selector(dataset_with_blobs):
+    with pytest.raises(
+        ValueError, match="Exactly one of ids, indices, or addresses must be specified"
+    ):
+        dataset_with_blobs.read_blobs("blobs", ids=[0], indices=[0])
+
+
+def test_read_blobs_requires_selector(dataset_with_blobs):
+    with pytest.raises(
+        ValueError, match="Exactly one of ids, indices, or addresses must be specified"
+    ):
+        dataset_with_blobs.read_blobs("blobs")
+
+
+def test_read_blobs_rejects_non_blob_column(dataset_with_blobs):
+    with pytest.raises(ValueError, match="not a blob column"):
+        dataset_with_blobs.read_blobs("idx", indices=[0])
+
+
+@pytest.mark.parametrize(
+    ("selection_kind", "selection_values", "expected"),
+    [
+        (
+            "ids",
+            pa.array([0, (1 << 32) + 1], type=pa.uint64()),
+            [(0, b"foo"), ((1 << 32) + 1, b"quux")],
+        ),
+        (
+            "addresses",
+            pa.array([0, (1 << 32) + 1], type=pa.uint64()),
+            [(0, b"foo"), ((1 << 32) + 1, b"quux")],
+        ),
+        (
+            "indices",
+            pa.array([0, 4], type=pa.uint64()),
+            [(0, b"foo"), ((1 << 32) + 1, b"quux")],
+        ),
+    ],
+)
+def test_read_blobs_accepts_arrow_array_selectors(
+    dataset_with_blobs, selection_kind, selection_values, expected
+):
+    kwargs = {selection_kind: selection_values}
+
+    blobs = dataset_with_blobs.read_blobs("blobs", **kwargs)
+
+    assert blobs == expected
+
+
+@pytest.mark.parametrize(
+    ("selection_kind", "selection_values"),
+    [
+        ("ids", []),
+        ("addresses", []),
+        ("indices", []),
+        ("ids", pa.array([], type=pa.uint64())),
+        ("addresses", pa.array([], type=pa.uint64())),
+        ("indices", pa.array([], type=pa.uint64())),
+    ],
+)
+def test_read_blobs_accepts_empty_selection(
+    dataset_with_blobs, selection_kind, selection_values
+):
+    kwargs = {selection_kind: selection_values}
+
+    assert dataset_with_blobs.read_blobs("blobs", **kwargs) == []
+
+
+@pytest.mark.parametrize(
+    ("planner_kwargs", "error_message"),
+    [
+        ({"io_buffer_size": 0}, "io_buffer_size must be greater than 0"),
+    ],
+)
+def test_read_blobs_rejects_invalid_planner_options(
+    dataset_with_blobs, planner_kwargs, error_message
+):
+    with pytest.raises(ValueError, match=error_message):
+        dataset_with_blobs.read_blobs("blobs", indices=[0], **planner_kwargs)
+
+
+@pytest.mark.parametrize("selection_kind", ["ids", "addresses", "indices"])
+def test_read_blobs_preserves_input_order(dataset_with_blobs, selection_kind):
+    selection_values, expected = _out_of_order_blob_selection(
+        dataset_with_blobs, selection_kind
+    )
+    kwargs = {selection_kind: selection_values}
+
+    blobs = dataset_with_blobs.read_blobs("blobs", **kwargs, preserve_order=True)
+
+    assert blobs == expected
+
+
+@pytest.mark.parametrize("selection_kind", ["ids", "addresses", "indices"])
+def test_read_blobs_without_preserve_order_returns_same_rows(
+    dataset_with_blobs, selection_kind
+):
+    selection_values, expected = _out_of_order_blob_selection(
+        dataset_with_blobs, selection_kind
+    )
+    kwargs = {selection_kind: selection_values}
+
+    blobs = dataset_with_blobs.read_blobs("blobs", **kwargs, preserve_order=False)
+
+    assert sorted(blobs) == sorted(expected)
+
+
 def test_blob_file_seek(tmp_path, dataset_with_blobs):
     row_ids = (
         dataset_with_blobs.to_table(columns=[], with_row_id=True)
@@ -378,6 +535,47 @@ def test_blob_extension_write_external(tmp_path):
         assert f.read() == b"hello"
 
 
+def test_blob_extension_write_external_ingest(tmp_path):
+    blob_path = tmp_path / "external_blob.bin"
+    blob_path.write_bytes(b"hello")
+    uri = blob_path.as_uri()
+
+    table = pa.table({"blob": lance.blob_array([uri])})
+    ds = lance.write_dataset(
+        table,
+        tmp_path / "test_ds_v2_external_ingest",
+        data_storage_version="2.2",
+        external_blob_mode="ingest",
+    )
+
+    blob_path.unlink()
+
+    blob = ds.take_blobs("blob", indices=[0])[0]
+    assert blob.size() == 5
+    with blob as f:
+        assert f.read() == b"hello"
+
+
+def test_blob_extension_write_external_ingest_rejects_reference_only_options(tmp_path):
+    blob_path = tmp_path / "external_blob.bin"
+    blob_path.write_bytes(b"hello")
+    uri = blob_path.as_uri()
+    message = (
+        "allow_external_blob_outside_bases only applies when "
+        'external_blob_mode="reference"'
+    )
+
+    table = pa.table({"blob": lance.blob_array([uri])})
+    with pytest.raises(OSError, match=message):
+        lance.write_dataset(
+            table,
+            tmp_path / "test_ds_v2_external_ingest_invalid",
+            data_storage_version="2.2",
+            external_blob_mode="ingest",
+            allow_external_blob_outside_bases=True,
+        )
+
+
 def test_blob_extension_write_external_slice(tmp_path):
     tar_path = tmp_path / "container.tar"
     names = ["a.bin", "b.bin", "c.bin"]
@@ -422,6 +620,55 @@ def test_blob_extension_write_external_slice(tmp_path):
         with blob_file as f:
             assert f.read() == expected
 
+    assert ds.read_blobs("blob", indices=[0, 1, 2]) == [
+        (0, b"alpha"),
+        (1, b"bravo"),
+        (2, b"charlie"),
+    ]
+
+
+def test_blob_extension_write_external_slice_ingest(tmp_path):
+    tar_path = tmp_path / "container.tar"
+    names = ["a.bin", "b.bin", "c.bin"]
+    payloads = [b"alpha", b"bravo", b"charlie"]
+
+    with tarfile.open(tar_path, "w") as tf:
+        for name, data in zip(names, payloads):
+            info = tarfile.TarInfo(name)
+            info.size = len(data)
+            tf.addfile(info, io.BytesIO(data))
+
+    positions: list[int] = []
+    sizes: list[int] = []
+    with tarfile.open(tar_path, "r") as tf:
+        for name in names:
+            member = tf.getmember(name)
+            positions.append(member.offset_data)
+            sizes.append(member.size)
+
+    uri = tar_path.as_uri()
+    blob_values = [
+        Blob.from_uri(uri, position, size) for position, size in zip(positions, sizes)
+    ]
+    table = pa.table({"blob": lance.blob_array(blob_values)})
+
+    ds = lance.write_dataset(
+        table,
+        tmp_path / "ds_ingest",
+        data_storage_version="2.2",
+        external_blob_mode="ingest",
+    )
+
+    tar_path.unlink()
+
+    blobs = ds.take_blobs("blob", indices=[0, 1, 2])
+    assert len(blobs) == len(payloads)
+
+    for expected, blob_file in zip(payloads, blobs):
+        assert blob_file.size() == len(expected)
+        with blob_file as f:
+            assert f.read() == expected
+
 
 @pytest.mark.parametrize(
     ("payload", "is_dataset_root"),
@@ -460,3 +707,121 @@ def test_blob_extension_take_blobs_multi_base(payload, is_dataset_root, tmp_path
     assert len(blobs) == 1
     with blobs[0] as f:
         assert f.read() == payload
+
+    assert ds.read_blobs("blob", indices=[0]) == [(0, payload)]
+
+
+@pytest.fixture
+def dataset_for_pandas_blob_tests(tmp_path):
+    table = pa.table(
+        {
+            "id": pa.array([1, 2, 3], pa.int64()),
+            "blob": pa.array([b"hello", None, b"world"], pa.large_binary()),
+            "bin": pa.array([b"x", b"y", b"z"], pa.large_binary()),
+        },
+        schema=pa.schema(
+            [
+                pa.field("id", pa.int64()),
+                pa.field(
+                    "blob", pa.large_binary(), metadata={"lance-encoding:blob": "true"}
+                ),
+                pa.field("bin", pa.large_binary()),
+            ]
+        ),
+    )
+    return lance.write_dataset(table, tmp_path / "blob_pandas_ds")
+
+
+def test_dataset_to_pandas_blob_lazy(dataset_for_pandas_blob_tests):
+    df = dataset_for_pandas_blob_tests.to_pandas()
+
+    assert list(df.columns) == ["id", "blob", "bin"]
+    assert isinstance(df.iloc[0]["blob"], BlobFile)
+    assert df.iloc[1]["blob"] is None
+    assert isinstance(df.iloc[2]["blob"], BlobFile)
+    assert df["bin"].tolist() == [b"x", b"y", b"z"]
+    assert [df.iloc[0]["blob"].readall(), df.iloc[2]["blob"].readall()] == [
+        b"hello",
+        b"world",
+    ]
+
+
+def test_dataset_to_pandas_blob_bytes(dataset_for_pandas_blob_tests):
+    df = dataset_for_pandas_blob_tests.to_pandas(blob_mode="bytes")
+
+    assert list(df.columns) == ["id", "blob", "bin"]
+    assert df["blob"].tolist() == [b"hello", None, b"world"]
+    assert df["bin"].tolist() == [b"x", b"y", b"z"]
+
+
+def test_dataset_to_pandas_blob_descriptions(dataset_for_pandas_blob_tests):
+    descriptions_df = dataset_for_pandas_blob_tests.to_pandas(blob_mode="descriptions")
+    table_df = dataset_for_pandas_blob_tests.to_table().to_pandas()
+
+    assert descriptions_df.equals(table_df)
+
+
+def test_scanner_to_pandas_blob_alias(dataset_for_pandas_blob_tests):
+    df = dataset_for_pandas_blob_tests.scanner(
+        columns={"video": "blob", "id": "id"}
+    ).to_pandas()
+
+    assert list(df.columns) == ["video", "id"]
+    assert isinstance(df.iloc[0]["video"], BlobFile)
+    assert df.iloc[1]["video"] is None
+    assert df.iloc[2]["video"].readall() == b"world"
+
+
+def test_scanner_to_pandas_blob_filter_limit_order(dataset_for_pandas_blob_tests):
+    df = dataset_for_pandas_blob_tests.scanner(
+        columns=["id", "blob"],
+        filter="id > 1",
+        limit=1,
+        order_by=["id"],
+    ).to_pandas(blob_mode="bytes")
+
+    assert list(df.columns) == ["id", "blob"]
+    assert df["id"].tolist() == [2]
+    assert df["blob"].tolist() == [None]
+
+
+def test_scanner_to_pandas_blob_empty_result(dataset_for_pandas_blob_tests):
+    df = dataset_for_pandas_blob_tests.scanner(
+        columns=["id", "blob"], filter="id > 10"
+    ).to_pandas()
+
+    assert list(df.columns) == ["id", "blob"]
+    assert df.empty
+
+
+def test_fragment_to_pandas_blob(dataset_for_pandas_blob_tests):
+    fragment = dataset_for_pandas_blob_tests.get_fragments()[0]
+    df = fragment.to_pandas(columns=["id", "blob"], blob_mode="bytes")
+
+    assert list(df.columns) == ["id", "blob"]
+    assert df["blob"].tolist() == [b"hello", None, b"world"]
+
+
+def test_dataset_to_pandas_invalid_blob_mode(dataset_for_pandas_blob_tests):
+    with pytest.raises(ValueError, match="blob_mode must be one of"):
+        dataset_for_pandas_blob_tests.to_pandas(blob_mode="inline")
+
+
+def test_blob_column_sources_rejects_unmappable_transform(
+    dataset_for_pandas_blob_tests,
+):
+    projected_schema = pa.schema(
+        [
+            pa.field(
+                "video",
+                pa.large_binary(),
+                metadata={"lance-encoding:blob": "true"},
+            )
+        ]
+    )
+    snapshot = {"_columns_with_transform": (("video", "concat(blob, blob)"),)}
+
+    with pytest.raises(NotImplementedError, match="direct blob column references"):
+        lance_dataset_module._blob_column_sources(
+            projected_schema, snapshot, dataset_for_pandas_blob_tests.schema
+        )
diff --git a/python/python/tests/test_commit_index.py b/python/python/tests/test_commit_index.py
index f7471d39175..4bd1d4b1c26 100644
--- a/python/python/tests/test_commit_index.py
+++ b/python/python/tests/test_commit_index.py
@@ -105,3 +105,122 @@ def test_commit_index(dataset_with_index, test_table, tmp_path):
         )
         plan = scanner.explain_plan()
         assert "ScalarIndexQuery: query=[meta = hello]@meta_idx" in plan
+
+
+def test_commit_index_with_files(dataset_with_index, test_table, tmp_path):
+    """Test that the files field on Index round-trips through commit."""
+    from lance.dataset import Index, IndexFile
+
+    # Get info about the existing index created by the fixture
+    original_desc = dataset_with_index.describe_indices()[0]
+    index_id = original_desc.segments[0].uuid
+
+    # Verify the original index has file sizes
+    original_size = original_desc.total_size_bytes
+    assert original_size is not None and original_size > 0
+
+    # Create a new dataset without index
+    dataset_without_index = lance.write_dataset(
+        test_table, tmp_path / "dataset_without_index"
+    )
+
+    # Copy the index files from dataset_with_index to dataset_without_index
+    src_index_dir = Path(dataset_with_index.uri) / "_indices" / index_id
+    dest_index_dir = Path(dataset_without_index.uri) / "_indices" / index_id
+    shutil.copytree(src_index_dir, dest_index_dir)
+
+    # Get the field id
+    field_id = _get_field_id_by_name(dataset_without_index.lance_schema, "meta")
+
+    # Create IndexFile objects with custom sizes to verify they round-trip
+    index_files = [
+        IndexFile(path="index.idx", size_bytes=1024),
+        IndexFile(path="auxiliary.bin", size_bytes=2048),
+    ]
+
+    # Create an Index object with the files field
+    index = Index(
+        uuid=index_id,
+        name="meta_idx",
+        fields=[field_id],
+        dataset_version=dataset_without_index.version,
+        fragment_ids=set(
+            [f.fragment_id for f in dataset_without_index.get_fragments()]
+        ),
+        index_version=0,
+        files=index_files,
+    )
+
+    create_index_op = lance.LanceOperation.CreateIndex(
+        new_indices=[index],
+        removed_indices=[],
+    )
+    dataset_without_index = lance.LanceDataset.commit(
+        dataset_without_index.uri,
+        create_index_op,
+        read_version=dataset_without_index.version,
+    )
+
+    # Read back the transaction to verify the files were stored
+    transactions = dataset_without_index.get_transactions(1)
+    assert len(transactions) == 1
+    transaction = transactions[0]
+    assert transaction is not None
+    assert transaction.operation is not None
+
+    # The operation should be a CreateIndex with our index that has files
+    op = transaction.operation
+    assert len(op.new_indices) == 1
+    committed_index = op.new_indices[0]
+    assert committed_index.files is not None
+    assert len(committed_index.files) == 2
+
+    # Verify the file sizes match what we set
+    files_by_path = {f.path: f.size_bytes for f in committed_index.files}
+    assert files_by_path["index.idx"] == 1024
+    assert files_by_path["auxiliary.bin"] == 2048
+
+
+def test_commit_index_with_index_details(dataset_with_index, test_table, tmp_path):
+    """Test that index_details round-trip through Python transaction bindings."""
+    from lance.dataset import Index
+
+    original_txn = dataset_with_index.get_transactions(1)[0]
+    original_index = original_txn.operation.new_indices[0]
+    assert original_index.index_details is not None
+
+    index_id = original_index.uuid
+    dataset_without_index = lance.write_dataset(
+        test_table, tmp_path / "dataset_without_index"
+    )
+
+    src_index_dir = Path(dataset_with_index.uri) / "_indices" / index_id
+    dest_index_dir = Path(dataset_without_index.uri) / "_indices" / index_id
+    shutil.copytree(src_index_dir, dest_index_dir)
+
+    field_id = _get_field_id_by_name(dataset_without_index.lance_schema, "meta")
+    index = Index(
+        uuid=index_id,
+        name="meta_idx",
+        fields=[field_id],
+        dataset_version=dataset_without_index.version,
+        fragment_ids=set(
+            [f.fragment_id for f in dataset_without_index.get_fragments()]
+        ),
+        index_version=0,
+        index_details=original_index.index_details,
+    )
+
+    create_index_op = lance.LanceOperation.CreateIndex(
+        new_indices=[index],
+        removed_indices=[],
+    )
+    dataset_without_index = lance.LanceDataset.commit(
+        dataset_without_index.uri,
+        create_index_op,
+        read_version=dataset_without_index.version,
+    )
+
+    committed_txn = dataset_without_index.get_transactions(1)[0]
+    committed_index = committed_txn.operation.new_indices[0]
+    assert committed_index.index_details == original_index.index_details
diff --git a/python/python/tests/test_dataset.py b/python/python/tests/test_dataset.py
index 3df324a13f7..18898413561 100644
--- a/python/python/tests/test_dataset.py
+++ b/python/python/tests/test_dataset.py
@@ -329,6 +329,11 @@ def test_version_id(tmp_path: Path):
     assert updated_ds.version == 2
     assert updated_ds.latest_version == 2
 
+    historical_ds = updated_ds.checkout_version(1)
+    assert historical_ds.version == 1
+    assert historical_ds.latest_version == 2
+    assert historical_ds.checkout_version(historical_ds.latest_version).version == 2
+
 
 def test_checkout(tmp_path: Path):
     tab = pa.table({"a": range(3)})
@@ -4210,6 +4215,45 @@ def test_late_materialization_batch_size(tmp_path: Path):
         assert batch.num_rows == 32
 
 
+def test_batch_size_bytes_chopping_fallback(tmp_path: Path):
+    # Write rows with large strings (~10KiB each). The file reader's byte-size
+    # estimation is based on on-disk size which, for strings, can differ
+    # significantly from in-memory size.  The rechunk fallback should still
+    # split oversized batches so no single batch exceeds the target by too much.
+    num_rows = 200
+    row_text = "x" * 10 * 1024  # 10 KiB per row
+    table = pa.table(
+        {
+            "id": pa.array(range(num_rows), type=pa.int64()),
+            "text": pa.array([row_text] * num_rows, type=pa.large_string()),
+        }
+    )
+    dataset = lance.write_dataset(table, tmp_path, data_storage_version="stable")
+
+    # Target ~50 KiB per batch with ~10 KiB per row → expect ~5 rows per batch.
+    # Without the rechunk fallback all 200 rows would arrive in a single batch.
+    target_bytes = 50 * 1024
+    batches = list(
+        dataset.to_batches(
+            filter="id >= 0",
+            batch_size_bytes=target_bytes,
+            late_materialization=False,
+        )
+    )
+
+    total_rows = sum(b.num_rows for b in batches)
+    assert total_rows == num_rows
+
+    # We should get many batches, not one giant batch.
+    assert len(batches) > 1, f"expected many batches, got {len(batches)}"
+    # Each batch should have a small number of rows (target ~5, allow up to 10).
+    for batch in batches:
+        assert batch.num_rows <= 10, (
+            f"batch has {batch.num_rows} rows, expected at most ~10 "
+            f"for a {target_bytes} byte target with ~10KiB rows"
+        )
+
+
 def test_use_scalar_index(tmp_path: Path):
     table = pa.table({"filter": range(100)})
     dataset = lance.write_dataset(table, tmp_path)
@@ -5226,11 +5270,16 @@ def test_branches(tmp_path: Path):
     branch1.tags.create("main_latest", (None, None))
     branch1.tags.create("main_latest2", ("main", None))
     branch1.create_branch("branch_from_main", ("main", None))
+    branches_with_main = branch1.branches.list()
     assert branch1.tags.list()["branch1_latest"]["branch"] == "branch1"
     assert branch1.tags.list()["main_latest"]["branch"] is None
     assert branch1.tags.list()["main_latest2"]["branch"] is None
-    assert branch1.branches.list()["branch_from_main"]["parent_branch"] is None
-    assert branch1.branches.list()["branch_from_main"]["parent_version"] == 1
+    assert branches_with_main["branch_from_main"]["parent_branch"] is None
+    assert branches_with_main["branch_from_main"]["branch_identifier"][0][0] == 1
+    assert isinstance(
+        branches_with_main["branch_from_main"]["branch_identifier"][0][1], str
+    )
+    assert branches_with_main["branch_from_main"]["parent_version"] == 1
     assert branch1.checkout_version("main_latest").latest_version == 1
     assert branch1.checkout_version("main_latest2").latest_version == 1
     assert branch1.checkout_version(("branch_from_main", None)).latest_version == 1
@@ -5256,6 +5305,9 @@ def test_branches(tmp_path: Path):
     b1_meta = branches["branch1"]
     assert isinstance(b1_meta["parent_version"], int)
     assert b1_meta["manifest_size"] > 0
+    assert b1_meta["branch_identifier"][0][0] == b1_meta["parent_version"]
+    assert isinstance(b1_meta["branch_identifier"][0][1], str)
+    assert len(b1_meta["branch_identifier"][0][1]) > 0
     assert "create_at" in b1_meta
 
     try:
diff --git a/python/python/tests/test_file.py b/python/python/tests/test_file.py
index ea89e7da4a3..c71654769b8 100644
--- a/python/python/tests/test_file.py
+++ b/python/python/tests/test_file.py
@@ -243,9 +243,10 @@ def test_metadata(tmp_path):
     assert len(column.pages) == 1
 
     page = column.pages[0]
-    assert len(page.buffers) == 1
-    assert page.buffers[0].position == 0
-    assert page.buffers[0].size == 24
+    assert len(page.buffers) > 0
+    for buffer in page.buffers:
+        assert buffer.position % 64 == 0
+        assert buffer.size > 0
 
     assert len(page.encoding) > 0
 
diff --git a/python/python/tests/test_fragment.py b/python/python/tests/test_fragment.py
index 40e2a69df22..c6cee711047 100644
--- a/python/python/tests/test_fragment.py
+++ b/python/python/tests/test_fragment.py
@@ -472,9 +472,16 @@ def test_fragment_metadata_pickle(tmp_path: Path, enable_stable_row_ids: bool):
 
     # Pickle and unpickle the fragment metadata
     round_trip = pickle.loads(pickle.dumps(frag_meta))
-
     assert frag_meta == round_trip
 
+    # JSON round-trip
+    json_data = frag_meta.to_json()
+    json_round_trip = FragmentMetadata.from_json(json.dumps(json_data))
+    assert frag_meta.id == json_round_trip.id
+    assert frag_meta.physical_rows == json_round_trip.physical_rows
+    if enable_stable_row_ids:
+        assert json_round_trip.row_id_meta is not None
+
 
 def test_deletion_file_with_base_id_serialization():
     """Test that DeletionFile with base_id serializes correctly."""
diff --git a/python/python/tests/test_indices.py b/python/python/tests/test_indices.py
index e29f02705e2..88cae659561 100644
--- a/python/python/tests/test_indices.py
+++ b/python/python/tests/test_indices.py
@@ -77,6 +77,31 @@ def test_ivf_centroids(tmpdir, rand_dataset):
     assert ivf.centroids == reloaded.centroids
 
 
+def test_ivf_centroids_hamming(tmpdir):
+    num_rows = NUM_ROWS
+    vectors = np.random.randint(0, 256, size=(num_rows, DIMENSION), dtype=np.uint8)
+    vectors_flat = vectors.reshape(-1)
+    vectors_arr = pa.FixedSizeListArray.from_arrays(
+        pa.array(vectors_flat, type=pa.uint8()), DIMENSION
+    )
+    table = pa.Table.from_arrays([vectors_arr], names=["vectors"])
+    uri = str(tmpdir / "hamming_dataset")
+    ds = lance.write_dataset(table, uri, max_rows_per_file=NUM_ROWS_PER_FRAGMENT)
+
+    ivf = IndicesBuilder(ds, "vectors").train_ivf(
+        sample_rate=16, distance_type="hamming"
+    )
+
+    assert ivf.distance_type == "hamming"
+    expected_partitions = round(math.sqrt(num_rows))
+    assert len(ivf.centroids) == expected_partitions
+
+    ivf.save(str(tmpdir / "ivf_hamming"))
+    reloaded = IvfModel.load(str(tmpdir / "ivf_hamming"))
+    assert reloaded.distance_type == "hamming"
+    assert ivf.centroids == reloaded.centroids
+
+
 @pytest.mark.parametrize("distance_type", ["l2", "cosine", "dot"])
 def test_ivf_centroids_mostly_null(mostly_null_dataset, distance_type):
     ivf = IndicesBuilder(mostly_null_dataset, "vectors").train_ivf(
@@ -159,6 +184,58 @@ def test_gen_pq(tmpdir, rand_dataset, rand_ivf):
     assert pq.codebook == reloaded.codebook
 
 
+def test_ivf_centroids_fragment_ids(tmpdir):
+    rows_per_fragment = 32
+    vectors = np.concatenate(
+        [
+            np.zeros((rows_per_fragment, DIMENSION), dtype=np.float32),
+            np.full((rows_per_fragment, DIMENSION), 10.0, dtype=np.float32),
+        ],
+        axis=0,
+    )
+    vectors.shape = -1
+    table = pa.Table.from_arrays(
+        [pa.FixedSizeListArray.from_arrays(vectors, DIMENSION)], names=["vectors"]
+    )
+    ds = lance.write_dataset(
+        table,
+        pathlib.Path(tmpdir) / "fragment_ivf",
+        max_rows_per_file=rows_per_fragment,
+    )
+    fragment_ids = [fragment.fragment_id for fragment in ds.get_fragments()]
+
+    first_ivf = IndicesBuilder(ds, "vectors").train_ivf(
+        num_partitions=1, sample_rate=2, fragment_ids=[fragment_ids[0]]
+    )
+    second_ivf = IndicesBuilder(ds, "vectors").train_ivf(
+        num_partitions=1, sample_rate=2, fragment_ids=[fragment_ids[1]]
+    )
+
+    first_centroid = first_ivf.centroids.values.to_numpy().reshape(-1, DIMENSION)[0]
+    second_centroid = second_ivf.centroids.values.to_numpy().reshape(-1, DIMENSION)[0]
+
+    assert np.allclose(first_centroid, 0.0, atol=1e-4)
+    assert np.allclose(second_centroid, 10.0, atol=1e-4)
+
+
+def test_pq_fragment_ids(rand_dataset):
+    fragment_id = rand_dataset.get_fragments()[0].fragment_id
+    ivf = IndicesBuilder(rand_dataset, "vectors").train_ivf(
+        num_partitions=4,
+        sample_rate=16,
+        fragment_ids=[fragment_id],
+    )
+
+    pq = IndicesBuilder(rand_dataset, "vectors").train_pq(
+        ivf,
+        sample_rate=2,
+        fragment_ids=[fragment_id],
+    )
+
+    assert pq.dimension == DIMENSION
+    assert pq.num_subvectors == NUM_SUBVECTORS
+
+
 def test_pq_invalid_sub_vectors(tmpdir, rand_dataset, rand_ivf):
     with pytest.raises(
         ValueError,
diff --git a/python/python/tests/test_integration.py b/python/python/tests/test_integration.py
index 399565f1024..3647ec83eef 100644
--- a/python/python/tests/test_integration.py
+++ b/python/python/tests/test_integration.py
@@ -16,7 +16,7 @@ def test_duckdb_filter_on_rowid(tmp_path):
     expected = tab.slice(1, 1)
     actual = duckdb.query(
         f"SELECT * FROM ds WHERE _rowid = {row_ids[1]}"
-    ).fetch_arrow_table()
+    ).to_arrow_table()
 
     assert actual.to_pydict() == expected.to_pydict()
 
@@ -37,11 +37,11 @@ def test_duckdb_pushdown_extension_types(tmp_path):
     )
     ds = lance.write_dataset(tab, str(tmp_path))  # noqa: F841
     expected = tab.slice(1, 1)
-    actual = duckdb.query("SELECT * FROM ds WHERE filterme = 2").fetch_arrow_table()
+    actual = duckdb.query("SELECT * FROM ds WHERE filterme = 2").to_arrow_table()
     assert actual.to_pydict() == expected.to_pydict()
 
     expected = tab.slice(0, 1)
-    actual = duckdb.query("SELECT * FROM ds WHERE othercol = 4").fetch_arrow_table()
+    actual = duckdb.query("SELECT * FROM ds WHERE othercol = 4").to_arrow_table()
     assert actual.to_pydict() == expected.to_pydict()
 
     # Not the best error message but hopefully this is short lived until datafusion
@@ -64,6 +64,6 @@ def test_duckdb_pushdown_extension_types(tmp_path):
         "filterme IS NOT NULL",
         "filterme < 2",
     ]:
-        expected = duckdb.query(f"SELECT * FROM tab WHERE {filt}").fetch_arrow_table()
-        actual = duckdb.query(f"SELECT * FROM ds WHERE {filt}").fetch_arrow_table()
+        expected = duckdb.query(f"SELECT * FROM tab WHERE {filt}").to_arrow_table()
+        actual = duckdb.query(f"SELECT * FROM ds WHERE {filt}").to_arrow_table()
         assert actual == expected
diff --git a/python/python/tests/test_memory_leaks.py b/python/python/tests/test_memory_leaks.py
index 29907089ba0..20ba1686875 100644
--- a/python/python/tests/test_memory_leaks.py
+++ b/python/python/tests/test_memory_leaks.py
@@ -5,6 +5,8 @@
 
 import gc
 import os
+import subprocess
+import sys
 from typing import Callable
 
 import lance
@@ -78,18 +80,40 @@ def assert_noleaks(
         )
 
 
+def run_isolated_probe(probe_name: str, *args: str) -> None:
+    result = subprocess.run(
+        [sys.executable, __file__, probe_name, *args],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 0, (
+        f"Isolated probe '{probe_name}' failed.\n"
+        f"stdout:\n{result.stdout}\n"
+        f"stderr:\n{result.stderr}"
+    )
+
+
+def probe_index_statistics_no_leak(dataset_path: str) -> None:
+    def access_index_stats() -> None:
+        d = lance.dataset(dataset_path)
+        for idx in d.describe_indices():
+            d.stats.index_stats(idx.name)
+
+    assert_noleaks(
+        access_index_stats, iterations=1000, threshold_mb=2.0, check_interval=25
+    )
+
+
 class TestMemoryLeaks:
     def test_index_statistics_no_leak(self, tmp_path) -> None:
         dataset_path = str(tmp_path / "dataset")
         data = pa.table({"id": [1]})
         ds = lance.write_dataset(data, dataset_path)
         ds.create_scalar_index("id", index_type="BTREE")
+        run_isolated_probe("index_statistics_no_leak", dataset_path)
 
-        def access_index_stats() -> None:
-            d = lance.dataset(dataset_path)
-            for idx in d.describe_indices():
-                d.stats.index_stats(idx.name)
 
-        assert_noleaks(
-            access_index_stats, iterations=1000, threshold_mb=2.0, check_interval=25
-        )
+if __name__ == "__main__":
+    if len(sys.argv) == 3 and sys.argv[1] == "index_statistics_no_leak":
+        probe_index_statistics_no_leak(sys.argv[2])
diff --git a/python/python/tests/test_namespace_dir.py b/python/python/tests/test_namespace_dir.py
index d1200d4a6c8..1f0a4180d5d 100644
--- a/python/python/tests/test_namespace_dir.py
+++ b/python/python/tests/test_namespace_dir.py
@@ -8,39 +8,175 @@
 namespace implementation for organizing Lance tables and nested namespaces.
 
 These tests mirror the Rust tests in rust/lance-namespace-impls/src/dir.rs
+
+Additionally, tests are parameterized to run with both the native DirectoryNamespace
+and a CustomNamespace wrapper to verify Python-Rust binding works correctly for
+custom namespace implementations.
 """
 
 import sys
 import tempfile
 import uuid
 from threading import Lock
+from typing import Dict, Optional
 
 import lance
 import lance.namespace
 import pyarrow as pa
 import pytest
+from lance.namespace import LanceNamespace
 from lance_namespace import (
+    CountTableRowsRequest,
     CreateNamespaceRequest,
+    CreateNamespaceResponse,
+    CreateTableIndexRequest,
+    CreateTableIndexResponse,
     CreateTableRequest,
+    CreateTableResponse,
     CreateTableVersionRequest,
     CreateTableVersionResponse,
     DeclareTableRequest,
+    DeclareTableResponse,
     DeregisterTableRequest,
+    DeregisterTableResponse,
     DescribeNamespaceRequest,
+    DescribeNamespaceResponse,
+    DescribeTableIndexStatsRequest,
     DescribeTableRequest,
+    DescribeTableResponse,
     DescribeTableVersionRequest,
     DescribeTableVersionResponse,
     DropNamespaceRequest,
+    DropNamespaceResponse,
     DropTableRequest,
+    DropTableResponse,
+    InsertIntoTableRequest,
+    InsertIntoTableResponse,
     ListNamespacesRequest,
+    ListNamespacesResponse,
+    ListTableIndicesRequest,
+    ListTableIndicesResponse,
     ListTablesRequest,
+    ListTablesResponse,
     ListTableVersionsRequest,
     ListTableVersionsResponse,
     NamespaceExistsRequest,
+    QueryTableRequest,
     RegisterTableRequest,
+    RegisterTableResponse,
     TableExistsRequest,
     connect,
 )
+from lance_namespace.errors import (
+    InvalidInputError,
+    NamespaceNotEmptyError,
+    NamespaceNotFoundError,
+    TableNotFoundError,
+)
+
+
+class CustomNamespace(LanceNamespace):
+    """A custom namespace wrapper that delegates to DirectoryNamespace.
+
+    This class verifies that the Python-Rust binding works correctly for
+    custom namespace implementations that wrap the native DirectoryNamespace.
+    All methods simply delegate to the underlying DirectoryNamespace instance.
+    """
+
+    def __init__(self, inner: lance.namespace.DirectoryNamespace):
+        self._inner = inner
+
+    def namespace_id(self) -> str:
+        return f"CustomNamespace[{self._inner.namespace_id()}]"
+
+    def create_namespace(
+        self, request: CreateNamespaceRequest
+    ) -> CreateNamespaceResponse:
+        return self._inner.create_namespace(request)
+
+    def describe_namespace(
+        self, request: DescribeNamespaceRequest
+    ) -> DescribeNamespaceResponse:
+        return self._inner.describe_namespace(request)
+
+    def namespace_exists(self, request: NamespaceExistsRequest) -> None:
+        return self._inner.namespace_exists(request)
+
+    def drop_namespace(self, request: DropNamespaceRequest) -> DropNamespaceResponse:
+        return self._inner.drop_namespace(request)
+
+    def list_namespaces(self, request: ListNamespacesRequest) -> ListNamespacesResponse:
+        return self._inner.list_namespaces(request)
+
+    def create_table(
+        self, request: CreateTableRequest, data: bytes
+    ) -> CreateTableResponse:
+        return self._inner.create_table(request, data)
+
+    def declare_table(self, request: DeclareTableRequest) -> DeclareTableResponse:
+        return self._inner.declare_table(request)
+
+    def describe_table(self, request: DescribeTableRequest) -> DescribeTableResponse:
+        return self._inner.describe_table(request)
+
+    def table_exists(self, request: TableExistsRequest) -> None:
+        return self._inner.table_exists(request)
+
+    def drop_table(self, request: DropTableRequest) -> DropTableResponse:
+        return self._inner.drop_table(request)
+
+    def list_tables(self, request: ListTablesRequest) -> ListTablesResponse:
+        return self._inner.list_tables(request)
+
+    def register_table(self, request: RegisterTableRequest) -> RegisterTableResponse:
+        return self._inner.register_table(request)
+
+    def deregister_table(
+        self, request: DeregisterTableRequest
+    ) -> DeregisterTableResponse:
+        return self._inner.deregister_table(request)
+
+    def list_table_versions(
+        self, request: ListTableVersionsRequest
+    ) -> ListTableVersionsResponse:
+        return self._inner.list_table_versions(request)
+
+    def describe_table_version(
+        self, request: DescribeTableVersionRequest
+    ) -> DescribeTableVersionResponse:
+        return self._inner.describe_table_version(request)
+
+    def create_table_version(
+        self, request: CreateTableVersionRequest
+    ) -> CreateTableVersionResponse:
+        return self._inner.create_table_version(request)
+
+    def create_table_index(
+        self, request: CreateTableIndexRequest
+    ) -> CreateTableIndexResponse:
+        return self._inner.create_table_index(request)
+
+    def list_table_indices(
+        self, request: ListTableIndicesRequest
+    ) -> ListTableIndicesResponse:
+        return self._inner.list_table_indices(request)
+
+    def count_table_rows(self, request: CountTableRowsRequest) -> int:
+        return self._inner.count_table_rows(request)
+
+    def insert_into_table(
+        self, request: InsertIntoTableRequest, request_data: bytes
+    ) -> InsertIntoTableResponse:
+        return self._inner.insert_into_table(request, request_data)
+
+    def query_table(self, request) -> bytes:
+        # Accept both QueryTableRequest and dict, like DirectoryNamespace does
+        if hasattr(request, "model_dump"):
+            request = request.model_dump()
+        return self._inner.query_table(request)
+
+    def retrieve_ops_metrics(self) -> Optional[Dict[str, int]]:
+        return self._inner.retrieve_ops_metrics()
 
 
 def create_test_data():
@@ -64,42 +200,57 @@ def table_to_ipc_bytes(table):
     return sink.getvalue()
 
 
-@pytest.fixture
-def temp_namespace():
-    """Create a temporary DirectoryNamespace for testing."""
+def _wrap_if_custom(ns_client, use_custom: bool):
+    """Wrap namespace client in CustomNamespace if use_custom is True."""
+    if use_custom:
+        return CustomNamespace(ns_client)
+    return ns_client
+
+
+@pytest.fixture(params=[False, True], ids=["DirectoryNamespace", "CustomNamespace"])
+def temp_ns_client(request):
+    """Create a temporary namespace client for testing.
+
+    Parameterized to test both DirectoryNamespace and CustomNamespace wrapper.
+    """
+    use_custom = request.param
     with tempfile.TemporaryDirectory() as tmpdir:
         # Use lance.namespace.connect() for consistency
         # Use high commit_retries for concurrent operation tests
-        ns = connect(
+        ns_client = connect(
             "dir", {"root": f"file://{tmpdir}", "commit_retries": "2147483647"}
         )
-        yield ns
+        yield _wrap_if_custom(ns_client, use_custom)
 
 
-@pytest.fixture
-def memory_namespace():
-    """Create a memory-based DirectoryNamespace for testing."""
+@pytest.fixture(params=[False, True], ids=["DirectoryNamespace", "CustomNamespace"])
+def memory_ns_client(request):
+    """Create a memory-based namespace client for testing.
+
+    Parameterized to test both DirectoryNamespace and CustomNamespace wrapper.
+    """
+    use_custom = request.param
     unique_id = uuid.uuid4().hex[:8]
     # Use lance.namespace.connect() for consistency
-    ns = connect("dir", {"root": f"memory://test_{unique_id}"})
-    yield ns
+    ns_client = connect("dir", {"root": f"memory://test_{unique_id}"})
+    yield _wrap_if_custom(ns_client, use_custom)
 
 
 class TestCreateTable:
     """Tests for create_table operation - mirrors Rust test_create_table."""
 
-    def test_create_table(self, memory_namespace):
+    def test_create_table(self, memory_ns_client):
         """Test creating a table with data."""
         # Create parent namespace first
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         # Create table with data
         table_data = create_test_data()
         ipc_data = table_to_ipc_bytes(table_data)
 
         create_req = CreateTableRequest(id=["workspace", "test_table"])
-        response = memory_namespace.create_table(create_req, ipc_data)
+        response = memory_ns_client.create_table(create_req, ipc_data)
 
         assert response is not None
         assert response.location is not None
@@ -108,40 +259,40 @@ def test_create_table(self, memory_namespace):
         assert "test_table" in response.location
         assert response.version == 1
 
-    def test_create_table_without_data(self, memory_namespace):
+    def test_create_table_without_data(self, memory_ns_client):
         """Test creating a table without data should fail."""
         # Create parent namespace first
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         create_req = CreateTableRequest(id=["workspace", "test_table"])
 
-        with pytest.raises(Exception) as exc_info:
-            memory_namespace.create_table(create_req, b"")
+        with pytest.raises(InvalidInputError) as exc_info:
+            memory_ns_client.create_table(create_req, b"")
 
         assert "Arrow IPC" in str(exc_info.value) or "required" in str(exc_info.value)
 
-    def test_create_table_with_invalid_id(self, memory_namespace):
+    def test_create_table_with_invalid_id(self, memory_ns_client):
         """Test creating a table with invalid ID should fail."""
         table_data = create_test_data()
         ipc_data = table_to_ipc_bytes(table_data)
 
         # Test with empty ID
         create_req = CreateTableRequest(id=[])
-        with pytest.raises(Exception):
-            memory_namespace.create_table(create_req, ipc_data)
+        with pytest.raises(InvalidInputError):
+            memory_ns_client.create_table(create_req, ipc_data)
 
-    def test_create_table_in_child_namespace(self, memory_namespace):
+    def test_create_table_in_child_namespace(self, memory_ns_client):
         """Test creating table in child namespace works with manifest enabled."""
         # Create parent namespace
         create_ns_req = CreateNamespaceRequest(id=["test_namespace"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         # Create table in the namespace
         table_data = create_test_data()
         ipc_data = table_to_ipc_bytes(table_data)
         create_req = CreateTableRequest(id=["test_namespace", "table"])
-        response = memory_namespace.create_table(create_req, ipc_data)
+        response = memory_ns_client.create_table(create_req, ipc_data)
 
         # Should succeed with manifest enabled
         assert response is not None
@@ -151,52 +302,52 @@ def test_create_table_in_child_namespace(self, memory_namespace):
 class TestListTables:
     """Tests for list_tables operation - mirrors Rust test_list_tables."""
 
-    def test_list_tables_empty(self, memory_namespace):
+    def test_list_tables_empty(self, memory_ns_client):
         """Test listing tables in empty namespace."""
         # Create parent namespace
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         # Initially, no tables
         list_req = ListTablesRequest(id=["workspace"])
-        response = memory_namespace.list_tables(list_req)
+        response = memory_ns_client.list_tables(list_req)
         assert len(response.tables) == 0
 
-    def test_list_tables_with_tables(self, memory_namespace):
+    def test_list_tables_with_tables(self, memory_ns_client):
         """Test listing tables after creating them."""
         # Create parent namespace
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         table_data = create_test_data()
         ipc_data = table_to_ipc_bytes(table_data)
 
         # Create table1
         create_req = CreateTableRequest(id=["workspace", "table1"])
-        memory_namespace.create_table(create_req, ipc_data)
+        memory_ns_client.create_table(create_req, ipc_data)
 
         # Create table2
         create_req = CreateTableRequest(id=["workspace", "table2"])
-        memory_namespace.create_table(create_req, ipc_data)
+        memory_ns_client.create_table(create_req, ipc_data)
 
         # List tables should return both
         list_req = ListTablesRequest(id=["workspace"])
-        response = memory_namespace.list_tables(list_req)
+        response = memory_ns_client.list_tables(list_req)
         assert len(response.tables) == 2
 
         # List tables returns table names as strings
         assert "table1" in response.tables
         assert "table2" in response.tables
 
-    def test_list_tables_with_namespace_id(self, memory_namespace):
+    def test_list_tables_with_namespace_id(self, memory_ns_client):
         """Test listing tables in a child namespace."""
         # Create child namespace
         create_ns_req = CreateNamespaceRequest(id=["test_namespace"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         # List tables in the child namespace
         list_req = ListTablesRequest(id=["test_namespace"])
-        response = memory_namespace.list_tables(list_req)
+        response = memory_ns_client.list_tables(list_req)
 
         # Should succeed and return empty list (no tables yet)
         assert len(response.tables) == 0
@@ -205,104 +356,100 @@ def test_list_tables_with_namespace_id(self, memory_namespace):
 class TestDescribeTable:
     """Tests for describe_table operation - mirrors Rust test_describe_table."""
 
-    def test_describe_table(self, memory_namespace):
+    def test_describe_table(self, memory_ns_client):
         """Test describing a table."""
         # Create parent namespace
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         # Create a table
         table_data = create_test_data()
         ipc_data = table_to_ipc_bytes(table_data)
         create_req = CreateTableRequest(id=["workspace", "test_table"])
-        memory_namespace.create_table(create_req, ipc_data)
+        memory_ns_client.create_table(create_req, ipc_data)
 
         # Describe the table
         describe_req = DescribeTableRequest(id=["workspace", "test_table"])
-        response = memory_namespace.describe_table(describe_req)
+        response = memory_ns_client.describe_table(describe_req)
 
         assert response is not None
         assert response.location is not None
         assert "test_table" in response.location
 
-    def test_describe_nonexistent_table(self, memory_namespace):
+    def test_describe_nonexistent_table(self, memory_ns_client):
         """Test describing a table that doesn't exist."""
         # Create parent namespace
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         describe_req = DescribeTableRequest(id=["workspace", "nonexistent"])
 
-        with pytest.raises(Exception) as exc_info:
-            memory_namespace.describe_table(describe_req)
-
-        # Check for error message indicating table doesn't exist
-        error_msg = str(exc_info.value).lower()
-        assert "not found" in error_msg or "does not exist" in error_msg
+        with pytest.raises(TableNotFoundError):
+            memory_ns_client.describe_table(describe_req)
 
 
 class TestTableOperations:
     """Tests for various table operations."""
 
-    def test_table_exists(self, memory_namespace):
+    def test_table_exists(self, memory_ns_client):
         """Test checking if a table exists."""
         # Create parent namespace
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         # Create a table
         table_data = create_test_data()
         ipc_data = table_to_ipc_bytes(table_data)
         create_req = CreateTableRequest(id=["workspace", "test_table"])
-        memory_namespace.create_table(create_req, ipc_data)
+        memory_ns_client.create_table(create_req, ipc_data)
 
         # Check it exists (should not raise)
         exists_req = TableExistsRequest(id=["workspace", "test_table"])
-        memory_namespace.table_exists(exists_req)
+        memory_ns_client.table_exists(exists_req)
 
-    def test_table_not_exists(self, memory_namespace):
-        """Test checking if a non-existent table exists."""
+    def test_table_not_exists(self, memory_ns_client):
+        """Test checking if a non-existent table exists raises TableNotFoundError."""
         # Create parent namespace
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         exists_req = TableExistsRequest(id=["workspace", "nonexistent"])
 
-        with pytest.raises(Exception):
-            memory_namespace.table_exists(exists_req)
+        with pytest.raises(TableNotFoundError):
+            memory_ns_client.table_exists(exists_req)
 
-    def test_drop_table(self, memory_namespace):
+    def test_drop_table(self, memory_ns_client):
         """Test dropping a table."""
         # Create parent namespace
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         # Create table
         table_data = create_test_data()
         ipc_data = table_to_ipc_bytes(table_data)
         create_req = CreateTableRequest(id=["workspace", "test_table"])
-        memory_namespace.create_table(create_req, ipc_data)
+        memory_ns_client.create_table(create_req, ipc_data)
 
         # Drop the table
         drop_req = DropTableRequest(id=["workspace", "test_table"])
-        response = memory_namespace.drop_table(drop_req)
+        response = memory_ns_client.drop_table(drop_req)
         assert response is not None
 
         # Verify table no longer exists
         exists_req = TableExistsRequest(id=["workspace", "test_table"])
-        with pytest.raises(Exception):
-            memory_namespace.table_exists(exists_req)
+        with pytest.raises(TableNotFoundError):
+            memory_ns_client.table_exists(exists_req)
 
-    def test_deregister_table(self, temp_namespace):
+    def test_deregister_table(self, temp_ns_client):
         """Test deregistering a table."""
         # Create parent namespace
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        temp_namespace.create_namespace(create_ns_req)
+        temp_ns_client.create_namespace(create_ns_req)
 
         # Create table using lance directly
         table_data = create_test_data()
         # Get root path from namespace
-        ns_id = temp_namespace.namespace_id()
+        ns_id = temp_ns_client.namespace_id()
         import re
 
         match = re.search(r'root: "([^"]+)"', ns_id)
@@ -317,11 +464,11 @@ def test_deregister_table(self, temp_namespace):
         register_req = RegisterTableRequest(
             id=["workspace", "test_table"], location="workspace/physical_table.lance"
         )
-        temp_namespace.register_table(register_req)
+        temp_ns_client.register_table(register_req)
 
         # Deregister it
         deregister_req = DeregisterTableRequest(id=["workspace", "test_table"])
-        response = temp_namespace.deregister_table(deregister_req)
+        response = temp_ns_client.deregister_table(deregister_req)
         assert response is not None
         # Should return full URI to deregistered table
         # (use endswith to handle path canonicalization)
@@ -331,15 +478,15 @@ def test_deregister_table(self, temp_namespace):
         )
         assert response.id == ["workspace", "test_table"]
 
-    def test_register_table(self, temp_namespace):
+    def test_register_table(self, temp_ns_client):
         """Test registering an existing table."""
         # Create parent namespace
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        temp_namespace.create_namespace(create_ns_req)
+        temp_ns_client.create_namespace(create_ns_req)
 
         # Create physical table using lance
         table_data = create_test_data()
-        ns_id = temp_namespace.namespace_id()
+        ns_id = temp_ns_client.namespace_id()
         import re
 
         match = re.search(r'root: "([^"]+)"', ns_id)
@@ -355,17 +502,17 @@ def test_register_table(self, temp_namespace):
             id=["workspace", "registered_table"],
             location="workspace/physical_table.lance",
         )
-        response = temp_namespace.register_table(register_req)
+        response = temp_ns_client.register_table(register_req)
         assert response is not None
         assert response.location == "workspace/physical_table.lance"
 
         # Verify table exists
         exists_req = TableExistsRequest(id=["workspace", "registered_table"])
-        temp_namespace.table_exists(exists_req)
+        temp_ns_client.table_exists(exists_req)
 
         # Verify we can read from it
         describe_req = DescribeTableRequest(id=["workspace", "registered_table"])
-        desc_response = temp_namespace.describe_table(describe_req)
+        desc_response = temp_ns_client.describe_table(describe_req)
         assert desc_response is not None
         # Should point to the same physical location
         # (use endswith to handle path canonicalization)
@@ -374,57 +521,57 @@ def test_register_table(self, temp_namespace):
             f"got {desc_response.location}"
         )
 
-    def test_register_table_rejects_absolute_uri(self, temp_namespace):
+    def test_register_table_rejects_absolute_uri(self, temp_ns_client):
         """Test that register_table rejects absolute URIs."""
         # Create parent namespace
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        temp_namespace.create_namespace(create_ns_req)
+        temp_ns_client.create_namespace(create_ns_req)
 
         # Try to register with absolute URI - should fail
         register_req = RegisterTableRequest(
             id=["workspace", "test_table"], location="s3://bucket/table.lance"
         )
-        with pytest.raises(Exception) as exc_info:
-            temp_namespace.register_table(register_req)
+        with pytest.raises(InvalidInputError) as exc_info:
+            temp_ns_client.register_table(register_req)
         assert "Absolute URIs are not allowed" in str(exc_info.value)
 
-    def test_register_table_rejects_absolute_path(self, temp_namespace):
+    def test_register_table_rejects_absolute_path(self, temp_ns_client):
         """Test that register_table rejects absolute paths."""
         # Create parent namespace
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        temp_namespace.create_namespace(create_ns_req)
+        temp_ns_client.create_namespace(create_ns_req)
 
         # Try to register with absolute path - should fail
         register_req = RegisterTableRequest(
             id=["workspace", "test_table"], location="/tmp/table.lance"
         )
-        with pytest.raises(Exception) as exc_info:
-            temp_namespace.register_table(register_req)
+        with pytest.raises(InvalidInputError) as exc_info:
+            temp_ns_client.register_table(register_req)
         assert "Absolute paths are not allowed" in str(exc_info.value)
 
-    def test_register_table_rejects_path_traversal(self, temp_namespace):
+    def test_register_table_rejects_path_traversal(self, temp_ns_client):
         """Test that register_table rejects path traversal attempts."""
         # Create parent namespace
         create_ns_req = CreateNamespaceRequest(id=["workspace"])
-        temp_namespace.create_namespace(create_ns_req)
+        temp_ns_client.create_namespace(create_ns_req)
 
         # Try to register with path traversal - should fail
         register_req = RegisterTableRequest(
             id=["workspace", "test_table"], location="../outside/table.lance"
         )
-        with pytest.raises(Exception) as exc_info:
-            temp_namespace.register_table(register_req)
+        with pytest.raises(InvalidInputError) as exc_info:
+            temp_ns_client.register_table(register_req)
         assert "Path traversal is not allowed" in str(exc_info.value)
 
 
 class TestChildNamespaceOperations:
     """Tests for operations in child namespaces - mirrors Rust tests."""
 
-    def test_create_table_in_child_namespace(self, memory_namespace):
+    def test_create_table_in_child_namespace(self, memory_ns_client):
         """Test creating multiple tables in a child namespace."""
         # Create child namespace
         create_ns_req = CreateNamespaceRequest(id=["test_ns"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         table_data = create_test_data()
         ipc_data = table_to_ipc_bytes(table_data)
@@ -432,11 +579,11 @@ def test_create_table_in_child_namespace(self, memory_namespace):
         # Create three tables
         for i in range(1, 4):
             create_req = CreateTableRequest(id=["test_ns", f"table{i}"])
-            memory_namespace.create_table(create_req, ipc_data)
+            memory_ns_client.create_table(create_req, ipc_data)
 
         # List tables
         list_req = ListTablesRequest(id=["test_ns"])
-        response = memory_namespace.list_tables(list_req)
+        response = memory_ns_client.list_tables(list_req)
 
         assert len(response.tables) == 3
         # List tables returns table names as strings
@@ -444,40 +591,40 @@ def test_create_table_in_child_namespace(self, memory_namespace):
         assert "table2" in response.tables
         assert "table3" in response.tables
 
-    def test_drop_table_in_child_namespace(self, memory_namespace):
+    def test_drop_table_in_child_namespace(self, memory_ns_client):
         """Test dropping a table in a child namespace."""
         # Create child namespace
         create_ns_req = CreateNamespaceRequest(id=["test_ns"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         # Create table
         table_data = create_test_data()
         ipc_data = table_to_ipc_bytes(table_data)
         create_req = CreateTableRequest(id=["test_ns", "table1"])
-        memory_namespace.create_table(create_req, ipc_data)
+        memory_ns_client.create_table(create_req, ipc_data)
 
         # Drop table
         drop_req = DropTableRequest(id=["test_ns", "table1"])
-        memory_namespace.drop_table(drop_req)
+        memory_ns_client.drop_table(drop_req)
 
         # Verify table no longer exists
         exists_req = TableExistsRequest(id=["test_ns", "table1"])
-        with pytest.raises(Exception):
-            memory_namespace.table_exists(exists_req)
+        with pytest.raises(TableNotFoundError):
+            memory_ns_client.table_exists(exists_req)
 
-    def test_declared_table_in_child_namespace(self, memory_namespace):
+    def test_declared_table_in_child_namespace(self, memory_ns_client):
         """Test declaring a table in a child namespace."""
         # Create child namespace
         create_ns_req = CreateNamespaceRequest(id=["test_ns"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         # Declare table
         declare_req = DeclareTableRequest(id=["test_ns", "declared_table"])
-        memory_namespace.declare_table(declare_req)
+        memory_ns_client.declare_table(declare_req)
 
         # Verify table exists
         exists_req = TableExistsRequest(id=["test_ns", "declared_table"])
-        memory_namespace.table_exists(exists_req)
+        memory_ns_client.table_exists(exists_req)
 
 
 class TestDeeplyNestedNamespaces:
@@ -486,14 +633,14 @@ class TestDeeplyNestedNamespaces:
     Mirrors Rust test_deeply_nested_namespace.
     """
 
-    def test_deeply_nested_namespace(self, memory_namespace):
+    def test_deeply_nested_namespace(self, memory_ns_client):
         """Test creating deeply nested namespace hierarchy."""
         # Create deeply nested namespace hierarchy
-        memory_namespace.create_namespace(CreateNamespaceRequest(id=["level1"]))
-        memory_namespace.create_namespace(
+        memory_ns_client.create_namespace(CreateNamespaceRequest(id=["level1"]))
+        memory_ns_client.create_namespace(
             CreateNamespaceRequest(id=["level1", "level2"])
         )
-        memory_namespace.create_namespace(
+        memory_ns_client.create_namespace(
             CreateNamespaceRequest(id=["level1", "level2", "level3"])
         )
 
@@ -501,17 +648,17 @@ def test_deeply_nested_namespace(self, memory_namespace):
         table_data = create_test_data()
         ipc_data = table_to_ipc_bytes(table_data)
         create_req = CreateTableRequest(id=["level1", "level2", "level3", "table1"])
-        memory_namespace.create_table(create_req, ipc_data)
+        memory_ns_client.create_table(create_req, ipc_data)
 
         # Verify table exists
         exists_req = TableExistsRequest(id=["level1", "level2", "level3", "table1"])
-        memory_namespace.table_exists(exists_req)
+        memory_ns_client.table_exists(exists_req)
 
 
 class TestNamespaceProperties:
     """Tests for namespace properties - mirrors Rust test_namespace_with_properties."""
 
-    def test_namespace_with_properties(self, memory_namespace):
+    def test_namespace_with_properties(self, memory_ns_client):
         """Test creating a namespace with properties."""
         # Create namespace with properties
         properties = {
@@ -520,11 +667,11 @@ def test_namespace_with_properties(self, memory_namespace):
         }
 
         create_req = CreateNamespaceRequest(id=["test_ns"], properties=properties)
-        memory_namespace.create_namespace(create_req)
+        memory_ns_client.create_namespace(create_req)
 
         # Describe namespace and verify properties
         describe_req = DescribeNamespaceRequest(id=["test_ns"])
-        response = memory_namespace.describe_namespace(describe_req)
+        response = memory_ns_client.describe_namespace(describe_req)
 
         assert response.properties is not None
         assert response.properties.get("owner") == "test_user"
@@ -534,120 +681,121 @@ def test_namespace_with_properties(self, memory_namespace):
 class TestNamespaceConstraints:
     """Tests for namespace constraints and isolation."""
 
-    def test_cannot_drop_namespace_with_tables(self, memory_namespace):
+    def test_cannot_drop_namespace_with_tables(self, memory_ns_client):
         """Test that dropping a namespace with tables should fail."""
         # Create namespace
         create_ns_req = CreateNamespaceRequest(id=["test_ns"])
-        memory_namespace.create_namespace(create_ns_req)
+        memory_ns_client.create_namespace(create_ns_req)
 
         # Create table in namespace
         table_data = create_test_data()
         ipc_data = table_to_ipc_bytes(table_data)
         create_req = CreateTableRequest(id=["test_ns", "table1"])
-        memory_namespace.create_table(create_req, ipc_data)
+        memory_ns_client.create_table(create_req, ipc_data)
 
         # Try to drop namespace - should fail
         drop_req = DropNamespaceRequest(id=["test_ns"])
-        with pytest.raises(Exception) as exc_info:
-            memory_namespace.drop_namespace(drop_req)
-
-        # Should contain an error message about non-empty namespace
-        assert (
-            "not empty" in str(exc_info.value).lower()
-            or "contains" in str(exc_info.value).lower()
-        )
+        with pytest.raises(NamespaceNotEmptyError):
+            memory_ns_client.drop_namespace(drop_req)
 
-    def test_isolation_between_namespaces(self, memory_namespace):
+    def test_isolation_between_namespaces(self, memory_ns_client):
         """Test that namespaces are isolated from each other."""
         # Create two namespaces
-        memory_namespace.create_namespace(CreateNamespaceRequest(id=["ns1"]))
-        memory_namespace.create_namespace(CreateNamespaceRequest(id=["ns2"]))
+        memory_ns_client.create_namespace(CreateNamespaceRequest(id=["ns1"]))
+        memory_ns_client.create_namespace(CreateNamespaceRequest(id=["ns2"]))
 
         # Create table with same name in both namespaces
         table_data = create_test_data()
         ipc_data = table_to_ipc_bytes(table_data)
 
         create_req1 = CreateTableRequest(id=["ns1", "table1"])
-        memory_namespace.create_table(create_req1, ipc_data)
+        memory_ns_client.create_table(create_req1, ipc_data)
 
         create_req2 = CreateTableRequest(id=["ns2", "table1"])
-        memory_namespace.create_table(create_req2, ipc_data)
+        memory_ns_client.create_table(create_req2, ipc_data)
 
         # List tables in each namespace
         list_req = ListTablesRequest(id=["ns1"])
-        response = memory_namespace.list_tables(list_req)
+        response = memory_ns_client.list_tables(list_req)
         assert len(response.tables) == 1
         assert "table1" in response.tables
 
         list_req = ListTablesRequest(id=["ns2"])
-        response = memory_namespace.list_tables(list_req)
+        response = memory_ns_client.list_tables(list_req)
         assert len(response.tables) == 1
         assert "table1" in response.tables
 
         # Drop table in ns1 shouldn't affect ns2
         drop_req = DropTableRequest(id=["ns1", "table1"])
-        memory_namespace.drop_table(drop_req)
+        memory_ns_client.drop_table(drop_req)
 
         # ns1 should have no tables
         list_req = ListTablesRequest(id=["ns1"])
-        response = memory_namespace.list_tables(list_req)
+        response = memory_ns_client.list_tables(list_req)
         assert len(response.tables) == 0
 
         # ns2 should still have the table
         list_req = ListTablesRequest(id=["ns2"])
-        response = memory_namespace.list_tables(list_req)
+        response = memory_ns_client.list_tables(list_req)
         assert len(response.tables) == 1
 
 
 class TestBasicNamespaceOperations:
     """Tests for basic namespace CRUD operations."""
 
-    def test_create_and_describe_namespace(self, memory_namespace):
+    def test_create_and_describe_namespace(self, memory_ns_client):
         """Test creating and describing a namespace."""
         # Create namespace
         create_req = CreateNamespaceRequest(id=["workspace"])
-        memory_namespace.create_namespace(create_req)
+        memory_ns_client.create_namespace(create_req)
 
         # Describe it
         describe_req = DescribeNamespaceRequest(id=["workspace"])
-        response = memory_namespace.describe_namespace(describe_req)
+        response = memory_ns_client.describe_namespace(describe_req)
         assert response is not None
 
-    def test_namespace_exists(self, memory_namespace):
+    def test_namespace_exists(self, memory_ns_client):
         """Test checking if a namespace exists."""
         # Create namespace
         create_req = CreateNamespaceRequest(id=["workspace"])
-        memory_namespace.create_namespace(create_req)
+        memory_ns_client.create_namespace(create_req)
 
         # Check it exists (should not raise)
         exists_req = NamespaceExistsRequest(id=["workspace"])
-        memory_namespace.namespace_exists(exists_req)
+        memory_ns_client.namespace_exists(exists_req)
+
+    def test_namespace_not_exists(self, memory_ns_client):
+        """Test that a non-existent namespace raises NamespaceNotFoundError."""
+        exists_req = NamespaceExistsRequest(id=["nonexistent"])
+
+        with pytest.raises(NamespaceNotFoundError):
+            memory_ns_client.namespace_exists(exists_req)
 
-    def test_drop_empty_namespace(self, memory_namespace):
+    def test_drop_empty_namespace(self, memory_ns_client):
         """Test dropping an empty namespace."""
         # Create namespace
         create_req = CreateNamespaceRequest(id=["workspace"])
-        memory_namespace.create_namespace(create_req)
+        memory_ns_client.create_namespace(create_req)
 
         # Drop it
         drop_req = DropNamespaceRequest(id=["workspace"])
-        response = memory_namespace.drop_namespace(drop_req)
+        response = memory_ns_client.drop_namespace(drop_req)
         assert response is not None
 
-    def test_list_namespaces(self, memory_namespace):
+    def test_list_namespaces(self, memory_ns_client):
         """Test listing namespaces."""
         # Create some child namespaces under a parent
-        memory_namespace.create_namespace(CreateNamespaceRequest(id=["parent"]))
-        memory_namespace.create_namespace(
+        memory_ns_client.create_namespace(CreateNamespaceRequest(id=["parent"]))
+        memory_ns_client.create_namespace(
             CreateNamespaceRequest(id=["parent", "child1"])
         )
-        memory_namespace.create_namespace(
+        memory_ns_client.create_namespace(
             CreateNamespaceRequest(id=["parent", "child2"])
         )
 
         # List namespaces under parent
         list_req = ListNamespacesRequest(id=["parent"])
-        response = memory_namespace.list_namespaces(list_req)
+        response = memory_ns_client.list_namespaces(list_req)
 
         assert response is not None
         # Should find the child namespaces
@@ -657,42 +805,48 @@ def test_list_namespaces(self, memory_namespace):
 class TestLanceNamespaceConnect:
     """Tests for lance.namespace.connect integration."""
 
-    def test_connect_with_properties(self):
+    @pytest.mark.parametrize(
+        "use_custom", [False, True], ids=["DirectoryNS", "CustomNS"]
+    )
+    def test_connect_with_properties(self, use_custom):
         """Test creating DirectoryNamespace via lance.namespace.connect()."""
-        import uuid
-
         unique_id = uuid.uuid4().hex[:8]
         properties = {
             "root": f"memory://test_connect_{unique_id}",
             "manifest_enabled": "true",
             "dir_listing_enabled": "true",
+            "dir_listing_to_manifest_migration_enabled": "true",
         }
 
         # Connect via lance.namespace.connect
         # should use lance.namespace.DirectoryNamespace
-        ns = connect("dir", properties)
+        inner_ns_client = connect("dir", properties)
 
         # Verify it's a DirectoryNamespace instance
-        assert isinstance(ns, lance.namespace.DirectoryNamespace)
+        assert isinstance(inner_ns_client, lance.namespace.DirectoryNamespace)
+
+        # Wrap if testing CustomNamespace
+        ns_client = _wrap_if_custom(inner_ns_client, use_custom)
 
         # Verify it works
         create_req = CreateTableRequest(id=["test_table"])
         table_data = create_test_data()
         ipc_data = table_to_ipc_bytes(table_data)
-        response = ns.create_table(create_req, ipc_data)
+        response = ns_client.create_table(create_req, ipc_data)
         assert response is not None
 
         # Verify we can list the table
         list_req = ListTablesRequest(id=[])
-        list_response = ns.list_tables(list_req)
+        list_response = ns_client.list_tables(list_req)
         assert len(list_response.tables) == 1
         # tables is a list of strings
         assert list_response.tables[0] == "test_table"
 
-    def test_connect_with_storage_options(self):
+    @pytest.mark.parametrize(
+        "use_custom", [False, True], ids=["DirectoryNS", "CustomNS"]
+    )
+    def test_connect_with_storage_options(self, use_custom):
         """Test creating DirectoryNamespace with storage options via connect()."""
-        import uuid
-
         unique_id = uuid.uuid4().hex[:8]
         properties = {
             "root": f"memory://test_storage_{unique_id}",
@@ -700,98 +854,29 @@ def test_connect_with_storage_options(self):
         }
 
         # This should work without errors
-        ns = connect("dir", properties)
-        assert isinstance(ns, lance.namespace.DirectoryNamespace)
-
+        inner_ns_client = connect("dir", properties)
+        assert isinstance(inner_ns_client, lance.namespace.DirectoryNamespace)
 
-class TableVersionTrackingNamespace(lance.namespace.DirectoryNamespace):
-    """Namespace wrapper that tracks table version API calls.
+        # Wrap if testing CustomNamespace
+        ns_client = _wrap_if_custom(inner_ns_client, use_custom)
 
-    Similar to the Rust TrackingNamespace and Java TableVersionTrackingNamespace,
-    this extends DirectoryNamespace with table_version_tracking_enabled=true and
-    counts create_table_version and describe_table_version calls.
-
-    This class implements the JSON bridge methods that PyLanceNamespace calls,
-    allowing API call tracking to work even when the calls go through Rust.
-
-    Unlike a wrapper approach, this extends DirectoryNamespace directly so that
-    Rust can detect it as a DirectoryNamespace subclass and use the native handle.
-    """
-
-    def __init__(self, root: str):
-        dir_props = {
-            "root": root,
-            "table_version_tracking_enabled": "true",
-            "manifest_enabled": "true",
-        }
-        super().__init__(**dir_props)
-        self.create_table_version_count = 0
-        self.describe_table_version_count = 0
-        self.list_table_versions_count = 0
-        self._lock = Lock()
-
-    def namespace_id(self) -> str:
-        return f"TableVersionTrackingNamespace {{ inner: {super().namespace_id()} }}"
-
-    def create_table_version(
-        self, request: CreateTableVersionRequest
-    ) -> CreateTableVersionResponse:
-        with self._lock:
-            self.create_table_version_count += 1
-        return super().create_table_version(request)
+        # Verify it can be used for basic operations
+        create_ns_req = CreateNamespaceRequest(id=["test_ns"])
+        ns_client.create_namespace(create_ns_req)
 
-    def describe_table_version(
-        self, request: DescribeTableVersionRequest
-    ) -> DescribeTableVersionResponse:
-        with self._lock:
-            self.describe_table_version_count += 1
-        return super().describe_table_version(request)
 
-    def list_table_versions(
-        self, request: ListTableVersionsRequest
-    ) -> ListTableVersionsResponse:
-        with self._lock:
-            self.list_table_versions_count += 1
-        return super().list_table_versions(request)
-
-    # JSON bridge methods for Rust PyLanceNamespace callbacks
-    # These call the parent's _inner (PyDirectoryNamespace) directly with dict API
-    def describe_table_version_json(self, request_json: str) -> str:
-        """JSON bridge that increments counter before delegating."""
-        import json
-
-        with self._lock:
-            self.describe_table_version_count += 1
-        request_dict = json.loads(request_json)
-        response_dict = self._inner.describe_table_version(request_dict)
-        return json.dumps(response_dict)
-
-    def create_table_version_json(self, request_json: str) -> str:
-        """JSON bridge that increments counter before delegating."""
-        import json
-
-        with self._lock:
-            self.create_table_version_count += 1
-        request_dict = json.loads(request_json)
-        response_dict = self._inner.create_table_version(request_dict)
-        return json.dumps(response_dict)
-
-    def list_table_versions_json(self, request_json: str) -> str:
-        """JSON bridge that increments counter before delegating."""
-        import json
-
-        with self._lock:
-            self.list_table_versions_count += 1
-        request_dict = json.loads(request_json)
-        response_dict = self._inner.list_table_versions(request_dict)
-        return json.dumps(response_dict)
+def _get_ops_metric(ns_client, metric_name: str) -> int:
+    """Helper to get a specific metric count from namespace client ops metrics."""
+    metrics = ns_client.retrieve_ops_metrics()
+    return metrics.get(metric_name, 0)
 
 
 @pytest.mark.skipif(
     sys.platform == "win32",
     reason="External manifest store has known issues on Windows",
 )
-def test_external_manifest_store_invokes_namespace_apis():
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_external_manifest_store_invokes_namespace_apis(use_custom):
     """Test that namespace APIs are invoked correctly for managed versioning.
 
     This test mirrors:
@@ -802,72 +887,90 @@ def test_external_manifest_store_invokes_namespace_apis():
     1. list_table_versions is called when opening dataset (latest version)
     2. create_table_version is called exactly once during append
     3. describe_table_version is called when opening specific version
+
+    Uses native ops_metrics_enabled to track API calls instead of custom wrapper.
+    The test is parameterized to verify both DirectoryNamespace and CustomNamespace.
     """
     with tempfile.TemporaryDirectory() as tmpdir:
-        namespace = TableVersionTrackingNamespace(root=tmpdir)
+        # Use native namespace with ops metrics enabled
+        inner_ns_client = lance.namespace.DirectoryNamespace(
+            root=tmpdir,
+            table_version_tracking_enabled="true",
+            manifest_enabled="true",
+            ops_metrics_enabled="true",
+        )
+        ns_client = _wrap_if_custom(inner_ns_client, use_custom)
 
         # Create parent namespace first (like Rust/Java tests)
-        namespace.create_namespace(CreateNamespaceRequest(id=["workspace"]))
+        ns_client.create_namespace(CreateNamespaceRequest(id=["workspace"]))
 
         table_id = ["workspace", "test_table"]
 
         # Create initial table
         table1 = pa.Table.from_pylist([{"a": 1, "b": 2}, {"a": 10, "b": 20}])
         ds = lance.write_dataset(
-            table1, namespace=namespace, table_id=table_id, mode="create"
+            table1, namespace_client=ns_client, table_id=table_id, mode="create"
         )
         assert ds.count_rows() == 2
         assert len(ds.versions()) == 1
 
         # Verify describe_table returns managed_versioning=True
-        describe_resp = namespace.describe_table(DescribeTableRequest(id=table_id))
+        describe_resp = ns_client.describe_table(DescribeTableRequest(id=table_id))
         assert describe_resp.managed_versioning is True, (
             f"Expected managed_versioning=True, got {describe_resp.managed_versioning}"
         )
 
         # Open dataset through namespace - should call list_table_versions for latest
-        initial_list_count = namespace.list_table_versions_count
-        ds_from_namespace = lance.dataset(namespace=namespace, table_id=table_id)
-        assert ds_from_namespace.count_rows() == 2
-        assert ds_from_namespace.version == 1
-        assert namespace.list_table_versions_count == initial_list_count + 1, (
-            "list_table_versions should be called once when opening latest version"
-        )
+        # Use inner_ns_client for metrics since CustomNamespace delegates to it
+        initial_list_count = _get_ops_metric(inner_ns_client, "list_table_versions")
+        ds_from_ns_client = lance.dataset(namespace_client=ns_client, table_id=table_id)
+        assert ds_from_ns_client.count_rows() == 2
+        assert ds_from_ns_client.version == 1
+        assert (
+            _get_ops_metric(inner_ns_client, "list_table_versions")
+            == initial_list_count + 1
+        ), "list_table_versions should be called once when opening latest version"
 
         # Verify create_table_version was called once during CREATE
-        assert namespace.create_table_version_count == 1, (
+        assert _get_ops_metric(inner_ns_client, "create_table_version") == 1, (
             "create_table_version should have been called once during CREATE"
         )
 
         # Append data - should call create_table_version again
         table2 = pa.Table.from_pylist([{"a": 100, "b": 200}, {"a": 1000, "b": 2000}])
         ds = lance.write_dataset(
-            table2, namespace=namespace, table_id=table_id, mode="append"
+            table2, namespace_client=ns_client, table_id=table_id, mode="append"
         )
         assert ds.count_rows() == 4
         assert len(ds.versions()) == 2
 
-        assert namespace.create_table_version_count == 2, (
+        assert _get_ops_metric(inner_ns_client, "create_table_version") == 2, (
             "create_table_version should be called twice (CREATE + APPEND)"
         )
 
         # Open latest version - should call list_table_versions
-        list_count_before_latest = namespace.list_table_versions_count
-        latest_ds = lance.dataset(namespace=namespace, table_id=table_id)
+        list_count_before_latest = _get_ops_metric(
+            inner_ns_client, "list_table_versions"
+        )
+        latest_ds = lance.dataset(namespace_client=ns_client, table_id=table_id)
         assert latest_ds.count_rows() == 4
         assert latest_ds.version == 2
-        assert namespace.list_table_versions_count == list_count_before_latest + 1, (
-            "list_table_versions should be called once when opening latest version"
-        )
+        assert (
+            _get_ops_metric(inner_ns_client, "list_table_versions")
+            == list_count_before_latest + 1
+        ), "list_table_versions should be called once when opening latest version"
 
         # Open specific version (v1) - should call describe_table_version
-        describe_count_before_v1 = namespace.describe_table_version_count
-        v1_ds = lance.dataset(namespace=namespace, table_id=table_id, version=1)
+        describe_count_before_v1 = _get_ops_metric(
+            inner_ns_client, "describe_table_version"
+        )
+        v1_ds = lance.dataset(namespace_client=ns_client, table_id=table_id, version=1)
         assert v1_ds.count_rows() == 2
         assert v1_ds.version == 1
-        assert namespace.describe_table_version_count == describe_count_before_v1 + 1, (
-            "describe_table_version should be called once when opening version 1"
-        )
+        assert (
+            _get_ops_metric(inner_ns_client, "describe_table_version")
+            == describe_count_before_v1 + 1
+        ), "describe_table_version should be called once when opening version 1"
 
 
 @pytest.mark.skipif(
@@ -879,16 +982,18 @@ class TestConcurrentOperations:
 
     These tests mirror the Rust and Java concurrent tests to ensure
     the DirectoryNamespace handles concurrent create/drop operations correctly.
+    Tests are parameterized via temp_ns_client fixture to test both
+    DirectoryNamespace and CustomNamespace.
     """
 
-    def test_concurrent_create_and_drop_single_instance(self, temp_namespace):
+    def test_concurrent_create_and_drop_single_instance(self, temp_ns_client):
         """Test concurrent create/drop with single namespace instance."""
         import concurrent.futures
 
         # Initialize namespace first - create parent namespace to ensure __manifest
         # table is created before concurrent operations
         create_ns_req = CreateNamespaceRequest(id=["test_ns"])
-        temp_namespace.create_namespace(create_ns_req)
+        temp_ns_client.create_namespace(create_ns_req)
 
         num_tables = 10
         success_count = 0
@@ -905,11 +1010,11 @@ def create_and_drop_table(table_index):
 
                 # Create table
                 create_req = CreateTableRequest(id=table_id)
-                temp_namespace.create_table(create_req, ipc_data)
+                temp_ns_client.create_table(create_req, ipc_data)
 
                 # Drop table
                 drop_req = DropTableRequest(id=table_id)
-                temp_namespace.drop_table(drop_req)
+                temp_ns_client.drop_table(drop_req)
 
                 with lock:
                     success_count += 1
@@ -931,22 +1036,26 @@ def create_and_drop_table(table_index):
 
         # Verify all tables are dropped
         list_req = ListTablesRequest(id=["test_ns"])
-        response = temp_namespace.list_tables(list_req)
+        response = temp_ns_client.list_tables(list_req)
         assert len(response.tables) == 0, "All tables should be dropped"
 
-    def test_concurrent_create_and_drop_multiple_instances(self):
+    @pytest.mark.parametrize(
+        "use_custom", [False, True], ids=["DirectoryNS", "CustomNS"]
+    )
+    def test_concurrent_create_and_drop_multiple_instances(self, use_custom):
         """Test concurrent create/drop with multiple namespace instances."""
         import concurrent.futures
 
         with tempfile.TemporaryDirectory() as tmpdir:
             # Initialize namespace first with a single instance to ensure __manifest
             # table is created and parent namespace exists before concurrent operations
-            init_ns = connect(
+            init_ns_client = connect(
                 "dir",
                 {"root": f"file://{tmpdir}", "commit_retries": "2147483647"},
             )
+            init_ns_client = _wrap_if_custom(init_ns_client, use_custom)
             create_ns_req = CreateNamespaceRequest(id=["test_ns"])
-            init_ns.create_namespace(create_ns_req)
+            init_ns_client.create_namespace(create_ns_req)
 
             num_tables = 10
             success_count = 0
@@ -956,12 +1065,13 @@ def test_concurrent_create_and_drop_multiple_instances(self):
             def create_and_drop_table(table_index):
                 nonlocal success_count, fail_count
                 try:
-                    # Each thread creates its own namespace instance
+                    # Each thread creates its own namespace client instance
                     # Use high commit_retries to handle version collisions
-                    ns = connect(
+                    local_ns_client = connect(
                         "dir",
                         {"root": f"file://{tmpdir}", "commit_retries": "2147483647"},
                     )
+                    local_ns_client = _wrap_if_custom(local_ns_client, use_custom)
 
                     table_name = f"multi_ns_table_{table_index}"
                     table_id = ["test_ns", table_name]
@@ -970,11 +1080,11 @@ def create_and_drop_table(table_index):
 
                     # Create table
                     create_req = CreateTableRequest(id=table_id)
-                    ns.create_table(create_req, ipc_data)
+                    local_ns_client.create_table(create_req, ipc_data)
 
                     # Drop table
                     drop_req = DropTableRequest(id=table_id)
-                    ns.drop_table(drop_req)
+                    local_ns_client.drop_table(drop_req)
 
                     with lock:
                         success_count += 1
@@ -996,27 +1106,32 @@ def create_and_drop_table(table_index):
             )
             assert fail_count == 0, f"Expected 0 failures, got {fail_count}"
 
-            # Verify with a fresh namespace instance
-            verify_ns = connect(
+            # Verify with a fresh namespace client instance
+            verify_ns_client = connect(
                 "dir", {"root": f"file://{tmpdir}", "commit_retries": "2147483647"}
             )
+            verify_ns_client = _wrap_if_custom(verify_ns_client, use_custom)
             list_req = ListTablesRequest(id=["test_ns"])
-            response = verify_ns.list_tables(list_req)
+            response = verify_ns_client.list_tables(list_req)
             assert len(response.tables) == 0, "All tables should be dropped"
 
-    def test_concurrent_create_then_drop_from_different_instance(self):
+    @pytest.mark.parametrize(
+        "use_custom", [False, True], ids=["DirectoryNS", "CustomNS"]
+    )
+    def test_concurrent_create_then_drop_from_different_instance(self, use_custom):
         """Test creating from one set of instances, dropping from different ones."""
         import concurrent.futures
 
         with tempfile.TemporaryDirectory() as tmpdir:
             # Initialize namespace first with a single instance to ensure __manifest
             # table is created and parent namespace exists before concurrent operations
-            init_ns = connect(
+            init_ns_client = connect(
                 "dir",
                 {"root": f"file://{tmpdir}", "commit_retries": "2147483647"},
             )
+            init_ns_client = _wrap_if_custom(init_ns_client, use_custom)
             create_ns_req = CreateNamespaceRequest(id=["test_ns"])
-            init_ns.create_namespace(create_ns_req)
+            init_ns_client.create_namespace(create_ns_req)
 
             num_tables = 10
 
@@ -1029,10 +1144,11 @@ def create_table(table_index):
                 nonlocal create_success_count, create_fail_count
                 try:
                     # Use high commit_retries to handle version collisions
-                    ns = connect(
+                    local_ns_client = connect(
                         "dir",
                         {"root": f"file://{tmpdir}", "commit_retries": "2147483647"},
                     )
+                    local_ns_client = _wrap_if_custom(local_ns_client, use_custom)
 
                     table_name = f"cross_instance_table_{table_index}"
                     table_id = ["test_ns", table_name]
@@ -1040,7 +1156,7 @@ def create_table(table_index):
                     ipc_data = table_to_ipc_bytes(table_data)
 
                     create_req = CreateTableRequest(id=table_id)
-                    ns.create_table(create_req, ipc_data)
+                    local_ns_client.create_table(create_req, ipc_data)
 
                     with create_lock:
                         create_success_count += 1
@@ -1068,16 +1184,17 @@ def drop_table(table_index):
                 nonlocal drop_success_count, drop_fail_count
                 try:
                     # Use high commit_retries to handle version collisions
-                    ns = connect(
+                    local_ns_client = connect(
                         "dir",
                         {"root": f"file://{tmpdir}", "commit_retries": "2147483647"},
                     )
+                    local_ns_client = _wrap_if_custom(local_ns_client, use_custom)
 
                     table_name = f"cross_instance_table_{table_index}"
                     table_id = ["test_ns", table_name]
 
                     drop_req = DropTableRequest(id=table_id)
-                    ns.drop_table(drop_req)
+                    local_ns_client.drop_table(drop_req)
 
                     with drop_lock:
                         drop_success_count += 1
@@ -1098,9 +1215,304 @@ def drop_table(table_index):
             assert drop_fail_count == 0, f"No drops should fail, got {drop_fail_count}"
 
             # Verify all tables are dropped
-            verify_ns = connect(
+            verify_ns_client = connect(
                 "dir", {"root": f"file://{tmpdir}", "commit_retries": "2147483647"}
             )
+            verify_ns_client = _wrap_if_custom(verify_ns_client, use_custom)
             list_req = ListTablesRequest(id=["test_ns"])
-            response = verify_ns.list_tables(list_req)
+            response = verify_ns_client.list_tables(list_req)
             assert len(response.tables) == 0, "All tables should be dropped"
+
+
+class TestDataManipulation:
+    """Tests for data manipulation operations."""
+
+    def test_count_table_rows(self, temp_ns_client):
+        """Test counting rows in a table."""
+        # Create namespace and table
+        create_ns_req = CreateNamespaceRequest(id=["workspace"])
+        temp_ns_client.create_namespace(create_ns_req)
+
+        table_data = create_test_data()  # 3 rows
+        ipc_data = table_to_ipc_bytes(table_data)
+        create_req = CreateTableRequest(id=["workspace", "test_table"])
+        temp_ns_client.create_table(create_req, ipc_data)
+
+        # Count rows
+        count_req = CountTableRowsRequest(id=["workspace", "test_table"])
+        count = temp_ns_client.count_table_rows(count_req)
+        assert count == 3
+
+    def test_count_table_rows_with_filter(self, temp_ns_client):
+        """Test counting rows with a filter predicate."""
+        # Create namespace and table
+        create_ns_req = CreateNamespaceRequest(id=["workspace"])
+        temp_ns_client.create_namespace(create_ns_req)
+
+        table_data = create_test_data()  # 3 rows with ages 30, 25, 35
+        ipc_data = table_to_ipc_bytes(table_data)
+        create_req = CreateTableRequest(id=["workspace", "test_table"])
+        temp_ns_client.create_table(create_req, ipc_data)
+
+        # Count rows with predicate
+        count_req = CountTableRowsRequest(
+            id=["workspace", "test_table"], predicate="age > 28"
+        )
+        count = temp_ns_client.count_table_rows(count_req)
+        assert count == 2  # Alice (30) and Charlie (35)
+
+    def test_insert_into_table(self, temp_ns_client):
+        """Test inserting data into a table."""
+        # Create namespace and table
+        create_ns_req = CreateNamespaceRequest(id=["workspace"])
+        temp_ns_client.create_namespace(create_ns_req)
+
+        table_data = create_test_data()  # 3 rows
+        ipc_data = table_to_ipc_bytes(table_data)
+        create_req = CreateTableRequest(id=["workspace", "test_table"])
+        temp_ns_client.create_table(create_req, ipc_data)
+
+        # Insert more data
+        new_data = pa.Table.from_pylist(
+            [
+                {"id": 4, "name": "David", "age": 40},
+                {"id": 5, "name": "Eve", "age": 22},
+            ]
+        )
+        new_ipc_data = table_to_ipc_bytes(new_data)
+        insert_req = InsertIntoTableRequest(
+            id=["workspace", "test_table"], mode="append"
+        )
+        response = temp_ns_client.insert_into_table(insert_req, new_ipc_data)
+        assert response is not None
+
+        # Verify row count increased
+        count_req = CountTableRowsRequest(id=["workspace", "test_table"])
+        count = temp_ns_client.count_table_rows(count_req)
+        assert count == 5
+
+    def test_query_table(self, temp_ns_client):
+        """Test querying a table."""
+        # Create namespace and table
+        create_ns_req = CreateNamespaceRequest(id=["workspace"])
+        temp_ns_client.create_namespace(create_ns_req)
+
+        table_data = create_test_data()  # 3 rows
+        ipc_data = table_to_ipc_bytes(table_data)
+        create_req = CreateTableRequest(id=["workspace", "test_table"])
+        temp_ns_client.create_table(create_req, ipc_data)
+
+        # Query table with empty vector (for non-vector queries)
+        query_req = QueryTableRequest(id=["workspace", "test_table"], k=10, vector={})
+        result_bytes = temp_ns_client.query_table(query_req)
+        assert result_bytes is not None
+        assert len(result_bytes) > 0
+
+        # Parse the result
+        reader = pa.ipc.open_file(pa.BufferReader(result_bytes))
+        result_table = reader.read_all()
+        assert result_table.num_rows == 3
+        assert "id" in result_table.column_names
+        assert "name" in result_table.column_names
+
+    def test_query_table_with_filter(self, temp_ns_client):
+        """Test querying a table with a filter."""
+        # Create namespace and table
+        create_ns_req = CreateNamespaceRequest(id=["workspace"])
+        temp_ns_client.create_namespace(create_ns_req)
+
+        table_data = create_test_data()  # 3 rows
+        ipc_data = table_to_ipc_bytes(table_data)
+        create_req = CreateTableRequest(id=["workspace", "test_table"])
+        temp_ns_client.create_table(create_req, ipc_data)
+
+        # Query with filter and empty vector
+        query_req = QueryTableRequest(
+            id=["workspace", "test_table"], filter="age >= 30", k=10, vector={}
+        )
+        result_bytes = temp_ns_client.query_table(query_req)
+        reader = pa.ipc.open_file(pa.BufferReader(result_bytes))
+        result_table = reader.read_all()
+        assert result_table.num_rows == 2  # Alice and Charlie
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32",
+    reason="Table version listing not supported on Windows",
+)
+class TestTableVersions:
+    """Tests for table version operations."""
+
+    def test_list_table_versions(self, temp_ns_client):
+        """Test listing table versions."""
+        # Create namespace and table
+        create_ns_req = CreateNamespaceRequest(id=["workspace"])
+        temp_ns_client.create_namespace(create_ns_req)
+
+        table_data = create_test_data()
+        ipc_data = table_to_ipc_bytes(table_data)
+        create_req = CreateTableRequest(id=["workspace", "test_table"])
+        temp_ns_client.create_table(create_req, ipc_data)
+
+        # List versions
+        list_req = ListTableVersionsRequest(id=["workspace", "test_table"])
+        response = temp_ns_client.list_table_versions(list_req)
+        assert response is not None
+        assert len(response.versions) >= 1
+
+    def test_describe_table_version(self, temp_ns_client):
+        """Test describing a specific table version."""
+        # Create namespace and table
+        create_ns_req = CreateNamespaceRequest(id=["workspace"])
+        temp_ns_client.create_namespace(create_ns_req)
+
+        table_data = create_test_data()
+        ipc_data = table_to_ipc_bytes(table_data)
+        create_req = CreateTableRequest(id=["workspace", "test_table"])
+        temp_ns_client.create_table(create_req, ipc_data)
+
+        # Describe version 1
+        describe_req = DescribeTableVersionRequest(
+            id=["workspace", "test_table"], version=1
+        )
+        response = temp_ns_client.describe_table_version(describe_req.model_dump())
+        assert response is not None
+        assert response.get("version") is not None
+
+    def test_multiple_versions_via_insert(self, temp_ns_client):
+        """Test that inserts create new versions."""
+        # Create namespace and table
+        create_ns_req = CreateNamespaceRequest(id=["workspace"])
+        temp_ns_client.create_namespace(create_ns_req)
+
+        table_data = create_test_data()
+        ipc_data = table_to_ipc_bytes(table_data)
+        create_req = CreateTableRequest(id=["workspace", "test_table"])
+        temp_ns_client.create_table(create_req, ipc_data)
+
+        # Insert more data to create version 2
+        new_data = pa.Table.from_pylist([{"id": 4, "name": "David", "age": 40}])
+        new_ipc_data = table_to_ipc_bytes(new_data)
+        insert_req = InsertIntoTableRequest(
+            id=["workspace", "test_table"], mode="append"
+        )
+        temp_ns_client.insert_into_table(insert_req, new_ipc_data)
+
+        # List versions - should have at least 2
+        list_req = ListTableVersionsRequest(id=["workspace", "test_table"])
+        response = temp_ns_client.list_table_versions(list_req)
+        assert len(response.versions) >= 2
+
+
+class TestIndexOperations:
+    """Tests for index operations."""
+
+    def test_list_indices_empty(self, temp_ns_client):
+        """Test listing indices on a table with no indices."""
+        # Create table with a vector column
+        import numpy as np
+
+        vector_data = pa.Table.from_pydict(
+            {
+                "id": [1, 2, 3],
+                "vector": pa.FixedSizeListArray.from_arrays(
+                    pa.array(np.random.rand(12).astype(np.float32)), 4
+                ),
+            }
+        )
+        ipc_data = table_to_ipc_bytes(vector_data)
+        create_req = CreateTableRequest(id=["vector_table"])
+        temp_ns_client.create_table(create_req, ipc_data)
+
+        # List indices (should be empty initially)
+        list_req = ListTableIndicesRequest(id=["vector_table"])
+        response = temp_ns_client.list_table_indices(list_req)
+        assert response is not None
+        # Initially no indices
+        assert len(response.indexes) == 0
+
+    def test_describe_table_index_stats(self, memory_ns_client):
+        """Test describing index stats (even when no index exists)."""
+        # Create namespace and table
+        create_ns_req = CreateNamespaceRequest(id=["workspace"])
+        memory_ns_client.create_namespace(create_ns_req)
+
+        table_data = create_test_data()
+        ipc_data = table_to_ipc_bytes(table_data)
+        create_req = CreateTableRequest(id=["workspace", "test_table"])
+        memory_ns_client.create_table(create_req, ipc_data)
+
+        # Describe index stats for non-existent index should return empty/null stats
+        describe_req = DescribeTableIndexStatsRequest(
+            id=["workspace", "test_table"], index_name="nonexistent"
+        )
+        # This may raise an error or return empty stats depending on implementation
+        try:
+            response = memory_ns_client.describe_table_index_stats(describe_req)
+            # If it succeeds, verify response structure
+            assert response is not None
+        except Exception:
+            # Expected if index doesn't exist
+            pass
+
+    def test_create_scalar_index(self, temp_ns_client):
+        """Test creating a scalar index."""
+        # Create table at root level (without namespace)
+        table_data = create_test_data()
+        ipc_data = table_to_ipc_bytes(table_data)
+        create_req = CreateTableRequest(id=["test_table"])
+        temp_ns_client.create_table(create_req, ipc_data)
+
+        # Create scalar index on 'id' column
+        create_index_req = CreateTableIndexRequest(
+            id=["test_table"],
+            column="id",
+            index_type="BTREE",
+            name="id_idx",
+        )
+        response = temp_ns_client.create_table_index(create_index_req)
+        assert response is not None
+
+        # List indices to verify
+        list_req = ListTableIndicesRequest(id=["test_table"])
+        list_response = temp_ns_client.list_table_indices(list_req)
+        assert len(list_response.indexes) == 1
+        assert list_response.indexes[0].index_name == "id_idx"
+        assert list_response.indexes[0].columns == ["id"]
+
+    def test_create_vector_index(self, temp_ns_client):
+        """Test creating a vector index."""
+        import numpy as np
+
+        # Create table with 256 rows of 8-dimensional vectors (enough for IVF)
+        num_rows = 256
+        dim = 8
+        vector_data = pa.Table.from_pydict(
+            {
+                "id": list(range(num_rows)),
+                "vector": pa.FixedSizeListArray.from_arrays(
+                    pa.array(np.random.rand(num_rows * dim).astype(np.float32)), dim
+                ),
+            }
+        )
+        ipc_data = table_to_ipc_bytes(vector_data)
+        create_req = CreateTableRequest(id=["vector_table"])
+        temp_ns_client.create_table(create_req, ipc_data)
+
+        # Create vector index using IVF_FLAT
+        create_index_req = CreateTableIndexRequest(
+            id=["vector_table"],
+            column="vector",
+            index_type="IVF_FLAT",
+            name="vector_idx",
+            distance_type="l2",
+        )
+        response = temp_ns_client.create_table_index(create_index_req)
+        assert response is not None
+
+        # List indices to verify
+        list_req = ListTableIndicesRequest(id=["vector_table"])
+        list_response = temp_ns_client.list_table_indices(list_req)
+        assert len(list_response.indexes) == 1
+        assert list_response.indexes[0].index_name == "vector_idx"
+        assert list_response.indexes[0].columns == ["vector"]
diff --git a/python/python/tests/test_namespace_integration.py b/python/python/tests/test_namespace_integration.py
index 14dd7a6531f..4605b755816 100644
--- a/python/python/tests/test_namespace_integration.py
+++ b/python/python/tests/test_namespace_integration.py
@@ -4,8 +4,11 @@
 """
 Integration tests for Lance Namespace with S3 and credential refresh.
 
-This test simulates a namespace server that returns incrementing credentials
-and verifies that the credential refresh mechanism works correctly.
+This test uses DirectoryNamespace with native ops_metrics and vend_input_storage_options
+features to track API calls and test credential refresh mechanisms.
+
+Tests are parameterized to run with both DirectoryNamespace and a CustomNamespace
+wrapper to verify Python-Rust binding works correctly for custom implementations.
 
 See DEVELOPMENT.md under heading "Integration Tests" for more information.
 """
@@ -14,18 +17,135 @@
 import time
 import uuid
 from threading import Lock
-from typing import Dict
+from typing import Dict, Optional
 
 import lance
 import pyarrow as pa
 import pytest
 from lance.namespace import (
     DeclareTableRequest,
-    DeclareTableResponse,
     DescribeTableRequest,
-    DescribeTableResponse,
+    DirectoryNamespace,
     LanceNamespace,
 )
+from lance_namespace import (
+    CreateNamespaceRequest,
+    CreateNamespaceResponse,
+    CreateTableRequest,
+    CreateTableResponse,
+    CreateTableVersionRequest,
+    CreateTableVersionResponse,
+    DeclareTableResponse,
+    DeregisterTableRequest,
+    DeregisterTableResponse,
+    DescribeNamespaceRequest,
+    DescribeNamespaceResponse,
+    DescribeTableResponse,
+    DescribeTableVersionRequest,
+    DescribeTableVersionResponse,
+    DropNamespaceRequest,
+    DropNamespaceResponse,
+    DropTableRequest,
+    DropTableResponse,
+    ListNamespacesRequest,
+    ListNamespacesResponse,
+    ListTablesRequest,
+    ListTablesResponse,
+    ListTableVersionsRequest,
+    ListTableVersionsResponse,
+    NamespaceExistsRequest,
+    RegisterTableRequest,
+    RegisterTableResponse,
+    TableExistsRequest,
+)
+
+
+class CustomNamespace(LanceNamespace):
+    """A custom namespace wrapper that delegates to DirectoryNamespace.
+
+    This class verifies that the Python-Rust binding works correctly for
+    custom namespace implementations that wrap the native DirectoryNamespace.
+    All methods simply delegate to the underlying DirectoryNamespace instance.
+    """
+
+    def __init__(self, inner: DirectoryNamespace):
+        self._inner = inner
+
+    def namespace_id(self) -> str:
+        return f"CustomNamespace[{self._inner.namespace_id()}]"
+
+    def create_namespace(
+        self, request: CreateNamespaceRequest
+    ) -> CreateNamespaceResponse:
+        return self._inner.create_namespace(request)
+
+    def describe_namespace(
+        self, request: DescribeNamespaceRequest
+    ) -> DescribeNamespaceResponse:
+        return self._inner.describe_namespace(request)
+
+    def namespace_exists(self, request: NamespaceExistsRequest) -> None:
+        return self._inner.namespace_exists(request)
+
+    def drop_namespace(self, request: DropNamespaceRequest) -> DropNamespaceResponse:
+        return self._inner.drop_namespace(request)
+
+    def list_namespaces(self, request: ListNamespacesRequest) -> ListNamespacesResponse:
+        return self._inner.list_namespaces(request)
+
+    def create_table(
+        self, request: CreateTableRequest, data: bytes
+    ) -> CreateTableResponse:
+        return self._inner.create_table(request, data)
+
+    def declare_table(self, request: DeclareTableRequest) -> DeclareTableResponse:
+        return self._inner.declare_table(request)
+
+    def describe_table(self, request: DescribeTableRequest) -> DescribeTableResponse:
+        return self._inner.describe_table(request)
+
+    def table_exists(self, request: TableExistsRequest) -> None:
+        return self._inner.table_exists(request)
+
+    def drop_table(self, request: DropTableRequest) -> DropTableResponse:
+        return self._inner.drop_table(request)
+
+    def list_tables(self, request: ListTablesRequest) -> ListTablesResponse:
+        return self._inner.list_tables(request)
+
+    def register_table(self, request: RegisterTableRequest) -> RegisterTableResponse:
+        return self._inner.register_table(request)
+
+    def deregister_table(
+        self, request: DeregisterTableRequest
+    ) -> DeregisterTableResponse:
+        return self._inner.deregister_table(request)
+
+    def list_table_versions(
+        self, request: ListTableVersionsRequest
+    ) -> ListTableVersionsResponse:
+        return self._inner.list_table_versions(request)
+
+    def describe_table_version(
+        self, request: DescribeTableVersionRequest
+    ) -> DescribeTableVersionResponse:
+        return self._inner.describe_table_version(request)
+
+    def create_table_version(
+        self, request: CreateTableVersionRequest
+    ) -> CreateTableVersionResponse:
+        return self._inner.create_table_version(request)
+
+    def retrieve_ops_metrics(self) -> Optional[Dict[str, int]]:
+        return self._inner.retrieve_ops_metrics()
+
+
+def _wrap_if_custom(ns_client: DirectoryNamespace, use_custom: bool):
+    """Wrap namespace client in CustomNamespace if use_custom is True."""
+    if use_custom:
+        return CustomNamespace(ns_client)
+    return ns_client
+
 
 # These are all keys that are accepted by storage_options
 CONFIG = {
@@ -74,250 +194,256 @@ def delete_bucket(s3, bucket_name):
         pass
 
 
-class TrackingNamespace(LanceNamespace):
-    """Mock namespace that wraps DirectoryNamespace and tracks API calls."""
-
-    def __init__(
-        self,
-        bucket_name: str,
-        storage_options: Dict[str, str],
-        credential_expires_in_seconds: int = 60,
-    ):
-        from lance.namespace import DirectoryNamespace
-
-        self.bucket_name = bucket_name
-        self.base_storage_options = storage_options
-        self.credential_expires_in_seconds = credential_expires_in_seconds
-        self.describe_call_count = 0
-        self.create_call_count = 0
-        self.lock = Lock()
-
-        # Create underlying DirectoryNamespace with storage options
-        dir_props = {f"storage.{k}": v for k, v in storage_options.items()}
-
-        if bucket_name.startswith("/") or bucket_name.startswith("file://"):
-            dir_props["root"] = f"{bucket_name}/namespace_root"
-        else:
-            dir_props["root"] = f"s3://{bucket_name}/namespace_root"
-
-        self.inner = DirectoryNamespace(**dir_props)
-
-    def get_describe_call_count(self) -> int:
-        with self.lock:
-            return self.describe_call_count
+def create_tracking_namespace(
+    bucket_name: str,
+    storage_options: dict,
+    credential_expires_in_seconds: int = 60,
+    use_custom: bool = False,
+):
+    """Create a DirectoryNamespace with ops metrics and credential vending enabled.
+
+    Uses native DirectoryNamespace features:
+    - ops_metrics_enabled=true: Tracks API call counts via retrieve_ops_metrics()
+    - vend_input_storage_options=true: Returns input storage options in responses
+    - vend_input_storage_options_refresh_interval_millis: Adds expires_at_millis
+
+    Args:
+        bucket_name: S3 bucket name or local path
+        storage_options: Storage options to pass through (credentials, endpoint, etc.)
+        credential_expires_in_seconds: Interval in seconds for credential expiration
+        use_custom: If True, wrap in CustomNamespace for testing custom implementations
+
+    Returns:
+        Tuple of (namespace_client, inner_namespace_client) where inner is always
+        the DirectoryNamespace (used for metrics retrieval)
+    """
+    # Add refresh_offset_millis to storage options so that credentials are not
+    # considered expired immediately. Set to 1 second (1000ms) so that refresh
+    # checks work correctly with short-lived credentials in tests.
+    storage_options_with_refresh = dict(storage_options)
+    storage_options_with_refresh["refresh_offset_millis"] = "1000"
 
-    def get_create_call_count(self) -> int:
-        with self.lock:
-            return self.create_call_count
+    dir_props = {f"storage.{k}": v for k, v in storage_options_with_refresh.items()}
 
-    def namespace_id(self) -> str:
-        return f"TrackingNamespace {{ inner: {self.inner.namespace_id()} }}"
-
-    def _modify_storage_options(
-        self, storage_options: Dict[str, str], count: int
-    ) -> Dict[str, str]:
-        """Add incrementing credentials with expiration timestamp."""
-        modified = copy.deepcopy(storage_options) if storage_options else {}
-
-        modified["aws_access_key_id"] = f"AKID_{count}"
-        modified["aws_secret_access_key"] = f"SECRET_{count}"
-        modified["aws_session_token"] = f"TOKEN_{count}"
-        expires_at_millis = int(
-            (time.time() + self.credential_expires_in_seconds) * 1000
-        )
-        modified["expires_at_millis"] = str(expires_at_millis)
-        # Set refresh offset to 1 second (1000ms) for short-lived credential tests
-        modified["refresh_offset_millis"] = "1000"
-
-        return modified
+    if bucket_name.startswith("/") or bucket_name.startswith("file://"):
+        dir_props["root"] = f"{bucket_name}/namespace_root"
+    else:
+        dir_props["root"] = f"s3://{bucket_name}/namespace_root"
 
-    def declare_table(self, request: DeclareTableRequest) -> DeclareTableResponse:
-        with self.lock:
-            self.create_call_count += 1
-            count = self.create_call_count
+    # Enable ops metrics tracking
+    dir_props["ops_metrics_enabled"] = "true"
+    # Enable storage options vending
+    dir_props["vend_input_storage_options"] = "true"
+    # Set refresh interval in milliseconds
+    dir_props["vend_input_storage_options_refresh_interval_millis"] = str(
+        credential_expires_in_seconds * 1000
+    )
 
-        response = self.inner.declare_table(request)
-        response.storage_options = self._modify_storage_options(
-            response.storage_options, count
-        )
+    inner_ns_client = DirectoryNamespace(**dir_props)
+    ns_client = _wrap_if_custom(inner_ns_client, use_custom)
+    return ns_client, inner_ns_client
 
-        return response
 
-    def describe_table(self, request: DescribeTableRequest) -> DescribeTableResponse:
-        with self.lock:
-            self.describe_call_count += 1
-            count = self.describe_call_count
+def get_describe_call_count(namespace_client) -> int:
+    """Get the number of describe_table calls made to the namespace client."""
+    return namespace_client.retrieve_ops_metrics().get("describe_table", 0)
 
-        response = self.inner.describe_table(request)
-        response.storage_options = self._modify_storage_options(
-            response.storage_options, count
-        )
 
-        return response
+def get_declare_call_count(namespace_client) -> int:
+    """Get the number of declare_table calls made to the namespace client."""
+    return namespace_client.retrieve_ops_metrics().get("declare_table", 0)
 
 
 @pytest.mark.integration
-def test_namespace_open_dataset(s3_bucket: str):
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_namespace_open_dataset(s3_bucket: str, use_custom: bool):
     """Test creating and opening datasets through namespace with credential tracking."""
     storage_options = copy.deepcopy(CONFIG)
 
-    namespace = TrackingNamespace(
+    ns_client, inner_ns_client = create_tracking_namespace(
         bucket_name=s3_bucket,
         storage_options=storage_options,
         credential_expires_in_seconds=3600,
+        use_custom=use_custom,
     )
 
     table1 = pa.Table.from_pylist([{"a": 1, "b": 2}, {"a": 10, "b": 20}])
     table_name = uuid.uuid4().hex
     table_id = ["test_ns", table_name]
 
-    assert namespace.get_create_call_count() == 0
-    assert namespace.get_describe_call_count() == 0
+    assert get_declare_call_count(inner_ns_client) == 0
+    assert get_describe_call_count(inner_ns_client) == 0
 
     ds = lance.write_dataset(
-        table1, namespace=namespace, table_id=table_id, mode="create"
+        table1,
+        namespace_client=ns_client,
+        table_id=table_id,
+        mode="create",
+        storage_options=storage_options,
     )
     assert len(ds.versions()) == 1
     assert ds.count_rows() == 2
-    assert namespace.get_create_call_count() == 1
+    assert get_declare_call_count(inner_ns_client) == 1
 
-    ds_from_namespace = lance.dataset(
-        namespace=namespace,
+    ds_from_ns_client = lance.dataset(
+        namespace_client=ns_client,
         table_id=table_id,
+        storage_options=storage_options,
     )
 
-    assert namespace.get_describe_call_count() == 1
-    assert ds_from_namespace.count_rows() == 2
-    result = ds_from_namespace.to_table()
+    # 1 describe call from lance.dataset() to get location
+    assert get_describe_call_count(inner_ns_client) == 1
+    assert ds_from_ns_client.count_rows() == 2
+    result = ds_from_ns_client.to_table()
     assert result == table1
 
     # Test credential caching
-    call_count_before_reads = namespace.get_describe_call_count()
+    call_count_before_reads = get_describe_call_count(inner_ns_client)
     for _ in range(3):
-        assert ds_from_namespace.count_rows() == 2
-    assert namespace.get_describe_call_count() == call_count_before_reads
+        assert ds_from_ns_client.count_rows() == 2
+    assert get_describe_call_count(inner_ns_client) == call_count_before_reads
 
 
 @pytest.mark.integration
-def test_namespace_with_refresh(s3_bucket: str):
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_namespace_with_refresh(s3_bucket: str, use_custom: bool):
     """Test credential refresh when credentials expire."""
     storage_options = copy.deepcopy(CONFIG)
 
-    namespace = TrackingNamespace(
+    ns_client, inner_ns_client = create_tracking_namespace(
         bucket_name=s3_bucket,
         storage_options=storage_options,
         credential_expires_in_seconds=3,
+        use_custom=use_custom,
     )
 
     table1 = pa.Table.from_pylist([{"a": 1, "b": 2}, {"a": 10, "b": 20}])
     table_name = uuid.uuid4().hex
     table_id = ["test_ns", table_name]
 
-    assert namespace.get_create_call_count() == 0
-    assert namespace.get_describe_call_count() == 0
+    assert get_declare_call_count(inner_ns_client) == 0
+    assert get_describe_call_count(inner_ns_client) == 0
 
     ds = lance.write_dataset(
         table1,
-        namespace=namespace,
+        namespace_client=ns_client,
         table_id=table_id,
         mode="create",
+        storage_options=storage_options,
     )
     assert ds.count_rows() == 2
-    assert namespace.get_create_call_count() == 1
+    assert get_declare_call_count(inner_ns_client) == 1
 
-    ds_from_namespace = lance.dataset(
-        namespace=namespace,
+    ds_from_ns_client = lance.dataset(
+        namespace_client=ns_client,
         table_id=table_id,
+        storage_options=storage_options,
     )
 
-    initial_call_count = namespace.get_describe_call_count()
+    # 1 describe call from lance.dataset() to get location
+    initial_call_count = get_describe_call_count(inner_ns_client)
     assert initial_call_count == 1
-    assert ds_from_namespace.count_rows() == 2
-    result = ds_from_namespace.to_table()
+    assert ds_from_ns_client.count_rows() == 2
+    result = ds_from_ns_client.to_table()
     assert result == table1
 
-    call_count_after_initial_reads = namespace.get_describe_call_count()
+    call_count_after_initial_reads = get_describe_call_count(inner_ns_client)
 
     time.sleep(5)
 
-    assert ds_from_namespace.count_rows() == 2
-    result2 = ds_from_namespace.to_table()
+    assert ds_from_ns_client.count_rows() == 2
+    result2 = ds_from_ns_client.to_table()
     assert result2 == table1
 
-    final_call_count = namespace.get_describe_call_count()
+    final_call_count = get_describe_call_count(inner_ns_client)
     assert final_call_count == call_count_after_initial_reads + 1
 
 
 @pytest.mark.integration
-def test_namespace_append_through_namespace(s3_bucket: str):
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_namespace_append_through_namespace(s3_bucket: str, use_custom: bool):
     """Test appending to dataset through namespace."""
     storage_options = copy.deepcopy(CONFIG)
 
-    namespace = TrackingNamespace(
+    ns_client, inner_ns_client = create_tracking_namespace(
         bucket_name=s3_bucket,
         storage_options=storage_options,
         credential_expires_in_seconds=3600,
+        use_custom=use_custom,
     )
 
     table1 = pa.Table.from_pylist([{"a": 1, "b": 2}])
     table_name = uuid.uuid4().hex
     table_id = ["test_ns", table_name]
 
-    assert namespace.get_create_call_count() == 0
-    assert namespace.get_describe_call_count() == 0
+    assert get_declare_call_count(inner_ns_client) == 0
+    assert get_describe_call_count(inner_ns_client) == 0
 
     ds = lance.write_dataset(
-        table1, namespace=namespace, table_id=table_id, mode="create"
+        table1,
+        namespace_client=ns_client,
+        table_id=table_id,
+        mode="create",
+        storage_options=storage_options,
     )
     assert ds.count_rows() == 1
     assert len(ds.versions()) == 1
-    assert namespace.get_create_call_count() == 1
-    initial_describe_count = namespace.get_describe_call_count()
+    assert get_declare_call_count(inner_ns_client) == 1
+    initial_describe_count = get_describe_call_count(inner_ns_client)
 
     table2 = pa.Table.from_pylist([{"a": 10, "b": 20}])
     ds = lance.write_dataset(
-        table2, namespace=namespace, table_id=table_id, mode="append"
+        table2,
+        namespace_client=ns_client,
+        table_id=table_id,
+        mode="append",
+        storage_options=storage_options,
     )
     assert ds.count_rows() == 2
     assert len(ds.versions()) == 2
-    assert namespace.get_create_call_count() == 1
-    assert namespace.get_describe_call_count() == initial_describe_count + 1
+    assert get_declare_call_count(inner_ns_client) == 1
+    assert get_describe_call_count(inner_ns_client) == initial_describe_count + 1
 
-    ds_from_namespace = lance.dataset(
-        namespace=namespace,
+    ds_from_ns_client = lance.dataset(
+        namespace_client=ns_client,
         table_id=table_id,
+        storage_options=storage_options,
     )
 
-    assert ds_from_namespace.count_rows() == 2
-    assert len(ds_from_namespace.versions()) == 2
-    assert namespace.get_describe_call_count() == initial_describe_count + 2
+    assert ds_from_ns_client.count_rows() == 2
+    assert len(ds_from_ns_client.versions()) == 2
+    # +1 for describe from lance.dataset()
+    assert get_describe_call_count(inner_ns_client) == initial_describe_count + 2
 
 
 @pytest.mark.integration
-def test_namespace_write_create_mode(s3_bucket: str):
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_namespace_write_create_mode(s3_bucket: str, use_custom: bool):
     """Test writing dataset through namespace in CREATE mode."""
     storage_options = copy.deepcopy(CONFIG)
 
-    namespace = TrackingNamespace(
+    ns_client, inner_ns_client = create_tracking_namespace(
         bucket_name=s3_bucket,
         storage_options=storage_options,
         credential_expires_in_seconds=3600,
+        use_custom=use_custom,
     )
 
     table1 = pa.Table.from_pylist([{"a": 1, "b": 2}, {"a": 10, "b": 20}])
     table_name = uuid.uuid4().hex
 
-    assert namespace.get_create_call_count() == 0
-    assert namespace.get_describe_call_count() == 0
+    assert get_declare_call_count(inner_ns_client) == 0
+    assert get_describe_call_count(inner_ns_client) == 0
 
     ds = lance.write_dataset(
         table1,
-        namespace=namespace,
+        namespace_client=ns_client,
         table_id=["test_ns", table_name],
         mode="create",
+        storage_options=storage_options,
     )
 
-    assert namespace.get_create_call_count() == 1
+    assert get_declare_call_count(inner_ns_client) == 1
     assert ds.count_rows() == 2
     assert len(ds.versions()) == 1
     result = ds.to_table()
@@ -325,133 +451,143 @@ def test_namespace_write_create_mode(s3_bucket: str):
 
 
 @pytest.mark.integration
-def test_namespace_write_append_mode(s3_bucket: str):
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_namespace_write_append_mode(s3_bucket: str, use_custom: bool):
     """Test writing dataset through namespace in APPEND mode."""
     storage_options = copy.deepcopy(CONFIG)
 
-    namespace = TrackingNamespace(
+    ns_client, inner_ns_client = create_tracking_namespace(
         bucket_name=s3_bucket,
         storage_options=storage_options,
         credential_expires_in_seconds=3600,
+        use_custom=use_custom,
     )
 
     table1 = pa.Table.from_pylist([{"a": 1, "b": 2}])
     table_name = uuid.uuid4().hex
     table_id = ["test_ns", table_name]
 
-    assert namespace.get_create_call_count() == 0
-    assert namespace.get_describe_call_count() == 0
+    assert get_declare_call_count(inner_ns_client) == 0
+    assert get_describe_call_count(inner_ns_client) == 0
 
     ds = lance.write_dataset(
-        table1, namespace=namespace, table_id=table_id, mode="create"
+        table1,
+        namespace_client=ns_client,
+        table_id=table_id,
+        mode="create",
+        storage_options=storage_options,
     )
     assert ds.count_rows() == 1
-    assert namespace.get_create_call_count() == 1
-    assert namespace.get_describe_call_count() == 0
+    assert get_declare_call_count(inner_ns_client) == 1
+    assert get_describe_call_count(inner_ns_client) == 0
 
     table2 = pa.Table.from_pylist([{"a": 10, "b": 20}])
 
     ds = lance.write_dataset(
         table2,
-        namespace=namespace,
+        namespace_client=ns_client,
         table_id=table_id,
         mode="append",
+        storage_options=storage_options,
     )
 
-    assert namespace.get_create_call_count() == 1
-    describe_count_after_append = namespace.get_describe_call_count()
+    assert get_declare_call_count(inner_ns_client) == 1
+    describe_count_after_append = get_describe_call_count(inner_ns_client)
     assert describe_count_after_append == 1
     assert ds.count_rows() == 2
     assert len(ds.versions()) == 2
 
-    call_count_before_reads = namespace.get_describe_call_count()
+    call_count_before_reads = get_describe_call_count(inner_ns_client)
     for _ in range(3):
         assert ds.count_rows() == 2
-    assert namespace.get_describe_call_count() == call_count_before_reads
+    assert get_describe_call_count(inner_ns_client) == call_count_before_reads
 
 
 @pytest.mark.integration
-def test_namespace_write_overwrite_mode(s3_bucket: str):
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_namespace_write_overwrite_mode(s3_bucket: str, use_custom: bool):
     """Test writing dataset through namespace in OVERWRITE mode."""
     storage_options = copy.deepcopy(CONFIG)
 
-    namespace = TrackingNamespace(
+    ns_client, inner_ns_client = create_tracking_namespace(
         bucket_name=s3_bucket,
         storage_options=storage_options,
         credential_expires_in_seconds=3600,
+        use_custom=use_custom,
     )
 
     table1 = pa.Table.from_pylist([{"a": 1, "b": 2}])
     table_name = uuid.uuid4().hex
     table_id = ["test_ns", table_name]
 
-    assert namespace.get_create_call_count() == 0
-    assert namespace.get_describe_call_count() == 0
+    assert get_declare_call_count(inner_ns_client) == 0
+    assert get_describe_call_count(inner_ns_client) == 0
 
     ds = lance.write_dataset(
-        table1, namespace=namespace, table_id=table_id, mode="create"
+        table1,
+        namespace_client=ns_client,
+        table_id=table_id,
+        mode="create",
+        storage_options=storage_options,
     )
     assert ds.count_rows() == 1
-    assert namespace.get_create_call_count() == 1
-    assert namespace.get_describe_call_count() == 0
+    assert get_declare_call_count(inner_ns_client) == 1
+    assert get_describe_call_count(inner_ns_client) == 0
 
     table2 = pa.Table.from_pylist([{"a": 10, "b": 20}, {"a": 100, "b": 200}])
 
     ds = lance.write_dataset(
         table2,
-        namespace=namespace,
+        namespace_client=ns_client,
         table_id=table_id,
         mode="overwrite",
+        storage_options=storage_options,
     )
 
-    assert namespace.get_create_call_count() == 1
-    describe_count_after_overwrite = namespace.get_describe_call_count()
+    assert get_declare_call_count(inner_ns_client) == 1
+    describe_count_after_overwrite = get_describe_call_count(inner_ns_client)
     assert describe_count_after_overwrite == 1
     assert ds.count_rows() == 2
     assert len(ds.versions()) == 2
     result = ds.to_table()
     assert result == table2
 
-    call_count_before_reads = namespace.get_describe_call_count()
+    call_count_before_reads = get_describe_call_count(inner_ns_client)
     for _ in range(3):
         assert ds.count_rows() == 2
-    assert namespace.get_describe_call_count() == call_count_before_reads
+    assert get_describe_call_count(inner_ns_client) == call_count_before_reads
 
 
 @pytest.mark.integration
-def test_namespace_distributed_write(s3_bucket: str):
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_namespace_distributed_write(s3_bucket: str, use_custom: bool):
     """Test distributed write pattern through namespace."""
     storage_options = copy.deepcopy(CONFIG)
 
-    namespace = TrackingNamespace(
+    ns_client, inner_ns_client = create_tracking_namespace(
         bucket_name=s3_bucket,
         storage_options=storage_options,
         credential_expires_in_seconds=3600,
+        use_custom=use_custom,
     )
 
     table_name = uuid.uuid4().hex
     table_id = ["test_ns", table_name]
 
     request = DeclareTableRequest(id=table_id, location=None)
-    response = namespace.declare_table(request)
+    response = ns_client.declare_table(request)
 
-    assert namespace.get_create_call_count() == 1
-    assert namespace.get_describe_call_count() == 0
+    assert get_declare_call_count(inner_ns_client) == 1
+    assert get_describe_call_count(inner_ns_client) == 0
 
     table_uri = response.location
     assert table_uri is not None
 
-    from lance.namespace import LanceNamespaceStorageOptionsProvider
-
-    namespace_storage_options = response.storage_options
-    assert namespace_storage_options is not None
-
-    storage_options_provider = LanceNamespaceStorageOptionsProvider(
-        namespace=namespace, table_id=table_id
-    )
+    ns_client_storage_options = response.storage_options
+    assert ns_client_storage_options is not None
 
     merged_options = dict(storage_options)
-    merged_options.update(namespace_storage_options)
+    merged_options.update(ns_client_storage_options)
 
     from lance.fragment import write_fragments
 
@@ -460,7 +596,8 @@ def test_namespace_distributed_write(s3_bucket: str):
         fragment1_data,
         table_uri,
         storage_options=merged_options,
-        storage_options_provider=storage_options_provider,
+        namespace_client=ns_client,
+        table_id=table_id,
     )
 
     fragment2_data = pa.Table.from_pylist([{"a": 10, "b": 20}, {"a": 30, "b": 40}])
@@ -468,7 +605,8 @@ def test_namespace_distributed_write(s3_bucket: str):
         fragment2_data,
         table_uri,
         storage_options=merged_options,
-        storage_options_provider=storage_options_provider,
+        namespace_client=ns_client,
+        table_id=table_id,
     )
 
     fragment3_data = pa.Table.from_pylist([{"a": 100, "b": 200}])
@@ -476,7 +614,8 @@ def test_namespace_distributed_write(s3_bucket: str):
         fragment3_data,
         table_uri,
         storage_options=merged_options,
-        storage_options_provider=storage_options_provider,
+        namespace_client=ns_client,
+        table_id=table_id,
     )
 
     all_fragments = fragment1 + fragment2 + fragment3
@@ -487,7 +626,8 @@ def test_namespace_distributed_write(s3_bucket: str):
         table_uri,
         operation,
         storage_options=merged_options,
-        storage_options_provider=storage_options_provider,
+        namespace_client=ns_client,
+        table_id=table_id,
     )
 
     assert ds.count_rows() == 5
@@ -505,50 +645,54 @@ def test_namespace_distributed_write(s3_bucket: str):
     )
     assert result == expected
 
-    ds_from_namespace = lance.dataset(
-        namespace=namespace,
+    ds_from_ns_client = lance.dataset(
+        namespace_client=ns_client,
         table_id=table_id,
+        storage_options=storage_options,
     )
-    assert ds_from_namespace.count_rows() == 5
+    assert ds_from_ns_client.count_rows() == 5
 
 
 @pytest.mark.integration
-def test_file_writer_with_storage_options_provider(s3_bucket: str):
-    """Test LanceFileWriter with storage_options_provider and credential refresh."""
-    from lance import LanceNamespaceStorageOptionsProvider
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_file_writer_with_namespace_client(s3_bucket: str, use_custom: bool):
+    """Test LanceFileWriter with namespace_client and credential refresh."""
     from lance.file import LanceFileReader, LanceFileWriter
 
     storage_options = copy.deepcopy(CONFIG)
 
-    namespace = TrackingNamespace(
+    ns_client, inner_ns_client = create_tracking_namespace(
         bucket_name=s3_bucket,
         storage_options=storage_options,
         credential_expires_in_seconds=3,
+        use_custom=use_custom,
     )
 
     table1 = pa.Table.from_pylist([{"a": 1, "b": 2}, {"a": 10, "b": 20}])
     table_name = uuid.uuid4().hex
     table_id = ["test_ns", table_name]
 
-    assert namespace.get_create_call_count() == 0
-    assert namespace.get_describe_call_count() == 0
+    assert get_declare_call_count(inner_ns_client) == 0
+    assert get_describe_call_count(inner_ns_client) == 0
 
     ds = lance.write_dataset(
-        table1, namespace=namespace, table_id=table_id, mode="create"
+        table1,
+        namespace_client=ns_client,
+        table_id=table_id,
+        mode="create",
+        storage_options=storage_options,
     )
     assert ds.count_rows() == 2
-    assert namespace.get_create_call_count() == 1
+    assert get_declare_call_count(inner_ns_client) == 1
 
-    describe_response = namespace.describe_table(
+    describe_response = ns_client.describe_table(
         DescribeTableRequest(id=table_id, version=None)
     )
-    namespace_storage_options = describe_response.storage_options
-
-    provider = LanceNamespaceStorageOptionsProvider(
-        namespace=namespace, table_id=table_id
-    )
+    merged_options = dict(storage_options)
+    if describe_response.storage_options:
+        merged_options.update(describe_response.storage_options)
 
-    initial_describe_count = namespace.get_describe_call_count()
+    initial_describe_count = get_describe_call_count(inner_ns_client)
 
     file_uri = f"s3://{s3_bucket}/{table_name}_file_test.lance"
     schema = pa.schema([pa.field("x", pa.int64()), pa.field("y", pa.int64())])
@@ -556,8 +700,9 @@ def test_file_writer_with_storage_options_provider(s3_bucket: str):
     writer = LanceFileWriter(
         file_uri,
         schema=schema,
-        storage_options=namespace_storage_options,
-        storage_options_provider=provider,
+        storage_options=merged_options,
+        namespace_client=ns_client,
+        table_id=table_id,
     )
 
     batch = pa.RecordBatch.from_pydict({"x": [1, 2, 3], "y": [4, 5, 6]}, schema=schema)
@@ -569,13 +714,14 @@ def test_file_writer_with_storage_options_provider(s3_bucket: str):
     writer.write_batch(batch2)
     writer.close()
 
-    describe_count_after_write = namespace.get_describe_call_count()
+    describe_count_after_write = get_describe_call_count(inner_ns_client)
     assert describe_count_after_write == initial_describe_count
 
     reader = LanceFileReader(
         file_uri,
-        storage_options=namespace_storage_options,
-        storage_options_provider=provider,
+        storage_options=merged_options,
+        namespace_client=ns_client,
+        table_id=table_id,
     )
     result = reader.read_all(batch_size=1024)
     result_table = result.to_table()
@@ -593,8 +739,9 @@ def test_file_writer_with_storage_options_provider(s3_bucket: str):
     writer2 = LanceFileWriter(
         file_uri2,
         schema=schema,
-        storage_options=namespace_storage_options,
-        storage_options_provider=provider,
+        storage_options=merged_options,
+        namespace_client=ns_client,
+        table_id=table_id,
     )
 
     batch3 = pa.RecordBatch.from_pydict(
@@ -603,13 +750,14 @@ def test_file_writer_with_storage_options_provider(s3_bucket: str):
     writer2.write_batch(batch3)
     writer2.close()
 
-    final_describe_count = namespace.get_describe_call_count()
+    final_describe_count = get_describe_call_count(inner_ns_client)
     assert final_describe_count == describe_count_after_write + 1
 
     reader2 = LanceFileReader(
         file_uri2,
-        storage_options=namespace_storage_options,
-        storage_options_provider=provider,
+        storage_options=merged_options,
+        namespace_client=ns_client,
+        table_id=table_id,
     )
     result2 = reader2.read_all(batch_size=1024)
     result_table2 = result2.to_table()
@@ -619,17 +767,18 @@ def test_file_writer_with_storage_options_provider(s3_bucket: str):
 
 
 @pytest.mark.integration
-def test_file_reader_with_storage_options_provider(s3_bucket: str):
-    """Test LanceFileReader with storage_options_provider and credential refresh."""
-    from lance import LanceNamespaceStorageOptionsProvider
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_file_reader_with_namespace_client(s3_bucket: str, use_custom: bool):
+    """Test LanceFileReader with namespace_client and credential refresh."""
     from lance.file import LanceFileReader, LanceFileWriter
 
     storage_options = copy.deepcopy(CONFIG)
 
-    namespace = TrackingNamespace(
+    ns_client, inner_ns_client = create_tracking_namespace(
         bucket_name=s3_bucket,
         storage_options=storage_options,
         credential_expires_in_seconds=3,
+        use_custom=use_custom,
     )
 
     table1 = pa.Table.from_pylist([{"a": 1, "b": 2}, {"a": 10, "b": 20}])
@@ -637,52 +786,57 @@ def test_file_reader_with_storage_options_provider(s3_bucket: str):
     table_id = ["test_ns", table_name]
 
     ds = lance.write_dataset(
-        table1, namespace=namespace, table_id=table_id, mode="create"
+        table1,
+        namespace_client=ns_client,
+        table_id=table_id,
+        mode="create",
+        storage_options=storage_options,
     )
     assert ds.count_rows() == 2
 
-    describe_response = namespace.describe_table(
+    describe_response = ns_client.describe_table(
         DescribeTableRequest(id=table_id, version=None)
     )
-    namespace_storage_options = describe_response.storage_options
-
-    provider = LanceNamespaceStorageOptionsProvider(
-        namespace=namespace, table_id=table_id
-    )
+    merged_options = dict(storage_options)
+    if describe_response.storage_options:
+        merged_options.update(describe_response.storage_options)
 
     file_uri = f"s3://{s3_bucket}/{table_name}_file_reader_test.lance"
     schema = pa.schema([pa.field("x", pa.int64()), pa.field("y", pa.int64())])
 
-    # Write a file first (without provider to keep it simple)
+    # Write a file first (without namespace_client to keep it simple)
     writer = LanceFileWriter(
         file_uri,
         schema=schema,
-        storage_options=namespace_storage_options,
+        storage_options=merged_options,
     )
     batch = pa.RecordBatch.from_pydict({"x": [1, 2, 3], "y": [4, 5, 6]}, schema=schema)
     writer.write_batch(batch)
     writer.close()
 
     # Get fresh credentials for reading
-    describe_response = namespace.describe_table(
+    describe_response = ns_client.describe_table(
         DescribeTableRequest(id=table_id, version=None)
     )
-    namespace_storage_options = describe_response.storage_options
+    merged_options = dict(storage_options)
+    if describe_response.storage_options:
+        merged_options.update(describe_response.storage_options)
 
-    initial_describe_count = namespace.get_describe_call_count()
+    initial_describe_count = get_describe_call_count(inner_ns_client)
 
     # First read should work without needing refresh
     reader = LanceFileReader(
         file_uri,
-        storage_options=namespace_storage_options,
-        storage_options_provider=provider,
+        storage_options=merged_options,
+        namespace_client=ns_client,
+        table_id=table_id,
     )
     result = reader.read_all(batch_size=1024)
     result_table = result.to_table()
     assert result_table.num_rows == 3
     assert result_table.schema == schema
 
-    describe_count_after_first_read = namespace.get_describe_call_count()
+    describe_count_after_first_read = get_describe_call_count(inner_ns_client)
     assert describe_count_after_first_read == initial_describe_count
 
     # Wait for credentials to expire
@@ -693,7 +847,7 @@ def test_file_reader_with_storage_options_provider(s3_bucket: str):
     writer2 = LanceFileWriter(
         file_uri2,
         schema=schema,
-        storage_options=namespace_storage_options,
+        storage_options=merged_options,
     )
     batch2 = pa.RecordBatch.from_pydict(
         {"x": [100, 200], "y": [300, 400]}, schema=schema
@@ -704,8 +858,9 @@ def test_file_reader_with_storage_options_provider(s3_bucket: str):
     # Second read should trigger credential refresh
     reader2 = LanceFileReader(
         file_uri2,
-        storage_options=namespace_storage_options,
-        storage_options_provider=provider,
+        storage_options=merged_options,
+        namespace_client=ns_client,
+        table_id=table_id,
     )
     result2 = reader2.read_all(batch_size=1024)
     result_table2 = result2.to_table()
@@ -713,22 +868,23 @@ def test_file_reader_with_storage_options_provider(s3_bucket: str):
     expected_table2 = pa.table({"x": [100, 200], "y": [300, 400]}, schema=schema)
     assert result_table2 == expected_table2
 
-    final_describe_count = namespace.get_describe_call_count()
+    final_describe_count = get_describe_call_count(inner_ns_client)
     assert final_describe_count == describe_count_after_first_read + 1
 
 
 @pytest.mark.integration
-def test_file_session_with_storage_options_provider(s3_bucket: str):
-    """Test LanceFileSession with storage_options_provider and credential refresh."""
-    from lance import LanceNamespaceStorageOptionsProvider
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_file_session_with_namespace_client(s3_bucket: str, use_custom: bool):
+    """Test LanceFileSession with namespace_client and credential refresh."""
     from lance.file import LanceFileSession
 
     storage_options = copy.deepcopy(CONFIG)
 
-    namespace = TrackingNamespace(
+    ns_client, inner_ns_client = create_tracking_namespace(
         bucket_name=s3_bucket,
         storage_options=storage_options,
         credential_expires_in_seconds=3,
+        use_custom=use_custom,
     )
 
     table1 = pa.Table.from_pylist([{"a": 1, "b": 2}, {"a": 10, "b": 20}])
@@ -736,26 +892,29 @@ def test_file_session_with_storage_options_provider(s3_bucket: str):
     table_id = ["test_ns", table_name]
 
     ds = lance.write_dataset(
-        table1, namespace=namespace, table_id=table_id, mode="create"
+        table1,
+        namespace_client=ns_client,
+        table_id=table_id,
+        mode="create",
+        storage_options=storage_options,
     )
     assert ds.count_rows() == 2
 
-    describe_response = namespace.describe_table(
+    describe_response = ns_client.describe_table(
         DescribeTableRequest(id=table_id, version=None)
     )
-    namespace_storage_options = describe_response.storage_options
-
-    provider = LanceNamespaceStorageOptionsProvider(
-        namespace=namespace, table_id=table_id
-    )
+    merged_options = dict(storage_options)
+    if describe_response.storage_options:
+        merged_options.update(describe_response.storage_options)
 
-    initial_describe_count = namespace.get_describe_call_count()
+    initial_describe_count = get_describe_call_count(inner_ns_client)
 
-    # Create session with storage_options_provider
+    # Create session with namespace_client
     session = LanceFileSession(
         f"s3://{s3_bucket}/{table_name}_session",
-        storage_options=namespace_storage_options,
-        storage_options_provider=provider,
+        storage_options=merged_options,
+        namespace_client=ns_client,
+        table_id=table_id,
     )
 
     # Test contains method
@@ -776,7 +935,7 @@ def test_file_session_with_storage_options_provider(s3_bucket: str):
     writer.write_batch(batch)
     writer.close()
 
-    describe_count_after_first_write = namespace.get_describe_call_count()
+    describe_count_after_first_write = get_describe_call_count(inner_ns_client)
     assert describe_count_after_first_write == initial_describe_count
 
     # Test contains method after write
@@ -792,7 +951,7 @@ def test_file_session_with_storage_options_provider(s3_bucket: str):
     expected_table = pa.table({"x": [1, 2, 3], "y": [4, 5, 6]}, schema=schema)
     assert result_table == expected_table
 
-    describe_count_after_first_read = namespace.get_describe_call_count()
+    describe_count_after_first_read = get_describe_call_count(inner_ns_client)
     assert describe_count_after_first_read == describe_count_after_first_write
 
     # Wait for credentials to expire
@@ -809,7 +968,7 @@ def test_file_session_with_storage_options_provider(s3_bucket: str):
     writer2.write_batch(batch2)
     writer2.close()
 
-    describe_count_after_second_write = namespace.get_describe_call_count()
+    describe_count_after_second_write = get_describe_call_count(inner_ns_client)
     assert describe_count_after_second_write == describe_count_after_first_read + 1
 
     # Read the second file - should not trigger another refresh since we just refreshed
@@ -820,7 +979,7 @@ def test_file_session_with_storage_options_provider(s3_bucket: str):
     expected_table2 = pa.table({"x": [100, 200], "y": [300, 400]}, schema=schema)
     assert result_table2 == expected_table2
 
-    final_describe_count = namespace.get_describe_call_count()
+    final_describe_count = get_describe_call_count(inner_ns_client)
     assert final_describe_count == describe_count_after_second_write
 
 
@@ -846,69 +1005,66 @@ def table_to_ipc_bytes(table):
 
 
 @pytest.mark.integration
-def test_basic_create_and_drop_on_s3(s3_bucket: str):
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_basic_create_and_drop_on_s3(s3_bucket: str, use_custom: bool):
     """Test basic create and drop table operations on S3.
 
     Mirrors Java: testBasicCreateAndDropOnS3
     """
-    from lance.namespace import DirectoryNamespace
-    from lance_namespace import (
-        DropTableRequest,
-        TableExistsRequest,
-    )
-
     test_prefix = f"test-{uuid.uuid4().hex[:8]}"
     storage_options = copy.deepcopy(CONFIG)
     dir_props = {f"storage.{k}": v for k, v in storage_options.items()}
     dir_props["root"] = f"s3://{s3_bucket}/{test_prefix}"
-    namespace = DirectoryNamespace(**dir_props)
+    inner_ns_client = DirectoryNamespace(**dir_props)
+    ns_client = _wrap_if_custom(inner_ns_client, use_custom)
 
     table_name = "basic_test_table"
     table_data = create_test_table_data()
     table_id = ["test_ns", table_name]
 
-    # Create table using lance.write_dataset (same as other passing tests)
+    # Create table using lance.write_dataset
     ds = lance.write_dataset(
-        table_data, namespace=namespace, table_id=table_id, mode="create"
+        table_data,
+        namespace_client=ns_client,
+        table_id=table_id,
+        mode="create",
+        storage_options=storage_options,
     )
     assert ds is not None
     assert ds.count_rows() == 3
 
     # Drop table
     drop_req = DropTableRequest(id=table_id)
-    drop_resp = namespace.drop_table(drop_req)
+    drop_resp = ns_client.drop_table(drop_req)
     assert drop_resp is not None
 
     # Verify table no longer exists
     exists_req = TableExistsRequest(id=table_id)
     with pytest.raises(Exception):
-        namespace.table_exists(exists_req)
+        ns_client.table_exists(exists_req)
 
 
 @pytest.mark.integration
-def test_concurrent_create_and_drop_single_instance_on_s3(s3_bucket: str):
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_concurrent_create_and_drop_single_instance_on_s3(
+    s3_bucket: str, use_custom: bool
+):
     """Test concurrent create/drop with single namespace instance on S3."""
     import concurrent.futures
 
-    from lance.namespace import DirectoryNamespace
-    from lance_namespace import (
-        CreateNamespaceRequest,
-        CreateTableRequest,
-        DropTableRequest,
-    )
-
     test_prefix = f"test-{uuid.uuid4().hex[:8]}"
     storage_options = copy.deepcopy(CONFIG)
     dir_props = {f"storage.{k}": v for k, v in storage_options.items()}
     dir_props["root"] = f"s3://{s3_bucket}/{test_prefix}"
     # Very high retry count to guarantee all operations succeed
     dir_props["commit_retries"] = "2147483647"
-    namespace = DirectoryNamespace(**dir_props)
+    inner_ns_client = DirectoryNamespace(**dir_props)
+    ns_client = _wrap_if_custom(inner_ns_client, use_custom)
 
     # Initialize namespace first - create parent namespace to ensure __manifest table
     # is created before concurrent operations
     create_ns_req = CreateNamespaceRequest(id=["test_ns"])
-    namespace.create_namespace(create_ns_req)
+    ns_client.create_namespace(create_ns_req)
 
     num_tables = 10
     success_count = 0
@@ -925,11 +1081,11 @@ def create_and_drop_table(table_index):
 
             # Create table using atomic create_table API
             create_req = CreateTableRequest(id=table_id)
-            namespace.create_table(create_req, ipc_data)
+            ns_client.create_table(create_req, ipc_data)
 
             # Drop table
             drop_req = DropTableRequest(id=table_id)
-            namespace.drop_table(drop_req)
+            ns_client.drop_table(drop_req)
 
             with lock:
                 success_count += 1
@@ -950,18 +1106,13 @@ def create_and_drop_table(table_index):
 
 
 @pytest.mark.integration
-def test_concurrent_create_and_drop_multiple_instances_on_s3(s3_bucket: str):
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_concurrent_create_and_drop_multiple_instances_on_s3(
+    s3_bucket: str, use_custom: bool
+):
     """Test concurrent create/drop with multiple namespace instances on S3."""
     import concurrent.futures
 
-    from lance.namespace import DirectoryNamespace
-    from lance_namespace import (
-        CreateNamespaceRequest,
-        CreateTableRequest,
-        DropTableRequest,
-        ListTablesRequest,
-    )
-
     test_prefix = f"test-{uuid.uuid4().hex[:8]}"
     storage_options = copy.deepcopy(CONFIG)
     base_dir_props = {f"storage.{k}": v for k, v in storage_options.items()}
@@ -971,9 +1122,10 @@ def test_concurrent_create_and_drop_multiple_instances_on_s3(s3_bucket: str):
 
     # Initialize namespace first with a single instance to ensure __manifest
     # table is created and parent namespace exists before concurrent operations
-    init_ns = DirectoryNamespace(**base_dir_props.copy())
+    inner_init_ns_client = DirectoryNamespace(**base_dir_props.copy())
+    init_ns_client = _wrap_if_custom(inner_init_ns_client, use_custom)
     create_ns_req = CreateNamespaceRequest(id=["test_ns"])
-    init_ns.create_namespace(create_ns_req)
+    init_ns_client.create_namespace(create_ns_req)
 
     num_tables = 10
     success_count = 0
@@ -983,8 +1135,9 @@ def test_concurrent_create_and_drop_multiple_instances_on_s3(s3_bucket: str):
     def create_and_drop_table(table_index):
         nonlocal success_count, fail_count
         try:
-            # Each thread creates its own namespace instance
-            ns = DirectoryNamespace(**base_dir_props.copy())
+            # Each thread creates its own namespace client instance
+            inner_local_ns_client = DirectoryNamespace(**base_dir_props.copy())
+            local_ns_client = _wrap_if_custom(inner_local_ns_client, use_custom)
 
             table_name = f"s3_multi_ns_table_{table_index}"
             table_data = create_test_table_data()
@@ -993,11 +1146,11 @@ def create_and_drop_table(table_index):
 
             # Create table using atomic create_table API
             create_req = CreateTableRequest(id=table_id)
-            ns.create_table(create_req, ipc_data)
+            local_ns_client.create_table(create_req, ipc_data)
 
             # Drop table
             drop_req = DropTableRequest(id=table_id)
-            ns.drop_table(drop_req)
+            local_ns_client.drop_table(drop_req)
 
             with lock:
                 success_count += 1
@@ -1017,24 +1170,20 @@ def create_and_drop_table(table_index):
     assert fail_count == 0, f"Expected 0 failures, got {fail_count}"
 
     # Verify remaining state is consistent (no corruption)
-    verify_ns = DirectoryNamespace(**base_dir_props)
+    inner_verify_ns_client = DirectoryNamespace(**base_dir_props)
+    verify_ns_client = _wrap_if_custom(inner_verify_ns_client, use_custom)
     list_req = ListTablesRequest(id=["test_ns"])
-    _ = verify_ns.list_tables(list_req)  # Should not error
+    _ = verify_ns_client.list_tables(list_req)  # Should not error
 
 
 @pytest.mark.integration
-def test_concurrent_create_then_drop_from_different_instance_on_s3(s3_bucket: str):
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_concurrent_create_then_drop_from_different_instance_on_s3(
+    s3_bucket: str, use_custom: bool
+):
     """Test creating from one set of instances, dropping from different ones on S3."""
     import concurrent.futures
 
-    from lance.namespace import DirectoryNamespace
-    from lance_namespace import (
-        CreateNamespaceRequest,
-        CreateTableRequest,
-        DropTableRequest,
-        ListTablesRequest,
-    )
-
     test_prefix = f"test-{uuid.uuid4().hex[:8]}"
     storage_options = copy.deepcopy(CONFIG)
     base_dir_props = {f"storage.{k}": v for k, v in storage_options.items()}
@@ -1044,13 +1193,14 @@ def test_concurrent_create_then_drop_from_different_instance_on_s3(s3_bucket: st
 
     # Initialize namespace first with a single instance to ensure __manifest
     # table is created and parent namespace exists before concurrent operations
-    init_ns = DirectoryNamespace(**base_dir_props.copy())
+    inner_init_ns_client = DirectoryNamespace(**base_dir_props.copy())
+    init_ns_client = _wrap_if_custom(inner_init_ns_client, use_custom)
     create_ns_req = CreateNamespaceRequest(id=["test_ns"])
-    init_ns.create_namespace(create_ns_req)
+    init_ns_client.create_namespace(create_ns_req)
 
     num_tables = 10
 
-    # Phase 1: Create all tables concurrently using separate namespace instances
+    # Phase 1: Create all tables concurrently using separate namespace client instances
     create_success_count = 0
     create_fail_count = 0
     create_lock = Lock()
@@ -1059,7 +1209,8 @@ def create_table(table_index):
         nonlocal create_success_count, create_fail_count
         table_name = f"s3_cross_instance_table_{table_index}"
         try:
-            ns = DirectoryNamespace(**base_dir_props.copy())
+            inner_local_ns_client = DirectoryNamespace(**base_dir_props.copy())
+            local_ns_client = _wrap_if_custom(inner_local_ns_client, use_custom)
 
             table_data = create_test_table_data()
             table_id = ["test_ns", table_name]
@@ -1067,7 +1218,7 @@ def create_table(table_index):
 
             # Create table using atomic create_table API
             create_req = CreateTableRequest(id=table_id)
-            ns.create_table(create_req, ipc_data)
+            local_ns_client.create_table(create_req, ipc_data)
 
             with create_lock:
                 create_success_count += 1
@@ -1096,13 +1247,14 @@ def create_table(table_index):
     def drop_table(table_index):
         nonlocal drop_success_count, drop_fail_count
         try:
-            ns = DirectoryNamespace(**base_dir_props.copy())
+            inner_local_ns_client = DirectoryNamespace(**base_dir_props.copy())
+            local_ns_client = _wrap_if_custom(inner_local_ns_client, use_custom)
 
             table_name = f"s3_cross_instance_table_{table_index}"
             table_id = ["test_ns", table_name]
 
             drop_req = DropTableRequest(id=table_id)
-            ns.drop_table(drop_req)
+            local_ns_client.drop_table(drop_req)
 
             with drop_lock:
                 drop_success_count += 1
@@ -1123,6 +1275,7 @@ def drop_table(table_index):
     assert drop_fail_count == 0, f"Expected 0 drop failures, got {drop_fail_count}"
 
     # Verify remaining state is consistent (no corruption)
-    verify_ns = DirectoryNamespace(**base_dir_props)
+    inner_verify_ns_client = DirectoryNamespace(**base_dir_props)
+    verify_ns_client = _wrap_if_custom(inner_verify_ns_client, use_custom)
     list_req = ListTablesRequest(id=["test_ns"])
-    _ = verify_ns.list_tables(list_req)  # Should not error
+    _ = verify_ns_client.list_tables(list_req)  # Should not error
diff --git a/python/python/tests/test_namespace_rest.py b/python/python/tests/test_namespace_rest.py
index 429066f5cff..140d9168c05 100644
--- a/python/python/tests/test_namespace_rest.py
+++ b/python/python/tests/test_namespace_rest.py
@@ -650,18 +650,18 @@ def test_connect_with_rest(self):
 
             with lance.namespace.RestAdapter("dir", backend_config, port=0) as adapter:
                 properties = {"uri": f"http://127.0.0.1:{adapter.port}"}
-                ns = connect("rest", properties)
+                ns_client = connect("rest", properties)
 
-                assert isinstance(ns, lance.namespace.RestNamespace)
+                assert isinstance(ns_client, lance.namespace.RestNamespace)
 
                 create_req = CreateTableRequest(id=["test_table"])
                 table_data = create_test_data()
                 ipc_data = table_to_ipc_bytes(table_data)
-                response = ns.create_table(create_req, ipc_data)
+                response = ns_client.create_table(create_req, ipc_data)
                 assert response is not None
 
                 list_req = ListTablesRequest(id=[])
-                list_response = ns.list_tables(list_req)
+                list_response = ns_client.list_tables(list_req)
                 assert len(list_response.tables) == 1
                 assert list_response.tables[0] == "test_table"
 
@@ -675,14 +675,14 @@ def test_connect_with_custom_delimiter(self):
                     "uri": f"http://127.0.0.1:{adapter.port}",
                     "delimiter": "@",
                 }
-                ns = connect("rest", properties)
+                ns_client = connect("rest", properties)
 
-                assert isinstance(ns, lance.namespace.RestNamespace)
+                assert isinstance(ns_client, lance.namespace.RestNamespace)
 
                 create_req = CreateTableRequest(id=["test_table"])
                 table_data = create_test_data()
                 ipc_data = table_to_ipc_bytes(table_data)
-                response = ns.create_table(create_req, ipc_data)
+                response = ns_client.create_table(create_req, ipc_data)
                 assert response is not None
 
 
@@ -705,17 +705,17 @@ def provide_context(self, info):
             backend_config = {"root": tmpdir}
 
             with lance.namespace.RestAdapter("dir", backend_config, port=0) as adapter:
-                ns = lance.namespace.RestNamespace(
+                ns_client = lance.namespace.RestNamespace(
                     uri=f"http://127.0.0.1:{adapter.port}",
                     context_provider=TestProvider(),
                 )
 
                 # Perform operations
                 create_req = CreateNamespaceRequest(id=["workspace"])
-                ns.create_namespace(create_req)
+                ns_client.create_namespace(create_req)
 
                 list_req = ListNamespacesRequest(id=[])
-                ns.list_namespaces(list_req)
+                ns_client.list_namespaces(list_req)
 
                 # Context provider should have been called
                 assert call_count["count"] >= 2
@@ -734,7 +734,7 @@ def provide_context(self, info):
 
             with lance.namespace.RestAdapter("dir", backend_config, port=0) as adapter:
                 # Pass both explicit provider and class path - explicit should win
-                ns = lance.namespace.RestNamespace(
+                ns_client = lance.namespace.RestNamespace(
                     context_provider=ExplicitProvider(),
                     **{
                         "uri": f"http://127.0.0.1:{adapter.port}",
@@ -743,7 +743,7 @@ def provide_context(self, info):
                 )
 
                 create_req = CreateNamespaceRequest(id=["workspace"])
-                ns.create_namespace(create_req)
+                ns_client.create_namespace(create_req)
 
                 # Explicit provider should have been used
                 assert explicit_called["called"]
diff --git a/python/python/tests/test_optimize.py b/python/python/tests/test_optimize.py
index 748e4294f93..728ee1955b5 100644
--- a/python/python/tests/test_optimize.py
+++ b/python/python/tests/test_optimize.py
@@ -74,28 +74,30 @@ def test_optimize_max_bytes(tmp_path: Path):
     arr = pa.FixedSizeListArray.from_arrays(arr, 1024)
     data = pa.table({"a": arr})
 
+    # Write out 4K rows and 32MB of data
     dataset = lance.write_dataset(
         data, base_dir, max_rows_per_file=2 * 1024, data_storage_version="stable"
     )
+    # We get 2 fragments
     assert len(dataset.get_fragments()) == 2
 
-    # max_bytes_per_file is too small and we get tiny files
+    # Now run compaction with a small max_bytes_per_file (1000 bytes) to get more
+    # fragments.  The exact number is a bit tricky to calculate because we don't
+    # split into a new fragment until we've actually written data and that depends
+    # on how much the file format chooses to accumulate, but it should be more than 2
     metrics = dataset.optimize.compact_files(
         target_rows_per_fragment=100 * 1024,
         materialize_deletions=False,
         max_bytes_per_file=1000,
         batch_size=128,
     )
-
-    # We get 4 fragments here because we don't actually write any data to the file
-    # until we've accumulated 8MiB for a page.
     assert metrics.fragments_removed == 2
-    assert metrics.fragments_added == 4
+    assert metrics.fragments_added > 2
     assert metrics.files_removed == 2
-    assert metrics.files_added == 4
+    assert metrics.files_added > 2
 
     num_frags = len(dataset.get_fragments())
-    assert num_frags == 4
+    assert num_frags == metrics.fragments_added
 
     dataset = lance.write_dataset(
         data,
@@ -115,9 +117,9 @@ def test_optimize_max_bytes(tmp_path: Path):
     results = [task.execute(dataset) for task in plan.tasks]
     metrics = Compaction.commit(dataset, results)
     assert metrics.fragments_removed == 2
-    assert metrics.fragments_added == 4
+    assert metrics.fragments_added > 2
     assert metrics.files_removed == 2
-    assert metrics.files_added == 4
+    assert metrics.files_added > 2
 
     dataset = lance.write_dataset(
         data,
diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py
index 124a107363b..ce338660118 100644
--- a/python/python/tests/test_scalar_index.py
+++ b/python/python/tests/test_scalar_index.py
@@ -7,6 +7,7 @@
 import re
 import shutil
 import string
+import uuid
 import zipfile
 from datetime import date, datetime, timedelta
 from pathlib import Path
@@ -15,6 +16,7 @@
 import numpy as np
 import pyarrow as pa
 import pytest
+from conftest import ProgressRecorder, progress_event_tags, stage_progress_values
 from lance.indices import IndexConfig
 from lance.query import (
     BooleanQuery,
@@ -262,6 +264,107 @@ def test_indexed_between(tmp_path):
     assert actual_data.num_rows == 0
 
 
+def test_create_inverted_index_progress_callback(tmp_path):
+    ds = generate_multi_fragment_dataset(
+        tmp_path, num_fragments=2, rows_per_fragment=75
+    )
+    progress_recorder = ProgressRecorder()
+
+    ds.create_scalar_index(
+        column="text",
+        index_type="INVERTED",
+        remove_stop_words=False,
+        progress_callback=progress_recorder,
+    )
+
+    tags = progress_event_tags(progress_recorder.events)
+    expected_order = [
+        "start:load_data",
+        "complete:load_data",
+        "start:tokenize_docs",
+        "complete:tokenize_docs",
+        "start:copy_partitions",
+        "complete:copy_partitions",
+        "start:write_metadata",
+        "complete:write_metadata",
+    ]
+    positions = [tags.index(tag) for tag in expected_order]
+    assert positions == sorted(positions)
+
+    tokenize_progress = stage_progress_values(progress_recorder.events, "tokenize_docs")
+    assert tokenize_progress
+    assert max(tokenize_progress) == ds.count_rows()
+
+    assert "progress:copy_partitions" in tags
+    assert "progress:write_metadata" in tags
+    assert "start:merge_partitions" not in tags
+
+
+def test_merge_index_metadata_inverted_index_progress_callback(tmp_path):
+    ds = generate_multi_fragment_dataset(
+        tmp_path, num_fragments=3, rows_per_fragment=60
+    )
+
+    inverted_index_id = str(uuid.uuid4())
+    for fragment in ds.get_fragments():
+        ds.create_scalar_index(
+            column="text",
+            index_type="INVERTED",
+            name="text_inverted_progress_idx",
+            replace=False,
+            index_uuid=inverted_index_id,
+            fragment_ids=[fragment.fragment_id],
+            remove_stop_words=False,
+        )
+
+    progress_recorder = ProgressRecorder()
+    ds.merge_index_metadata(
+        inverted_index_id,
+        index_type="INVERTED",
+        progress_callback=progress_recorder,
+    )
+
+    tags = progress_event_tags(progress_recorder.events)
+    expected_order = [
+        "start:read_partition_metadata",
+        "complete:read_partition_metadata",
+        "start:remap_partition_files",
+        "complete:remap_partition_files",
+        "start:write_merged_metadata",
+        "complete:write_merged_metadata",
+    ]
+    positions = [tags.index(tag) for tag in expected_order]
+    assert positions == sorted(positions)
+
+    metadata_progress = stage_progress_values(
+        progress_recorder.events, "read_partition_metadata"
+    )
+    assert metadata_progress
+    assert metadata_progress[-1] == len(ds.get_fragments())
+    assert "progress:remap_partition_files" in tags
+    assert "progress:write_merged_metadata" in tags
+
+
+def test_create_inverted_index_progress_callback_error_after_completion_is_ignored(
+    tmp_path,
+):
+    ds = generate_multi_fragment_dataset(
+        tmp_path, num_fragments=2, rows_per_fragment=75
+    )
+    progress_recorder = ProgressRecorder(fail_on_tag="complete:write_metadata")
+
+    ds.create_scalar_index(
+        column="text",
+        index_type="INVERTED",
+        remove_stop_words=False,
+        progress_callback=progress_recorder,
+    )
+
+    tags = progress_event_tags(progress_recorder.events)
+    assert tags[-1] == "complete:write_metadata"
+    assert any(idx.index_type == "Inverted" for idx in ds.describe_indices())
+
+
 def test_index_combination(tmp_path):
     # This test regresses a bug in the index combination logic.
 
@@ -849,7 +952,12 @@ def test_fts_ngram_tokenizer(tmp_path):
 
 def test_fts_stats(dataset):
     dataset.create_scalar_index(
-        "doc", index_type="INVERTED", with_position=False, remove_stop_words=True
+        "doc",
+        index_type="INVERTED",
+        with_position=False,
+        remove_stop_words=True,
+        memory_limit=4096,
+        num_workers=2,
     )
     stats = dataset.stats.index_stats("doc_idx")
     assert stats["index_type"] == "Inverted"
@@ -864,6 +972,8 @@ def test_fts_stats(dataset):
     assert params["stem"] is True
     assert params["remove_stop_words"] is True
     assert params["ascii_folding"] is True
+    assert "memory_limit" not in params
+    assert "num_workers" not in params
 
 
 def test_fts_score(tmp_path):
@@ -2125,10 +2235,9 @@ def test_label_list_index_null_list_match(tmp_path: Path):
         "array_has_any(labels, ['foo'])",
         "array_has_all(labels, ['foo'])",
         "array_contains(labels, 'foo')",
-        # TODO(issue #5904): Enable after fixing NOT filters with whole-list NULLs
-        # "NOT array_has_any(labels, ['foo'])",
-        # "NOT array_has_all(labels, ['foo'])",
-        # "NOT array_contains(labels, 'foo')",
+        "NOT array_has_any(labels, ['foo'])",
+        "NOT array_has_all(labels, ['foo'])",
+        "NOT array_contains(labels, 'foo')",
     ]
     expected = {
         f: dataset.to_table(filter=f).column("labels").to_pylist() for f in filters
@@ -2358,6 +2467,37 @@ def scan_stats_callback(stats: lance.ScanStatistics):
     ).to_table()
     assert scan_stats.parts_loaded == 0
 
+    phrase_path = tmp_path / "phrase"
+    phrase_table = pa.table(
+        {
+            "fts": ["word word" for _ in range(test_table_size)],
+        }
+    )
+    ds = lance.write_dataset(phrase_table, phrase_path)
+    ds.create_scalar_index("fts", index_type="INVERTED", with_position=True)
+
+    ds = lance.dataset(phrase_path)
+    ds.prewarm_index("fts_idx")
+    cache_entries_after_prewarm = ds._ds.index_cache_entry_count()
+    results = ds.to_table(full_text_query=PhraseQuery("word word", "fts"))
+    assert results.num_rows == test_table_size
+    cache_entries_after_query = ds._ds.index_cache_entry_count()
+    assert cache_entries_after_query > cache_entries_after_prewarm
+
+    ds = lance.dataset(phrase_path)
+    ds.prewarm_index("fts_idx", with_position=True)
+    cache_entries_after_prewarm = ds._ds.index_cache_entry_count()
+    results = ds.to_table(full_text_query=PhraseQuery("word word", "fts"))
+    assert results.num_rows == test_table_size
+    cache_entries_after_query = ds._ds.index_cache_entry_count()
+    assert cache_entries_after_query == cache_entries_after_prewarm
+
+    with pytest.raises(
+        TypeError,
+        match="takes 2 positional arguments",
+    ):
+        ds.prewarm_index("fts_idx", True)
+
 
 def test_btree_prewarm(tmp_path: Path):
     scan_stats = None
@@ -4254,7 +4394,9 @@ def test_json_inverted_match_query(tmp_path):
     assert results.num_rows == 1
 
 
-def test_describe_indices(tmp_path):
+@pytest.mark.parametrize("fts_format_version", ["1", "2"])
+def test_describe_indices(tmp_path, monkeypatch, fts_format_version):
+    monkeypatch.setenv("LANCE_FTS_FORMAT_VERSION", fts_format_version)
     data = pa.table(
         {
             "id": range(100),
@@ -4284,9 +4426,13 @@ def test_describe_indices(tmp_path):
     assert indices[0].segments[0].uuid is not None
     assert indices[0].segments[0].fragment_ids == {0}
     assert indices[0].segments[0].dataset_version_at_last_update == 1
-    assert indices[0].segments[0].index_version == 1
+    assert indices[0].segments[0].index_version == int(fts_format_version)
     assert indices[0].segments[0].created_at is not None
     assert isinstance(indices[0].segments[0].created_at, datetime)
+    assert indices[0].segments[0].size_bytes is not None
+    assert indices[0].segments[0].size_bytes > 0
+    assert indices[0].total_size_bytes is not None
+    assert indices[0].total_size_bytes > 0
 
     details = indices[0].details
     assert details is not None and len(details) > 0
@@ -4367,6 +4513,10 @@ def test_describe_indices(tmp_path):
         assert indices[i].segments[0].index_version == 0
         assert indices[i].segments[0].created_at is not None
         assert isinstance(indices[i].segments[0].created_at, datetime)
+        assert indices[i].segments[0].size_bytes is not None
+        assert indices[i].segments[0].size_bytes > 0
+        assert indices[i].total_size_bytes is not None
+        assert indices[i].total_size_bytes > 0
         assert indices[i].details == json.loads(details[i])
 
     ds.delete("id < 50")
diff --git a/python/python/tests/test_table_ops.py b/python/python/tests/test_table_ops.py
index 4e61feff6d0..777b89da05a 100644
--- a/python/python/tests/test_table_ops.py
+++ b/python/python/tests/test_table_ops.py
@@ -106,3 +106,95 @@ def test_replacement_after_index(tmp_path: str):
             ),
             read_version=ds3.version,
         )
+
+
+def test_data_file_create_basic(tmp_path: str):
+    """DataFile.create should read file metadata and produce correct fields/indices."""
+    table = pa.table({"a": range(10), "b": range(10, 20)})
+    ds = lance.write_dataset(table, tmp_path)
+
+    # Write a lance file with both columns
+    new_file_name = f"{uuid.uuid4()}.lance"
+    new_file_path = f"{tmp_path}/data/{new_file_name}"
+    with LanceFileWriter(new_file_path) as writer:
+        writer.write_batch(table)
+
+    df = DataFile.create(ds, new_file_name)
+
+    # Should have both field IDs from the dataset
+    frag = ds.get_fragments()[0]
+    expected_fields = frag.data_files()[0].fields
+    assert df.fields == expected_fields
+    assert df.column_indices == [0, 1]
+    assert df.file_major_version == int(stable_version().split(".")[0])
+    assert df.file_minor_version == int(stable_version().split(".")[1])
+    assert df.file_size_bytes is not None and df.file_size_bytes > 0
+
+
+def test_data_file_create_subset_columns(tmp_path: str):
+    """DataFile.create should work for a file with a subset of dataset columns."""
+    table = pa.table({"a": range(10), "b": range(10, 20)})
+    ds = lance.write_dataset(table, tmp_path)
+    ds.add_columns({"c": "a + b"})
+    ds = lance.dataset(tmp_path)
+
+    # Write a file with only column b
+    new_file_name = f"{uuid.uuid4()}.lance"
+    new_file_path = f"{tmp_path}/data/{new_file_name}"
+    with LanceFileWriter(new_file_path, pa.schema([("b", pa.int64())])) as writer:
+        writer.write_batch(pa.table({"b": range(100, 110)}))
+
+    df = DataFile.create(ds, new_file_name)
+
+    # Should only have b's field ID
+    frag = ds.get_fragments()[0]
+    all_fields = frag.data_files()[0].fields
+    # b is the second field in the original data file
+    b_field_id = all_fields[1]
+    assert df.fields == [b_field_id]
+    assert df.column_indices == [0]
+
+
+def test_data_file_create_end_to_end(tmp_path: str):
+    """DataFile.create should work end-to-end with DataReplacement."""
+    table = pa.table({"a": range(100)})
+    ds = lance.write_dataset(table, tmp_path)
+    ds.add_columns({"b": "a + 1"})
+    ds = lance.dataset(tmp_path)
+
+    # Write a replacement file for column b
+    new_file_name = f"{uuid.uuid4()}.lance"
+    new_file_path = f"{tmp_path}/data/{new_file_name}"
+    replacement_data = pa.table({"b": range(200, 300)})
+    with LanceFileWriter(new_file_path, pa.schema([("b", pa.int64())])) as writer:
+        writer.write_batch(replacement_data)
+
+    # Use DataFile.create instead of manual construction
+    df = DataFile.create(ds, new_file_name)
+
+    ds.commit(
+        ds.uri,
+        lance.LanceOperation.DataReplacement(
+            [lance.LanceOperation.DataReplacementGroup(0, df)]
+        ),
+        read_version=ds.version,
+    )
+
+    result = lance.dataset(tmp_path).to_table()
+    assert result.column("b").to_pylist() == list(range(200, 300))
+    assert result.column("a").to_pylist() == list(range(100))
+
+
+def test_data_file_create_unknown_column(tmp_path: str):
+    """DataFile.create should raise an error for a file with unknown columns."""
+    table = pa.table({"a": range(10)})
+    ds = lance.write_dataset(table, tmp_path)
+
+    # Write a file with a column not in the dataset
+    new_file_name = f"{uuid.uuid4()}.lance"
+    new_file_path = f"{tmp_path}/data/{new_file_name}"
+    with LanceFileWriter(new_file_path) as writer:
+        writer.write_batch(pa.table({"z": range(10)}))
+
+    with pytest.raises(Exception, match="z"):
+        DataFile.create(ds, new_file_name)
diff --git a/python/python/tests/test_tf.py b/python/python/tests/test_tf.py
index 432be52b482..3652df0e938 100644
--- a/python/python/tests/test_tf.py
+++ b/python/python/tests/test_tf.py
@@ -116,12 +116,12 @@ def fake_dataset(uri=None, **kwargs):
     ns = object()
     ds = from_lance(
         None,
-        namespace=ns,
+        namespace_client=ns,
         table_id=["tbl"],
         ignore_namespace_table_storage_options=True,
     )
 
-    assert calls["kwargs"]["namespace"] is ns
+    assert calls["kwargs"]["namespace_client"] is ns
     assert calls["kwargs"]["table_id"] == ["tbl"]
     assert calls["kwargs"]["ignore_namespace_table_storage_options"] is True
 
diff --git a/python/python/tests/test_vector_index.py b/python/python/tests/test_vector_index.py
index cf3f38158b8..5371aa1f2a7 100644
--- a/python/python/tests/test_vector_index.py
+++ b/python/python/tests/test_vector_index.py
@@ -9,7 +9,6 @@
 import string
 import tempfile
 import time
-import uuid
 from pathlib import Path
 from typing import Optional
 
@@ -18,8 +17,9 @@
 import pyarrow as pa
 import pyarrow.compute as pc
 import pytest
+from conftest import ProgressRecorder, progress_event_tags, stage_progress_values
 from lance import LanceDataset, LanceFragment
-from lance.dataset import Index, VectorIndexReader
+from lance.dataset import VectorIndexReader
 from lance.indices import IndexFileVersion, IndicesBuilder
 from lance.query import MatchQuery, PhraseQuery
 from lance.util import validate_vector_index  # noqa: E402
@@ -184,6 +184,62 @@ def test_ann(indexed_dataset):
     run(indexed_dataset)
 
 
+def test_create_index_progress_callback_vector(tmp_path):
+    ds = _make_sample_dataset_base(tmp_path, "vector_progress", 1500, 128)
+    recorder = ProgressRecorder()
+
+    ds.create_index(
+        column="vector",
+        index_type="IVF_PQ",
+        num_partitions=4,
+        num_sub_vectors=4,
+        progress_callback=recorder,
+    )
+
+    tags = progress_event_tags(recorder.events)
+    expected_order = [
+        "start:train_ivf",
+        "complete:train_ivf",
+        "start:train_quantizer",
+        "complete:train_quantizer",
+        "start:shuffle",
+        "complete:shuffle",
+        "start:merge_partitions",
+        "complete:merge_partitions",
+    ]
+    positions = [tags.index(tag) for tag in expected_order]
+    assert positions == sorted(positions)
+
+    shuffle_progress = stage_progress_values(recorder.events, "shuffle")
+    assert shuffle_progress
+    assert shuffle_progress[-1] == ds.count_rows()
+
+    merge_progress = stage_progress_values(recorder.events, "merge_partitions")
+    assert merge_progress
+    assert merge_progress[-1] == 4
+
+
+def test_create_index_progress_callback_error_before_completion_propagates(tmp_path):
+    ds = _make_sample_dataset_base(
+        tmp_path, "vector_progress_post_commit_error", 1500, 128
+    )
+    recorder = ProgressRecorder(fail_on_tag="start:train_ivf")
+
+    with pytest.raises(RuntimeError, match="progress callback failure"):
+        ds.create_index(
+            column="vector",
+            index_type="IVF_PQ",
+            num_partitions=4,
+            num_sub_vectors=4,
+            progress_callback=recorder,
+        )
+
+    tags = progress_event_tags(recorder.events)
+    assert tags == ["start:train_ivf"]
+    assert not ds.has_index
+    assert ds.describe_indices() == []
+
+
 def test_distributed_ivf_pq_partition_window_env_override(tmp_path, monkeypatch):
     # Keep this before other distributed vector merge tests so the process-level
     # lazy window size initialization reads this override.
@@ -1644,6 +1700,22 @@ def test_optimize_indices(indexed_dataset):
     assert stats["num_indices"] == 2
 
 
+def test_logical_and_physical_index_views(indexed_dataset):
+    data = create_table()
+    indexed_dataset = lance.write_dataset(data, indexed_dataset.uri, mode="append")
+    indexed_dataset.optimize.optimize_indices(num_indices_to_merge=0)
+
+    logical_indices = indexed_dataset.describe_indices()
+    assert len(logical_indices) == 1
+    assert logical_indices[0].name == "vector_idx"
+    assert len(logical_indices[0].segments) == 2
+    assert all(segment.fragment_ids for segment in logical_indices[0].segments)
+
+    stats = indexed_dataset.stats.index_stats("vector_idx")
+    assert stats["num_segments"] == stats["num_indices"] == 2
+    assert stats["segments"] == stats["indices"]
+
+
 @pytest.mark.skip(reason="retrain is deprecated")
 def test_retrain_indices(indexed_dataset):
     data = create_table()
@@ -2112,38 +2184,64 @@ def build_distributed_vector_index(
     world=2,
     **index_params,
 ):
-    """Build a distributed vector index over fragment groups and commit.
-
-    Steps:
-    - Partition fragments into `world` groups
-    - For each group, call create_index with fragment_ids and a shared index_uuid
-    - Merge metadata (commit index manifest)
-
-    Returns the dataset (post-merge) for querying.
-    """
+    """Build a distributed vector index over fragment groups and commit."""
 
     frags = dataset.get_fragments()
     frag_ids = [f.fragment_id for f in frags]
     groups = _split_fragments_evenly(frag_ids, world)
-    shared_uuid = str(uuid.uuid4())
+    segments = []
 
     for g in groups:
         if not g:
             continue
-        dataset.create_index(
-            column=column,
-            index_type=index_type,
-            fragment_ids=g,
-            index_uuid=shared_uuid,
-            num_partitions=num_partitions,
-            num_sub_vectors=num_sub_vectors,
-            **index_params,
+        segments.append(
+            dataset.create_index_uncommitted(
+                column=column,
+                index_type=index_type,
+                fragment_ids=g,
+                num_partitions=num_partitions,
+                num_sub_vectors=num_sub_vectors,
+                **index_params,
+            )
         )
 
-    # Merge physical index metadata and commit manifest for VECTOR
-    dataset.merge_index_metadata(shared_uuid, index_type)
-    dataset = _commit_index_helper(dataset, shared_uuid, column="vector")
-    return dataset
+    return dataset.commit_existing_index_segments(f"{column}_idx", column, segments)
+
+
+def _commit_segments_helper(
+    ds, segments, column: str, index_name: Optional[str] = None
+):
+    if index_name is None:
+        index_name = f"{column}_idx"
+    return ds.commit_existing_index_segments(index_name, column, segments)
+
+
+def _build_segments(
+    ds,
+    column: str,
+    index_type: str,
+    fragment_groups,
+    *,
+    index_name: Optional[str] = None,
+    **index_kwargs,
+):
+    if index_name is None:
+        index_name = f"{column}_idx"
+
+    segments = []
+    for group in fragment_groups:
+        if not group:
+            continue
+        segments.append(
+            ds.create_index_uncommitted(
+                column=column,
+                index_type=index_type,
+                name=index_name,
+                fragment_ids=group,
+                **index_kwargs,
+            )
+        )
+    return segments
 
 
 def assert_distributed_vector_consistency(
@@ -2458,7 +2556,6 @@ def test_metadata_merge_pq_success(tmp_path):
     mid = max(1, len(frags) // 2)
     node1 = [f.fragment_id for f in frags[:mid]]
     node2 = [f.fragment_id for f in frags[mid:]]
-    shared_uuid = str(uuid.uuid4())
     builder = IndicesBuilder(ds, "vector")
     pre = builder.prepare_global_ivf_pq(
         num_partitions=8,
@@ -2468,28 +2565,18 @@ def test_metadata_merge_pq_success(tmp_path):
         max_iters=20,
     )
     try:
-        ds.create_index(
-            column="vector",
-            index_type="IVF_PQ",
-            fragment_ids=node1,
-            index_uuid=shared_uuid,
-            num_partitions=8,
-            num_sub_vectors=16,
-            ivf_centroids=pre["ivf_centroids"],
-            pq_codebook=pre["pq_codebook"],
-        )
-        ds.create_index(
-            column="vector",
-            index_type="IVF_PQ",
-            fragment_ids=node2,
-            index_uuid=shared_uuid,
+        segments = _build_segments(
+            ds,
+            "vector",
+            "IVF_PQ",
+            [node1, node2],
+            index_name="vector_idx",
             num_partitions=8,
             num_sub_vectors=16,
             ivf_centroids=pre["ivf_centroids"],
             pq_codebook=pre["pq_codebook"],
         )
-        ds.merge_index_metadata(shared_uuid, "IVF_PQ")
-        ds = _commit_index_helper(ds, shared_uuid, "vector")
+        ds = _commit_segments_helper(ds, segments, "vector")
         q = np.random.rand(128).astype(np.float32)
         results = ds.to_table(nearest={"column": "vector", "q": q, "k": 10})
         assert 0 < len(results) <= 10
@@ -2504,7 +2591,6 @@ def test_distributed_workflow_merge_and_search(tmp_path):
     frags = ds.get_fragments()
     if len(frags) < 2:
         pytest.skip("Need at least 2 fragments for distributed testing")
-    shared_uuid = str(uuid.uuid4())
     mid = len(frags) // 2
     node1 = [f.fragment_id for f in frags[:mid]]
     node2 = [f.fragment_id for f in frags[mid:]]
@@ -2517,28 +2603,18 @@ def test_distributed_workflow_merge_and_search(tmp_path):
         max_iters=20,
     )
     try:
-        ds.create_index(
-            column="vector",
-            index_type="IVF_PQ",
-            fragment_ids=node1,
-            index_uuid=shared_uuid,
-            num_partitions=4,
-            num_sub_vectors=4,
-            ivf_centroids=pre["ivf_centroids"],
-            pq_codebook=pre["pq_codebook"],
-        )
-        ds.create_index(
-            column="vector",
-            index_type="IVF_PQ",
-            fragment_ids=node2,
-            index_uuid=shared_uuid,
+        segments = _build_segments(
+            ds,
+            "vector",
+            "IVF_PQ",
+            [node1, node2],
+            index_name="vector_idx",
             num_partitions=4,
             num_sub_vectors=4,
             ivf_centroids=pre["ivf_centroids"],
             pq_codebook=pre["pq_codebook"],
         )
-        ds._ds.merge_index_metadata(shared_uuid, "IVF_PQ")
-        ds = _commit_index_helper(ds, shared_uuid, "vector")
+        ds = _commit_segments_helper(ds, segments, "vector")
         q = np.random.rand(128).astype(np.float32)
         results = ds.to_table(nearest={"column": "vector", "q": q, "k": 10})
         assert 0 < len(results) <= 10
@@ -2552,8 +2628,6 @@ def test_vector_merge_two_shards_success_flat(tmp_path):
     assert len(frags) >= 2
     shard1 = [frags[0].fragment_id]
     shard2 = [frags[1].fragment_id]
-    shared_uuid = str(uuid.uuid4())
-
     # Global preparation
     builder = IndicesBuilder(ds, "vector")
     preprocessed = builder.prepare_global_ivf_pq(
@@ -2564,28 +2638,18 @@ def test_vector_merge_two_shards_success_flat(tmp_path):
         max_iters=20,
     )
 
-    ds.create_index(
-        column="vector",
-        index_type="IVF_FLAT",
-        fragment_ids=shard1,
-        index_uuid=shared_uuid,
-        num_partitions=4,
-        num_sub_vectors=128,
-        ivf_centroids=preprocessed["ivf_centroids"],
-        pq_codebook=preprocessed["pq_codebook"],
-    )
-    ds.create_index(
-        column="vector",
-        index_type="IVF_FLAT",
-        fragment_ids=shard2,
-        index_uuid=shared_uuid,
+    segments = _build_segments(
+        ds,
+        "vector",
+        "IVF_FLAT",
+        [shard1, shard2],
+        index_name="vector_idx",
         num_partitions=4,
         num_sub_vectors=128,
         ivf_centroids=preprocessed["ivf_centroids"],
         pq_codebook=preprocessed["pq_codebook"],
     )
-    ds._ds.merge_index_metadata(shared_uuid, "IVF_FLAT", None)
-    ds = _commit_index_helper(ds, shared_uuid, column="vector")
+    ds = _commit_segments_helper(ds, segments, column="vector")
     q = np.random.rand(128).astype(np.float32)
     result = ds.to_table(nearest={"column": "vector", "q": q, "k": 5})
     assert 0 < len(result) <= 5
@@ -2605,8 +2669,6 @@ def test_distributed_ivf_parameterized(tmp_path, index_type, num_sub_vectors):
     mid = len(frags) // 2
     node1 = [f.fragment_id for f in frags[:mid]]
     node2 = [f.fragment_id for f in frags[mid:]]
-    shared_uuid = str(uuid.uuid4())
-
     builder = IndicesBuilder(ds, "vector")
     pre = builder.prepare_global_ivf_pq(
         num_partitions=4,
@@ -2620,7 +2682,6 @@ def test_distributed_ivf_parameterized(tmp_path, index_type, num_sub_vectors):
         base_kwargs = dict(
             column="vector",
             index_type=index_type,
-            index_uuid=shared_uuid,
             num_partitions=4,
             num_sub_vectors=num_sub_vectors,
         )
@@ -2636,11 +2697,11 @@ def test_distributed_ivf_parameterized(tmp_path, index_type, num_sub_vectors):
                 ivf_centroids=pre["ivf_centroids"], pq_codebook=pre["pq_codebook"]
             )
 
-        ds.create_index(**kwargs1)
-        ds.create_index(**kwargs2)
-
-        ds._ds.merge_index_metadata(shared_uuid, index_type, None)
-        ds = _commit_index_helper(ds, shared_uuid, "vector")
+        segments = [
+            ds.create_index_uncommitted(**kwargs1),
+            ds.create_index_uncommitted(**kwargs2),
+        ]
+        ds = _commit_segments_helper(ds, segments, "vector")
 
         q = np.random.rand(128).astype(np.float32)
         results = ds.to_table(nearest={"column": "vector", "q": q, "k": 10})
@@ -2649,45 +2710,6 @@ def test_distributed_ivf_parameterized(tmp_path, index_type, num_sub_vectors):
         raise e
 
 
-def _commit_index_helper(
-    ds, index_uuid: str, column: str, index_name: Optional[str] = None
-):
-    """Helper to finalize index commit after merge_index_metadata.
-
-    Builds a lance.dataset.Index record and commits a CreateIndex operation.
-    Returns the updated dataset object.
-    """
-
-    # Resolve field id for the target column
-    lance_field = ds.lance_schema.field(column)
-    if lance_field is None:
-        raise KeyError(f"{column} not found in schema")
-    field_id = lance_field.id()
-
-    # Default index name if not provided
-    if index_name is None:
-        index_name = f"{column}_idx"
-
-    # Build fragment id set
-    frag_ids = set(f.fragment_id for f in ds.get_fragments())
-
-    # Construct Index dataclass and commit operation
-    index = Index(
-        uuid=index_uuid,
-        name=index_name,
-        fields=[field_id],
-        dataset_version=ds.version,
-        fragment_ids=frag_ids,
-        index_version=0,
-    )
-    create_index_op = lance.LanceOperation.CreateIndex(
-        new_indices=[index], removed_indices=[]
-    )
-    ds = lance.LanceDataset.commit(ds.uri, create_index_op, read_version=ds.version)
-    # Ensure unified index partitions are materialized
-    return ds
-
-
 @pytest.mark.parametrize(
     "index_type,num_sub_vectors",
     [
@@ -2701,8 +2723,6 @@ def test_merge_two_shards_parameterized(tmp_path, index_type, num_sub_vectors):
     assert len(frags) >= 2
     shard1 = [frags[0].fragment_id]
     shard2 = [frags[1].fragment_id]
-    shared_uuid = str(uuid.uuid4())
-
     builder = IndicesBuilder(ds, "vector")
     pre = builder.prepare_global_ivf_pq(
         num_partitions=4,
@@ -2715,7 +2735,6 @@ def test_merge_two_shards_parameterized(tmp_path, index_type, num_sub_vectors):
     base_kwargs = {
         "column": "vector",
         "index_type": index_type,
-        "index_uuid": shared_uuid,
         "num_partitions": 4,
     }
 
@@ -2729,7 +2748,7 @@ def test_merge_two_shards_parameterized(tmp_path, index_type, num_sub_vectors):
         # only PQ has pq_codebook
         if "pq_codebook" in pre:
             kwargs1["pq_codebook"] = pre["pq_codebook"]
-    ds.create_index(**kwargs1)
+    segment1 = ds.create_index_uncommitted(**kwargs1)
 
     # second shard
     kwargs2 = dict(base_kwargs)
@@ -2740,10 +2759,51 @@ def test_merge_two_shards_parameterized(tmp_path, index_type, num_sub_vectors):
         kwargs2["ivf_centroids"] = pre["ivf_centroids"]
         if "pq_codebook" in pre:
             kwargs2["pq_codebook"] = pre["pq_codebook"]
-    ds.create_index(**kwargs2)
+    segment2 = ds.create_index_uncommitted(**kwargs2)
 
-    ds._ds.merge_index_metadata(shared_uuid, index_type, None)
-    ds = _commit_index_helper(ds, shared_uuid, column="vector")
+    merged_segment = ds.merge_existing_index_segments([segment1, segment2])
+    ds = _commit_segments_helper(ds, [merged_segment], column="vector")
+
+    q = np.random.rand(128).astype(np.float32)
+    results = ds.to_table(nearest={"column": "vector", "q": q, "k": 5})
+    assert 0 < len(results) <= 5
+
+
+def test_merge_existing_index_segments_builds_vector_segment(tmp_path):
+    ds = _make_sample_dataset_base(tmp_path, "merge_existing_segments_ds", 2000, 128)
+    frags = ds.get_fragments()
+    assert len(frags) >= 2
+    builder = IndicesBuilder(ds, "vector")
+    preprocessed = builder.prepare_global_ivf_pq(
+        num_partitions=4,
+        num_subvectors=4,
+        distance_type="l2",
+        sample_rate=7,
+        max_iters=20,
+    )
+
+    segments = [
+        ds.create_index_uncommitted(
+            "vector",
+            "IVF_FLAT",
+            name="vector_idx",
+            train=True,
+            fragment_ids=[fragment.fragment_id],
+            num_partitions=4,
+            num_sub_vectors=128,
+            ivf_centroids=preprocessed["ivf_centroids"],
+            pq_codebook=preprocessed["pq_codebook"],
+        )
+        for fragment in frags[:2]
+    ]
+    assert all(segment.index_details is not None for segment in segments)
+
+    merged_segment = ds.merge_existing_index_segments(segments)
+    assert merged_segment.fragment_ids is not None
+    assert sorted(merged_segment.fragment_ids) == sorted(
+        [fragment.fragment_id for fragment in frags[:2]]
+    )
+    ds = ds.commit_existing_index_segments("vector_idx", "vector", [merged_segment])
 
     q = np.random.rand(128).astype(np.float32)
     results = ds.to_table(nearest={"column": "vector", "q": q, "k": 5})
@@ -2792,21 +2852,19 @@ def test_distributed_ivf_pq_order_invariance(tmp_path: Path):
         pytest.skip("Failed to split fragments into two non-empty groups (order_21)")
 
     def build_distributed_ivf_pq(ds_copy, shard_order):
-        shared_uuid = str(uuid.uuid4())
         try:
-            for shard in shard_order:
-                ds_copy.create_index(
-                    column="vector",
-                    index_type="IVF_PQ",
-                    fragment_ids=shard,
-                    index_uuid=shared_uuid,
-                    num_partitions=4,
-                    num_sub_vectors=16,
-                    ivf_centroids=pre["ivf_centroids"],
-                    pq_codebook=pre["pq_codebook"],
-                )
-            ds_copy.merge_index_metadata(shared_uuid, "IVF_PQ")
-            return _commit_index_helper(ds_copy, shared_uuid, column="vector")
+            segments = _build_segments(
+                ds_copy,
+                "vector",
+                "IVF_PQ",
+                shard_order,
+                index_name="vector_idx",
+                num_partitions=4,
+                num_sub_vectors=16,
+                ivf_centroids=pre["ivf_centroids"],
+                pq_codebook=pre["pq_codebook"],
+            )
+            return _commit_segments_helper(ds_copy, segments, column="vector")
         except ValueError as e:
             raise e
 
diff --git a/python/python/tests/torch_tests/test_data.py b/python/python/tests/torch_tests/test_data.py
index 890a536cc9e..734ca3ce558 100644
--- a/python/python/tests/torch_tests/test_data.py
+++ b/python/python/tests/torch_tests/test_data.py
@@ -12,7 +12,11 @@
 from lance.sampler import ShardedBatchSampler, ShardedFragmentSampler
 
 torch = pytest.importorskip("torch")
-from lance.torch.data import LanceDataset  # noqa: E402
+from lance.torch.data import (  # noqa: E402
+    LanceDataset,
+    SafeLanceDataset,
+    _bf16_to_tensor,
+)
 
 
 def test_iter_over_dataset_fixed_shape_tensor(tmp_path):
@@ -324,3 +328,121 @@ def to_tensor_fn(batch, *args, **kwargs):
     assert first["int"].shape == (4,)
     assert first["val"].dtype == torch.uint8
     assert first["val"].shape == (4, 100)
+
+
+def test_iter_over_dataset_bfloat16(tmp_path):
+    """Test that bfloat16 vector columns convert to torch.bfloat16 tensors."""
+    ml_dtypes = pytest.importorskip("ml_dtypes")
+    from lance.arrow import BFloat16Array
+
+    dim = 32
+    num_rows = 128
+    # Create random bfloat16 vectors via float32 → bfloat16 cast
+    f32_data = np.random.random(num_rows * dim).astype("f")
+    bf16_data = f32_data.astype(ml_dtypes.bfloat16)
+
+    # Build a FixedSizeList<bf16> column
+    inner = BFloat16Array.from_numpy(bf16_data)
+    fsl = pa.FixedSizeListArray.from_arrays(inner, dim)
+    ids = pa.array(range(num_rows), type=pa.int32())
+    tbl = pa.Table.from_arrays([ids, fsl], ["ids", "vec"])
+
+    ds = lance.write_dataset(tbl, tmp_path / "data.lance", max_rows_per_group=32)
+
+    torch_ds = LanceDataset(ds, batch_size=16, columns=["ids", "vec"])
+
+    total_rows = 0
+    for batch in torch_ds:
+        assert set(batch.keys()) == {"ids", "vec"}
+        assert batch["vec"].dtype == torch.bfloat16
+        assert batch["vec"].shape[1] == dim
+        assert batch["ids"].dtype == torch.int32
+        total_rows += batch["vec"].shape[0]
+    assert total_rows == num_rows
+
+
+def test_scalar_bfloat16_column(tmp_path):
+    """Test that a scalar bfloat16 column converts to torch.bfloat16 tensor."""
+    ml_dtypes = pytest.importorskip("ml_dtypes")
+    from lance.arrow import BFloat16Array
+
+    num_rows = 64
+    f32_data = np.random.random(num_rows).astype("f")
+    bf16_data = f32_data.astype(ml_dtypes.bfloat16)
+
+    arr = BFloat16Array.from_numpy(bf16_data)
+    tbl = pa.Table.from_arrays([arr], ["val"])
+
+    ds = lance.write_dataset(tbl, tmp_path / "data.lance")
+
+    torch_ds = LanceDataset(ds, batch_size=16, columns=["val"])
+
+    total_rows = 0
+    for batch in torch_ds:
+        assert batch.dtype == torch.bfloat16
+        total_rows += batch.shape[0]
+    assert total_rows == num_rows
+
+
+def test_bf16_to_tensor_zero_copy_without_nulls():
+    """Non-null bf16 arrays should alias the Arrow data buffer."""
+    ml_dtypes = pytest.importorskip("ml_dtypes")
+    from lance.arrow import BFloat16Array
+
+    values = np.array([1.0, 2.0, 3.0, 4.0], dtype=ml_dtypes.bfloat16)
+    arr = BFloat16Array.from_numpy(values).slice(1, 2)
+
+    tensor = _bf16_to_tensor(arr)
+
+    assert tensor.dtype == torch.bfloat16
+    assert torch.equal(
+        tensor.to(torch.float32),
+        torch.tensor([2.0, 3.0], dtype=torch.float32),
+    )
+    assert (
+        tensor.data_ptr() == arr.storage.buffers()[1].address + arr.storage.offset * 2
+    )
+
+
+def test_bf16_to_tensor_clones_when_nulls_present():
+    """Null replacement requires a writable tensor, so the Arrow buffer is cloned."""
+    arr = lance.arrow.bfloat16_array([1.0, None, 3.0])
+
+    tensor = _bf16_to_tensor(arr)
+
+    assert tensor.dtype == torch.bfloat16
+    assert (
+        tensor.data_ptr() != arr.storage.buffers()[1].address + arr.storage.offset * 2
+    )
+    assert tensor[0].to(torch.float32).item() == pytest.approx(1.0)
+    assert torch.isnan(tensor[1])
+    assert tensor[2].to(torch.float32).item() == pytest.approx(3.0)
+
+
+def test_safe_lance_dataset_worker_uses_dataset_options(tmp_path: Path):
+    """Worker processes must reopen the dataset with dataset_options.
+
+    Regression test for: worker init called lance.dataset(uri) without
+    dataset_options, silently dropping version, storage_options, etc.
+    """
+    tbl_v1 = pa.table({"id": pa.array([1, 2, 3], pa.int64())})
+    ds = lance.write_dataset(tbl_v1, tmp_path / "data.lance")
+    version_1 = ds.version
+
+    # Write a second version with different data so we can distinguish them.
+    tbl_v2 = pa.table({"id": pa.array([10, 20, 30], pa.int64())})
+    lance.write_dataset(tbl_v2, tmp_path / "data.lance", mode="overwrite")
+
+    # Pin to version 1 via dataset_options.
+    safe_ds = SafeLanceDataset(
+        str(tmp_path / "data.lance"),
+        dataset_options={"version": version_1},
+    )
+
+    # Simulate worker-process state: _ds is None so __getitems__ must reopen.
+    safe_ds._ds = None
+    rows = safe_ds.__getitems__([0, 1, 2])
+
+    assert [r["id"] for r in rows] == [1, 2, 3], (
+        "Worker reopened dataset without dataset_options (got version 2 data)"
+    )
diff --git a/python/src/dataset.rs b/python/src/dataset.rs
index d29b54f84df..1885b78bd64 100644
--- a/python/src/dataset.rs
+++ b/python/src/dataset.rs
@@ -3,7 +3,8 @@
 
 use std::collections::HashMap;
 use std::str;
-use std::sync::Arc;
+use std::sync::mpsc::{self, Receiver, SyncSender, TrySendError};
+use std::sync::{Arc, Mutex};
 
 use arrow::array::AsArray;
 use arrow::datatypes::UInt8Type;
@@ -11,6 +12,7 @@ use arrow::ffi_stream::ArrowArrayStreamReader;
 use arrow::pyarrow::*;
 use arrow_array::Array;
 use arrow_array::{RecordBatch, RecordBatchReader, make_array};
+use arrow_cast::cast_with_options;
 use arrow_data::ArrayData;
 use arrow_schema::{DataType, Schema as ArrowSchema};
 use async_trait::async_trait;
@@ -46,16 +48,17 @@ use lance::dataset::{
 };
 use lance::dataset::{ColumnAlteration, ProjectionRequest};
 use lance::dataset::{
-    Dataset as LanceDataset, DeleteBuilder, MergeInsertBuilder as LanceMergeInsertBuilder,
-    ReadParams, UncommittedMergeInsert, UpdateBuilder, Version, WhenMatched, WhenNotMatched,
-    WhenNotMatchedBySource, WriteMode, WriteParams,
+    Dataset as LanceDataset, DeleteBuilder, ExternalBlobMode,
+    MergeInsertBuilder as LanceMergeInsertBuilder, ReadParams, UncommittedMergeInsert,
+    UpdateBuilder, Version, WhenMatched, WhenNotMatched, WhenNotMatchedBySource, WriteMode,
+    WriteParams,
     fragment::FileFragment as LanceFileFragment,
     progress::WriteFragmentProgress,
     scanner::Scanner as LanceScanner,
     transaction::{Operation, Transaction},
 };
 use lance::index::vector::utils::get_vector_type;
-use lance::index::{DatasetIndexInternalExt, vector::VectorIndexParams};
+use lance::index::{DatasetIndexExt, DatasetIndexInternalExt, vector::VectorIndexParams};
 use lance::{dataset::builder::DatasetBuilder, index::vector::IndexFileVersion};
 use lance_arrow::as_fixed_size_list_array;
 use lance_core::Error;
@@ -67,8 +70,9 @@ use lance_index::scalar::inverted::query::{
     BooleanQuery, BoostQuery, FtsQuery, MatchQuery, MultiMatchQuery, Operator, PhraseQuery,
 };
 use lance_index::{
-    DatasetIndexExt, IndexParams, IndexType,
+    FtsPrewarmOptions, IndexParams, IndexType, PrewarmOptions,
     optimize::OptimizeOptions,
+    progress::{IndexBuildProgress, NoopIndexBuildProgress},
     scalar::{FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams},
     vector::{
         Query as VectorQuery, hnsw::builder::HnswBuildParams, ivf::IvfBuildParams,
@@ -78,7 +82,9 @@ use lance_index::{
 use lance_index::{
     infer_system_index_type, metrics::NoOpMetricsCollector, scalar::inverted::query::Occur,
 };
-use lance_io::object_store::ObjectStoreParams;
+use lance_io::object_store::{
+    LanceNamespaceStorageOptionsProvider, ObjectStoreParams, StorageOptionsAccessor,
+};
 use lance_linalg::distance::MetricType;
 use lance_table::format::{BasePath, Fragment, IndexMetadata};
 use lance_table::io::commit::CommitHandler;
@@ -111,6 +117,31 @@ pub mod stats;
 
 const DEFAULT_NPROBES: usize = 1;
 const LANCE_COMMIT_MESSAGE_KEY: &str = "__lance_commit_message";
+const INDEX_PROGRESS_QUEUE_SIZE: usize = 1024;
+
+fn read_blobs_to_python(
+    py: Python<'_>,
+    blobs: Vec<lance::dataset::ReadBlob>,
+) -> Vec<(u64, Py<PyBytes>)> {
+    blobs
+        .into_iter()
+        .map(|blob| (blob.row_address, PyBytes::new(py, &blob.data).unbind()))
+        .collect()
+}
+
+fn configure_read_blobs_builder(
+    mut builder: lance::dataset::ReadBlobsBuilder,
+    io_buffer_size: Option<u64>,
+    preserve_order: Option<bool>,
+) -> lance::dataset::ReadBlobsBuilder {
+    if let Some(bytes) = io_buffer_size {
+        builder = builder.with_io_buffer_size_bytes(bytes);
+    }
+    if let Some(preserve) = preserve_order {
+        builder = builder.preserve_order(preserve);
+    }
+    builder
+}
 
 fn convert_reader(reader: &Bound<PyAny>) -> PyResult<Box<dyn RecordBatchReader + Send>> {
     let py = reader.py();
@@ -482,7 +513,7 @@ impl Dataset {
     #[allow(clippy::too_many_arguments)]
     #[allow(deprecated)]
     #[new]
-    #[pyo3(signature=(uri, version=None, block_size=None, index_cache_size=None, metadata_cache_size=None, commit_handler=None, storage_options=None, manifest=None, metadata_cache_size_bytes=None, index_cache_size_bytes=None, read_params=None, session=None, storage_options_provider=None, namespace=None, table_id=None))]
+    #[pyo3(signature=(uri, version=None, block_size=None, index_cache_size=None, metadata_cache_size=None, commit_handler=None, storage_options=None, manifest=None, metadata_cache_size_bytes=None, index_cache_size_bytes=None, read_params=None, session=None, namespace_client=None, table_id=None, namespace_client_managed_versioning=false))]
     fn new(
         py: Python,
         uri: String,
@@ -497,9 +528,9 @@ impl Dataset {
         index_cache_size_bytes: Option<usize>,
         read_params: Option<&Bound<PyDict>>,
         session: Option<Session>,
-        storage_options_provider: Option<&Bound<'_, PyAny>>,
-        namespace: Option<&Bound<'_, PyAny>>,
+        namespace_client: Option<&Bound<'_, PyAny>>,
         table_id: Option<Vec<String>>,
+        namespace_client_managed_versioning: bool,
     ) -> PyResult<Self> {
         let mut params = ReadParams::default();
         if let Some(metadata_cache_size_bytes) = metadata_cache_size_bytes {
@@ -569,6 +600,9 @@ impl Dataset {
                 ));
             };
         }
+        // Save a copy of storage options for potential namespace-based credential refresh
+        let initial_storage_options = storage_options.clone();
+
         if let Some(mut storage_options) = storage_options {
             if let Some(user_agent) = storage_options.get_mut("user_agent") {
                 user_agent.push_str(&format!(" pylance/{}", env!("CARGO_PKG_VERSION")));
@@ -589,21 +623,34 @@ impl Dataset {
             builder = builder.with_session(session.inner.clone());
         }
 
-        // Add storage options provider if provided
-        if let Some(provider_obj) = storage_options_provider {
-            use crate::storage_options::py_object_to_storage_options_provider;
-            let provider = py_object_to_storage_options_provider(provider_obj)?;
-            builder = builder.with_storage_options_provider(provider);
-        }
+        // Set up namespace-based features if namespace_client and table_id are provided
+        if let (Some(ns_client), Some(tid)) = (&namespace_client, &table_id) {
+            let ns_client = extract_namespace_arc(py, ns_client)?;
 
-        // Set up namespace commit handler if namespace and table_id are provided
-        if let (Some(ns), Some(tid)) = (namespace, table_id) {
-            let ns_arc = extract_namespace_arc(py, ns)?;
-            let external_store = LanceNamespaceExternalManifestStore::new(ns_arc, tid);
-            let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
-                external_manifest_store: Arc::new(external_store),
-            });
-            builder = builder.with_commit_handler(commit_handler);
+            // Auto-create storage options provider from namespace client
+            // when storage_options are present (meaning credentials came from namespace.describe_table)
+            if initial_storage_options.is_some() {
+                let provider: Arc<dyn lance_io::object_store::StorageOptionsProvider> = Arc::new(
+                    LanceNamespaceStorageOptionsProvider::new(ns_client.clone(), tid.clone()),
+                );
+                // Create accessor with initial options and provider for credential refresh
+                let accessor = Arc::new(StorageOptionsAccessor::with_initial_and_provider(
+                    initial_storage_options.clone().unwrap_or_default(),
+                    provider,
+                ));
+                builder = builder.with_storage_options_accessor(accessor);
+            }
+
+            // Set up commit handler only if namespace manages versioning
+            if namespace_client_managed_versioning {
+                let external_store =
+                    LanceNamespaceExternalManifestStore::new(ns_client, tid.clone());
+                let commit_handler: Arc<dyn CommitHandler> =
+                    Arc::new(ExternalManifestCommitHandler {
+                        external_manifest_store: Arc::new(external_store),
+                    });
+                builder = builder.with_commit_handler(commit_handler);
+            }
         }
 
         let dataset = rt().block_on(Some(py), builder.load())?;
@@ -778,7 +825,7 @@ impl Dataset {
     }
 
     #[allow(clippy::too_many_arguments)]
-    #[pyo3(signature=(columns=None, columns_with_transform=None, filter=None, search_filter=None, prefilter=None, limit=None, offset=None, nearest=None, batch_size=None, io_buffer_size=None, batch_readahead=None, fragment_readahead=None, scan_in_order=None, fragments=None, with_row_id=None, with_row_address=None, use_stats=None, substrait_filter=None, fast_search=None, full_text_query=None, late_materialization=None, blob_handling=None, use_scalar_index=None, include_deleted_rows=None, scan_stats_callback=None, strict_batch_size=None, order_by=None, disable_scoring_autoprojection=None, substrait_aggregate=None))]
+    #[pyo3(signature=(columns=None, columns_with_transform=None, filter=None, search_filter=None, prefilter=None, limit=None, offset=None, nearest=None, batch_size=None, batch_size_bytes=None, io_buffer_size=None, batch_readahead=None, fragment_readahead=None, scan_in_order=None, fragments=None, with_row_id=None, with_row_address=None, use_stats=None, substrait_filter=None, fast_search=None, full_text_query=None, late_materialization=None, blob_handling=None, use_scalar_index=None, include_deleted_rows=None, scan_stats_callback=None, strict_batch_size=None, order_by=None, disable_scoring_autoprojection=None, substrait_aggregate=None))]
     fn scanner(
         self_: PyRef<'_, Self>,
         columns: Option<Vec<String>>,
@@ -790,6 +837,7 @@ impl Dataset {
         offset: Option<i64>,
         nearest: Option<&Bound<PyDict>>,
         batch_size: Option<usize>,
+        batch_size_bytes: Option<u64>,
         io_buffer_size: Option<u64>,
         batch_readahead: Option<usize>,
         fragment_readahead: Option<usize>,
@@ -934,6 +982,9 @@ impl Dataset {
         if let Some(batch_size) = batch_size {
             scanner.batch_size(batch_size);
         }
+        if let Some(batch_size_bytes) = batch_size_bytes {
+            scanner.batch_size_bytes(batch_size_bytes);
+        }
         if let Some(io_buffer_size) = io_buffer_size {
             scanner.io_buffer_size(io_buffer_size);
         }
@@ -1220,6 +1271,90 @@ impl Dataset {
         Ok(blobs.into_iter().map(LanceBlobFile::from).collect())
     }
 
+    #[pyo3(signature=(
+        row_ids,
+        blob_column,
+        io_buffer_size=None,
+        preserve_order=None
+    ))]
+    fn read_blobs(
+        self_: PyRef<'_, Self>,
+        row_ids: Vec<u64>,
+        blob_column: &str,
+        io_buffer_size: Option<u64>,
+        preserve_order: Option<bool>,
+    ) -> PyResult<Vec<(u64, Py<PyBytes>)>> {
+        let builder = configure_read_blobs_builder(
+            self_
+                .ds
+                .read_blobs(blob_column)
+                .infer_error()?
+                .with_row_ids(row_ids),
+            io_buffer_size,
+            preserve_order,
+        );
+        let blobs = rt()
+            .block_on(Some(self_.py()), builder.execute())?
+            .infer_error()?;
+        Ok(read_blobs_to_python(self_.py(), blobs))
+    }
+
+    #[pyo3(signature=(
+        row_addresses,
+        blob_column,
+        io_buffer_size=None,
+        preserve_order=None
+    ))]
+    fn read_blobs_by_addresses(
+        self_: PyRef<'_, Self>,
+        row_addresses: Vec<u64>,
+        blob_column: &str,
+        io_buffer_size: Option<u64>,
+        preserve_order: Option<bool>,
+    ) -> PyResult<Vec<(u64, Py<PyBytes>)>> {
+        let builder = configure_read_blobs_builder(
+            self_
+                .ds
+                .read_blobs(blob_column)
+                .infer_error()?
+                .with_row_addresses(row_addresses),
+            io_buffer_size,
+            preserve_order,
+        );
+        let blobs = rt()
+            .block_on(Some(self_.py()), builder.execute())?
+            .infer_error()?;
+        Ok(read_blobs_to_python(self_.py(), blobs))
+    }
+
+    #[pyo3(signature=(
+        row_indices,
+        blob_column,
+        io_buffer_size=None,
+        preserve_order=None
+    ))]
+    fn read_blobs_by_indices(
+        self_: PyRef<'_, Self>,
+        row_indices: Vec<u64>,
+        blob_column: &str,
+        io_buffer_size: Option<u64>,
+        preserve_order: Option<bool>,
+    ) -> PyResult<Vec<(u64, Py<PyBytes>)>> {
+        let builder = configure_read_blobs_builder(
+            self_
+                .ds
+                .read_blobs(blob_column)
+                .infer_error()?
+                .with_row_indices(row_indices),
+            io_buffer_size,
+            preserve_order,
+        );
+        let blobs = rt()
+            .block_on(Some(self_.py()), builder.execute())?
+            .infer_error()?;
+        Ok(read_blobs_to_python(self_.py(), blobs))
+    }
+
     #[pyo3(signature = (row_slices, columns = None, batch_readahead = 10))]
     fn take_scan(
         &self,
@@ -1459,7 +1594,7 @@ impl Dataset {
 
     /// Fetches the currently checked out version of the dataset.
     fn version(&self) -> PyResult<u64> {
-        Ok(self.ds.version().version)
+        Ok(self.ds.version_id())
     }
 
     fn latest_version(self_: PyRef<'_, Self>) -> PyResult<u64> {
@@ -1746,6 +1881,7 @@ impl Dataset {
         for (name, meta) in branches.iter() {
             let dict = PyDict::new(py);
             dict.set_item("parent_branch", meta.parent_branch.clone())?;
+            dict.set_item("branch_identifier", meta.identifier.version_mapping.clone())?;
             dict.set_item("parent_version", meta.parent_version)?;
             dict.set_item("create_at", meta.create_at)?;
             dict.set_item("manifest_size", meta.manifest_size)?;
@@ -1780,6 +1916,7 @@ impl Dataset {
         for (name, meta) in ordered.into_iter() {
             let dict = PyDict::new(py);
             dict.set_item("parent_branch", meta.parent_branch.clone())?;
+            dict.set_item("branch_identifier", meta.identifier.version_mapping.clone())?;
             dict.set_item("parent_version", meta.parent_version)?;
             dict.set_item("create_at", meta.create_at)?;
             dict.set_item("manifest_size", meta.manifest_size)?;
@@ -1941,8 +2078,11 @@ impl Dataset {
                     if let Some(prefix_only) = kwargs.get_item("prefix_only")? {
                         params = params.ngram_prefix_only(prefix_only.extract()?);
                     }
-                    if let Some(skip_merge) = kwargs.get_item("skip_merge")? {
-                        params = params.skip_merge(skip_merge.extract()?);
+                    if let Some(memory_limit) = kwargs.get_item("memory_limit")? {
+                        params = params.memory_limit_mb(memory_limit.extract()?);
+                    }
+                    if let Some(num_workers) = kwargs.get_item("num_workers")? {
+                        params = params.num_workers(num_workers.extract()?);
                     }
                 }
                 Box::new(params)
@@ -1961,6 +2101,7 @@ impl Dataset {
         let replace = replace.unwrap_or(true);
         let train = train.unwrap_or(true); // Default to true for backward compatibility
 
+        let mut progress_handler = Self::make_index_progress_handler_from_kwargs(kwargs)?;
         let mut new_self = self.ds.as_ref().clone();
         let mut builder = new_self
             .create_index_builder(&columns, idx_type, params.as_ref())
@@ -1997,6 +2138,9 @@ impl Dataset {
         if let Some(index_uuid) = index_uuid {
             builder = builder.index_uuid(index_uuid);
         }
+        if let Some(progress_handler) = progress_handler.as_ref() {
+            builder = builder.progress(progress_handler.progress.clone());
+        }
 
         use std::future::IntoFuture;
 
@@ -2004,11 +2148,13 @@ impl Dataset {
         let index_metadata = if has_fragment_ids {
             // For fragment-level indexing, use execute_uncommitted
             // Note: We don't update self.ds here as the index is not committed
-            rt().block_on(None, builder.execute_uncommitted())?
+            Self::run_index_future(builder.execute_uncommitted(), progress_handler.as_mut())?
                 .infer_error()?
         } else {
             // For regular indexing, use the standard execute path
-            let index_metadata = rt().block_on(None, builder.into_future())?.infer_error()?;
+            let index_metadata =
+                Self::run_index_future(builder.into_future(), progress_handler.as_mut())?
+                    .infer_error()?;
             self.ds = Arc::new(new_self);
             index_metadata
         };
@@ -2016,6 +2162,40 @@ impl Dataset {
         Ok(PyLance(index_metadata))
     }
 
+    fn merge_existing_index_segments(
+        &self,
+        segments: Vec<PyLance<IndexMetadata>>,
+    ) -> PyResult<PyLance<IndexMetadata>> {
+        let merged = rt()
+            .block_on(
+                None,
+                self.ds
+                    .merge_existing_index_segments(segments.into_iter().map(|s| s.0).collect()),
+            )?
+            .infer_error()?;
+        Ok(PyLance(merged))
+    }
+
+    fn commit_existing_index_segments(
+        &mut self,
+        index_name: &str,
+        column: &str,
+        segments: Vec<PyLance<IndexMetadata>>,
+    ) -> PyResult<()> {
+        let mut new_self = self.ds.as_ref().clone();
+        rt().block_on(
+            None,
+            new_self.commit_existing_index_segments(
+                index_name,
+                column,
+                segments.into_iter().map(|segment| segment.0).collect(),
+            ),
+        )?
+        .infer_error()?;
+        self.ds = Arc::new(new_self);
+        Ok(())
+    }
+
     fn drop_index(&mut self, name: &str) -> PyResult<()> {
         let mut new_self = self.ds.as_ref().clone();
         rt().block_on(None, new_self.drop_index(name))?
@@ -2025,23 +2205,51 @@ impl Dataset {
         Ok(())
     }
 
-    fn prewarm_index(&self, name: &str) -> PyResult<()> {
-        rt().block_on(None, self.ds.prewarm_index(name))?
-            .infer_error()
+    #[pyo3(signature = (name, *, with_position = false))]
+    fn prewarm_index(&self, name: &str, with_position: bool) -> PyResult<()> {
+        rt().block_on(None, async {
+            if with_position {
+                self.ds
+                    .prewarm_index_with_options(
+                        name,
+                        &PrewarmOptions::Fts(FtsPrewarmOptions::new().with_position(true)),
+                    )
+                    .await
+            } else {
+                self.ds.prewarm_index(name).await
+            }
+        })?
+        .infer_error()
     }
 
-    #[pyo3(signature = (index_uuid, index_type, batch_readhead=None))]
+    #[pyo3(signature = (index_uuid, index_type, batch_readhead=None, progress_callback=None))]
     fn merge_index_metadata(
         &self,
         index_uuid: &str,
         index_type: &str,
         batch_readhead: Option<usize>,
+        progress_callback: Option<&Bound<'_, PyAny>>,
     ) -> PyResult<()> {
-        rt().block_on(None, async {
-            self.ds
-                .merge_index_metadata(index_uuid, IndexType::try_from(index_type)?, batch_readhead)
-                .await
-        })?
+        let mut progress_handler =
+            Self::make_index_progress_handler_from_callback(progress_callback)?;
+        let progress: Arc<dyn IndexBuildProgress> = progress_handler
+            .as_ref()
+            .map(|handler| handler.progress.clone() as Arc<dyn IndexBuildProgress>)
+            .unwrap_or_else(|| Arc::new(NoopIndexBuildProgress));
+
+        Self::run_index_future(
+            async {
+                self.ds
+                    .merge_index_metadata(
+                        index_uuid,
+                        IndexType::try_from(index_type)?,
+                        batch_readhead,
+                        progress,
+                    )
+                    .await
+            },
+            progress_handler.as_mut(),
+        )?
         .map_err(|err| PyValueError::new_err(err.to_string()))
     }
 
@@ -2135,21 +2343,21 @@ impl Dataset {
 
     #[allow(clippy::too_many_arguments)]
     #[staticmethod]
-    #[pyo3(signature = (dest, operation, read_version = None, commit_lock = None, storage_options = None, storage_options_provider = None, enable_v2_manifest_paths = None, detached = None, max_retries = None, commit_message = None, enable_stable_row_ids = None, namespace = None, table_id = None))]
+    #[pyo3(signature = (dest, operation, read_version = None, commit_lock = None, storage_options = None, enable_v2_manifest_paths = None, detached = None, max_retries = None, commit_message = None, enable_stable_row_ids = None, namespace_client = None, table_id = None, namespace_client_managed_versioning = false))]
     fn commit(
         dest: PyWriteDest,
         operation: PyLance<Operation>,
         read_version: Option<u64>,
         commit_lock: Option<&Bound<'_, PyAny>>,
         storage_options: Option<HashMap<String, String>>,
-        storage_options_provider: Option<&Bound<'_, PyAny>>,
         enable_v2_manifest_paths: Option<bool>,
         detached: Option<bool>,
         max_retries: Option<u32>,
         commit_message: Option<String>,
         enable_stable_row_ids: Option<bool>,
-        namespace: Option<&Bound<'_, PyAny>>,
+        namespace_client: Option<&Bound<'_, PyAny>>,
         table_id: Option<Vec<String>>,
+        namespace_client_managed_versioning: bool,
     ) -> PyResult<Self> {
         let mut transaction = Transaction::new(read_version.unwrap_or_default(), operation.0, None);
 
@@ -2165,37 +2373,35 @@ impl Dataset {
             PyLance(transaction),
             commit_lock,
             storage_options,
-            storage_options_provider,
             enable_v2_manifest_paths,
             detached,
             max_retries,
             enable_stable_row_ids,
-            namespace,
+            namespace_client,
             table_id,
+            namespace_client_managed_versioning,
         )
     }
 
     #[allow(clippy::too_many_arguments)]
     #[allow(deprecated)]
     #[staticmethod]
-    #[pyo3(signature = (dest, transaction, commit_lock = None, storage_options = None, storage_options_provider = None, enable_v2_manifest_paths = None, detached = None, max_retries = None, enable_stable_row_ids = None, namespace = None, table_id = None))]
+    #[pyo3(signature = (dest, transaction, commit_lock = None, storage_options = None, enable_v2_manifest_paths = None, detached = None, max_retries = None, enable_stable_row_ids = None, namespace_client = None, table_id = None, namespace_client_managed_versioning = false))]
     fn commit_transaction(
         dest: PyWriteDest,
         transaction: PyLance<Transaction>,
         commit_lock: Option<&Bound<'_, PyAny>>,
         storage_options: Option<HashMap<String, String>>,
-        storage_options_provider: Option<&Bound<'_, PyAny>>,
         enable_v2_manifest_paths: Option<bool>,
         detached: Option<bool>,
         max_retries: Option<u32>,
         enable_stable_row_ids: Option<bool>,
-        namespace: Option<&Bound<'_, PyAny>>,
+        namespace_client: Option<&Bound<'_, PyAny>>,
         table_id: Option<Vec<String>>,
+        namespace_client_managed_versioning: bool,
     ) -> PyResult<Self> {
-        let accessor = crate::storage_options::create_accessor_from_python(
-            storage_options.clone(),
-            storage_options_provider,
-        )?;
+        let accessor =
+            crate::storage_options::create_accessor_from_storage_options(storage_options.clone())?;
 
         let object_store_params = if accessor.is_some() {
             Some(ObjectStoreParams {
@@ -2206,7 +2412,8 @@ impl Dataset {
             None
         };
 
-        // Create commit_handler: prefer user-provided commit_lock, then namespace-based handler
+        // Create commit_handler: prefer user-provided commit_lock, then namespace client-based handler
+        // (only if namespace_client_managed_versioning is true)
         let commit_handler: Option<Arc<dyn CommitHandler>> =
             if let Some(commit_lock) = commit_lock.as_ref() {
                 // User provided a commit_lock
@@ -2215,10 +2422,13 @@ impl Dataset {
                         .into_py_any(commit_lock.py())
                         .map(|cl| Arc::new(PyCommitLock::new(cl)) as Arc<dyn CommitHandler>)?,
                 )
-            } else if let (Some(ns), Some(tid)) = (namespace, table_id) {
-                // Create ExternalManifestCommitHandler from namespace and table_id
-                let ns_arc = extract_namespace_arc(ns.py(), ns)?;
-                let external_store = LanceNamespaceExternalManifestStore::new(ns_arc, tid);
+            } else if namespace_client_managed_versioning
+                && let (Some(ns_client), Some(tid)) = (namespace_client, table_id)
+            {
+                // Create ExternalManifestCommitHandler from namespace client and table_id
+                // only when namespace manages versioning
+                let ns_client = extract_namespace_arc(ns_client.py(), ns_client)?;
+                let external_store = LanceNamespaceExternalManifestStore::new(ns_client, tid);
                 Some(Arc::new(ExternalManifestCommitHandler {
                     external_manifest_store: Arc::new(external_store),
                 }) as Arc<dyn CommitHandler>)
@@ -2260,21 +2470,18 @@ impl Dataset {
     #[allow(clippy::too_many_arguments)]
     #[allow(deprecated)]
     #[staticmethod]
-    #[pyo3(signature = (dest, transactions, commit_lock = None, storage_options = None, storage_options_provider = None, enable_v2_manifest_paths = None, detached = None, max_retries = None))]
+    #[pyo3(signature = (dest, transactions, commit_lock = None, storage_options = None, enable_v2_manifest_paths = None, detached = None, max_retries = None))]
     fn commit_batch(
         dest: PyWriteDest,
         transactions: Vec<PyLance<Transaction>>,
         commit_lock: Option<&Bound<'_, PyAny>>,
         storage_options: Option<HashMap<String, String>>,
-        storage_options_provider: Option<&Bound<'_, PyAny>>,
         enable_v2_manifest_paths: Option<bool>,
         detached: Option<bool>,
         max_retries: Option<u32>,
     ) -> PyResult<(Self, PyLance<Transaction>)> {
-        let accessor = crate::storage_options::create_accessor_from_python(
-            storage_options.clone(),
-            storage_options_provider,
-        )?;
+        let accessor =
+            crate::storage_options::create_accessor_from_storage_options(storage_options.clone())?;
 
         let object_store_params = if accessor.is_some() {
             Some(ObjectStoreParams {
@@ -2880,6 +3087,190 @@ impl PyWriteDest {
     }
 }
 
+#[derive(Debug, Clone, Copy)]
+enum IndexProgressEventType {
+    Start,
+    Progress,
+    Complete,
+}
+
+impl IndexProgressEventType {
+    fn as_str(self) -> &'static str {
+        match self {
+            Self::Start => "start",
+            Self::Progress => "progress",
+            Self::Complete => "complete",
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+struct IndexProgressEvent {
+    event: IndexProgressEventType,
+    stage: String,
+    completed: Option<u64>,
+    total: Option<u64>,
+    unit: String,
+}
+
+#[derive(Debug, Default)]
+struct ActiveIndexProgressStage {
+    stage: Option<String>,
+    completed: Option<u64>,
+    total: Option<u64>,
+    unit: String,
+}
+
+#[derive(Debug)]
+struct PyIndexBuildProgress {
+    sender: SyncSender<IndexProgressEvent>,
+    active_stage: Mutex<ActiveIndexProgressStage>,
+}
+
+impl PyIndexBuildProgress {
+    fn new(sender: SyncSender<IndexProgressEvent>) -> Self {
+        Self {
+            sender,
+            active_stage: Mutex::new(ActiveIndexProgressStage::default()),
+        }
+    }
+
+    /// Send a lifecycle event (start/complete) with blocking semantics.
+    ///
+    /// These events are rare relative to the channel capacity
+    /// (`INDEX_PROGRESS_QUEUE_SIZE`), so blocking here will not stall
+    /// the tokio runtime in practice.  Progress events use the
+    /// non-blocking `try_send_progress_event` instead.
+    fn send_event(&self, event: IndexProgressEvent) -> lance::Result<()> {
+        self.sender.send(event).map_err(|_| {
+            Error::invalid_input("Index progress callback receiver disconnected".to_string())
+        })
+    }
+
+    fn try_send_progress_event(&self, event: IndexProgressEvent) -> lance::Result<()> {
+        match self.sender.try_send(event) {
+            Ok(()) | Err(TrySendError::Full(_)) => Ok(()),
+            Err(TrySendError::Disconnected(_)) => Err(Error::invalid_input(
+                "Index progress callback receiver disconnected".to_string(),
+            )),
+        }
+    }
+}
+
+#[async_trait]
+impl IndexBuildProgress for PyIndexBuildProgress {
+    async fn stage_start(&self, stage: &str, total: Option<u64>, unit: &str) -> lance::Result<()> {
+        {
+            let mut active_stage = self.active_stage.lock().unwrap();
+            *active_stage = ActiveIndexProgressStage {
+                stage: Some(stage.to_string()),
+                completed: Some(0),
+                total,
+                unit: unit.to_string(),
+            };
+        }
+
+        self.send_event(IndexProgressEvent {
+            event: IndexProgressEventType::Start,
+            stage: stage.to_string(),
+            completed: Some(0),
+            total,
+            unit: unit.to_string(),
+        })
+    }
+
+    async fn stage_progress(&self, stage: &str, completed: u64) -> lance::Result<()> {
+        let event = {
+            let mut active_stage = self.active_stage.lock().unwrap();
+            if active_stage.stage.is_none() {
+                active_stage.stage = Some(stage.to_string());
+            }
+            active_stage.completed = Some(completed);
+            IndexProgressEvent {
+                event: IndexProgressEventType::Progress,
+                stage: stage.to_string(),
+                completed: Some(completed),
+                total: active_stage.total,
+                unit: active_stage.unit.clone(),
+            }
+        };
+
+        self.try_send_progress_event(event)
+    }
+
+    async fn stage_complete(&self, stage: &str) -> lance::Result<()> {
+        let event = {
+            let mut active_stage = self.active_stage.lock().unwrap();
+            let completed = active_stage.completed.or(active_stage.total);
+            let total = active_stage.total;
+            let unit = active_stage.unit.clone();
+            *active_stage = ActiveIndexProgressStage::default();
+            IndexProgressEvent {
+                event: IndexProgressEventType::Complete,
+                stage: stage.to_string(),
+                completed,
+                total,
+                unit,
+            }
+        };
+
+        self.send_event(event)
+    }
+}
+
+struct IndexProgressDispatcher {
+    callback: Py<PyAny>,
+    index_progress_cls: Py<PyAny>,
+    receiver: Receiver<IndexProgressEvent>,
+}
+
+impl IndexProgressDispatcher {
+    fn new(
+        py: Python<'_>,
+        callback: Py<PyAny>,
+        receiver: Receiver<IndexProgressEvent>,
+    ) -> PyResult<Self> {
+        let index_progress_cls = py
+            .import("lance.progress")?
+            .getattr("IndexProgress")?
+            .unbind();
+        Ok(Self {
+            callback,
+            index_progress_cls,
+            receiver,
+        })
+    }
+
+    fn drain(&mut self) -> PyResult<()> {
+        while let Ok(event) = self.receiver.try_recv() {
+            self.dispatch(event)?;
+        }
+        Ok(())
+    }
+
+    fn dispatch(&self, event: IndexProgressEvent) -> PyResult<()> {
+        Python::attach(|py| {
+            let progress = self.index_progress_cls.call1(
+                py,
+                (
+                    event.event.as_str(),
+                    event.stage,
+                    event.completed,
+                    event.total,
+                    event.unit,
+                ),
+            )?;
+            self.callback.call1(py, (progress,))?;
+            Ok(())
+        })
+    }
+}
+
+struct IndexProgressHandler {
+    progress: Arc<PyIndexBuildProgress>,
+    dispatcher: IndexProgressDispatcher,
+}
+
 impl Dataset {
     fn transform_ref(&self, reference: Option<Bound<PyAny>>) -> PyResult<Ref> {
         if let Some(reference) = reference {
@@ -2907,7 +3298,7 @@ impl Dataset {
         } else {
             Ok(Ref::Version(
                 self.ds.manifest.branch.clone(),
-                Some(self.ds.version().version),
+                Some(self.ds.version_id()),
             ))
         }
     }
@@ -2972,6 +3363,62 @@ impl Dataset {
             });
         }))
     }
+
+    fn make_index_progress_handler(
+        py: Python<'_>,
+        callback: Py<PyAny>,
+    ) -> PyResult<IndexProgressHandler> {
+        if !callback.bind(py).is_callable() {
+            return Err(PyValueError::new_err("Progress callback must be callable"));
+        }
+
+        let (sender, receiver) = mpsc::sync_channel(INDEX_PROGRESS_QUEUE_SIZE);
+        let progress = Arc::new(PyIndexBuildProgress::new(sender));
+        let dispatcher = IndexProgressDispatcher::new(py, callback, receiver)?;
+        Ok(IndexProgressHandler {
+            progress,
+            dispatcher,
+        })
+    }
+
+    fn make_index_progress_handler_from_kwargs(
+        kwargs: Option<&Bound<'_, PyDict>>,
+    ) -> PyResult<Option<IndexProgressHandler>> {
+        let Some(kwargs) = kwargs else {
+            return Ok(None);
+        };
+
+        let Some(callback) = get_dict_opt::<Py<PyAny>>(kwargs, "progress_callback")? else {
+            return Ok(None);
+        };
+
+        Self::make_index_progress_handler(kwargs.py(), callback).map(Some)
+    }
+
+    fn make_index_progress_handler_from_callback(
+        callback: Option<&Bound<'_, PyAny>>,
+    ) -> PyResult<Option<IndexProgressHandler>> {
+        let Some(callback) = callback else {
+            return Ok(None);
+        };
+
+        Self::make_index_progress_handler(callback.py(), callback.clone().unbind()).map(Some)
+    }
+
+    fn run_index_future<F>(
+        future: F,
+        progress_handler: Option<&mut IndexProgressHandler>,
+    ) -> PyResult<F::Output>
+    where
+        F: std::future::Future + Send,
+        F::Output: Send,
+    {
+        if let Some(progress_handler) = progress_handler {
+            rt().block_on_pumping(None, future, || progress_handler.dispatcher.drain())
+        } else {
+            rt().block_on(None, future)
+        }
+    }
 }
 
 #[pyfunction(name = "_write_dataset")]
@@ -3076,20 +3523,35 @@ pub fn get_write_params(options: &Bound<'_, PyDict>) -> PyResult<Option<WritePar
         }
 
         let storage_options = get_dict_opt::<HashMap<String, String>>(options, "storage_options")?;
-        let storage_options_provider =
-            get_dict_opt::<Py<PyAny>>(options, "storage_options_provider")?;
-
-        if storage_options.is_some() || storage_options_provider.is_some() {
-            let accessor = crate::storage_options::create_accessor_from_python(
-                storage_options,
-                storage_options_provider
-                    .as_ref()
-                    .map(|py_obj| py_obj.bind(options.py())),
-            )?;
-            p.store_params = Some(ObjectStoreParams {
-                storage_options_accessor: accessor,
-                ..Default::default()
-            });
+
+        // Extract namespace_client and table_id for storage options provider creation
+        let namespace_client_opt = get_dict_opt::<Bound<PyAny>>(options, "namespace_client")?;
+        let table_id_opt = get_dict_opt::<Vec<String>>(options, "table_id")?;
+
+        if let Some(so) = storage_options.clone() {
+            // If namespace_client and table_id are provided, create storage options provider from them
+            if let (Some(ns_client), Some(table_id)) =
+                (namespace_client_opt.as_ref(), table_id_opt.as_ref())
+            {
+                let ns_client = extract_namespace_arc(options.py(), ns_client)?;
+                let provider: Arc<dyn lance_io::object_store::StorageOptionsProvider> = Arc::new(
+                    LanceNamespaceStorageOptionsProvider::new(ns_client, table_id.clone()),
+                );
+                let accessor = Arc::new(StorageOptionsAccessor::with_initial_and_provider(
+                    so, provider,
+                ));
+                p.store_params = Some(ObjectStoreParams {
+                    storage_options_accessor: Some(accessor),
+                    ..Default::default()
+                });
+            } else {
+                // No namespace, just use storage options directly
+                let accessor = Arc::new(StorageOptionsAccessor::with_static_options(so));
+                p.store_params = Some(ObjectStoreParams {
+                    storage_options_accessor: Some(accessor),
+                    ..Default::default()
+                });
+            }
         }
 
         if let Some(enable_stable_row_ids) = get_dict_opt::<bool>(options, "enable_stable_row_ids")?
@@ -3158,6 +3620,14 @@ pub fn get_write_params(options: &Bound<'_, PyDict>) -> PyResult<Option<WritePar
         {
             p = p.with_allow_external_blob_outside_bases(allow_external);
         }
+        if let Some(external_blob_mode) = get_dict_opt::<String>(options, "external_blob_mode")? {
+            p = p.with_external_blob_mode(
+                ExternalBlobMode::try_from(external_blob_mode.as_str()).infer_error()?,
+            );
+        }
+        if let Some(max_bytes) = get_dict_opt::<usize>(options, "blob_pack_file_size_threshold")? {
+            p = p.with_blob_pack_file_size_threshold(max_bytes);
+        }
 
         // Handle properties
         if let Some(props) =
@@ -3174,21 +3644,22 @@ pub fn get_write_params(options: &Bound<'_, PyDict>) -> PyResult<Option<WritePar
             p.transaction_properties = Some(Arc::new(new_props));
         }
 
-        // Handle namespace and table_id for managed versioning (external manifest store)
-        // Only set if commit_handler is not already set by user
-        if p.commit_handler.is_none() {
-            let namespace_opt = get_dict_opt::<Bound<PyAny>>(options, "namespace")?;
-            let table_id_opt = get_dict_opt::<Vec<String>>(options, "table_id")?;
-
-            if let (Some(ns), Some(table_id)) = (namespace_opt, table_id_opt) {
-                let ns_arc = extract_namespace_arc(options.py(), &ns)?;
-                let external_store = LanceNamespaceExternalManifestStore::new(ns_arc, table_id);
-                let commit_handler: Arc<dyn CommitHandler> =
-                    Arc::new(ExternalManifestCommitHandler {
-                        external_manifest_store: Arc::new(external_store),
-                    });
-                p.commit_handler = Some(commit_handler);
-            }
+        // Handle namespace_client and table_id for managed versioning (external manifest store)
+        // Only set if commit_handler is not already set by user and namespace_client_managed_versioning is true
+        let namespace_client_managed_versioning =
+            get_dict_opt::<bool>(options, "namespace_client_managed_versioning")?.unwrap_or(false);
+        if p.commit_handler.is_none()
+            && namespace_client_managed_versioning
+            && let (Some(ns_client), Some(table_id)) =
+                (namespace_client_opt.as_ref(), table_id_opt.as_ref())
+        {
+            let ns_client = extract_namespace_arc(options.py(), ns_client)?;
+            let external_store =
+                LanceNamespaceExternalManifestStore::new(ns_client, table_id.clone());
+            let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
+                external_manifest_store: Arc::new(external_store),
+            });
+            p.commit_handler = Some(commit_handler);
         }
 
         Some(p)
@@ -3251,14 +3722,13 @@ fn prepare_vector_index_params(
             // as the vectors that will be indexed.
             let mut centroids: Arc<dyn Array> = batch.column(0).clone();
             if centroids.data_type() != column_type {
-                centroids = lance_arrow::cast::cast_with_options(
-                    centroids.as_ref(),
-                    column_type,
-                    &Default::default(),
-                )
-                .map_err(|e| {
-                    PyValueError::new_err(format!("Failed to cast centroids to column type: {}", e))
-                })?;
+                centroids = cast_with_options(centroids.as_ref(), column_type, &Default::default())
+                    .map_err(|e| {
+                        PyValueError::new_err(format!(
+                            "Failed to cast centroids to column type: {}",
+                            e
+                        ))
+                    })?;
             }
             let centroids = as_fixed_size_list_array(centroids.as_ref());
 
diff --git a/python/src/dataset/blob.rs b/python/src/dataset/blob.rs
index 569a0e3ed8e..1f6272075f2 100644
--- a/python/src/dataset/blob.rs
+++ b/python/src/dataset/blob.rs
@@ -62,6 +62,23 @@ impl LanceBlobFile {
         Ok(PyBytes::new(py, &data))
     }
 
+    /// Read a blob-local byte range without changing the current cursor.
+    pub fn read_range<'a>(
+        &'a self,
+        py: Python<'a>,
+        offset: u64,
+        length: usize,
+    ) -> PyResult<Bound<'a, PyBytes>> {
+        let end = offset
+            .checked_add(length as u64)
+            .ok_or_else(|| PyValueError::new_err("offset + length overflowed"))?;
+        let inner = self.inner.clone();
+        let data = rt()
+            .block_on(Some(py), inner.read_range(offset..end))?
+            .infer_error()?;
+        Ok(PyBytes::new(py, &data))
+    }
+
     pub fn read_into(&self, dst: Bound<'_, PyByteArray>) -> PyResult<usize> {
         let inner = self.inner.clone();
 
diff --git a/python/src/dataset/optimize.rs b/python/src/dataset/optimize.rs
index d2e60ec0a00..4af47cbf0df 100644
--- a/python/src/dataset/optimize.rs
+++ b/python/src/dataset/optimize.rs
@@ -23,8 +23,12 @@ use pyo3::{exceptions::PyNotImplementedError, pyclass::CompareOp, types::PyTuple
 
 use super::*;
 
-fn parse_compaction_options(options: &Bound<'_, PyDict>) -> PyResult<CompactionOptions> {
-    let mut opts = CompactionOptions::default();
+fn parse_compaction_options(
+    options: &Bound<'_, PyDict>,
+    config: &std::collections::HashMap<String, String>,
+) -> PyResult<CompactionOptions> {
+    let mut opts = CompactionOptions::from_dataset_config(config)
+        .map_err(|e| PyValueError::new_err(e.to_string()))?;
 
     for (key, value) in options.into_iter() {
         let key: String = key.extract()?;
@@ -66,6 +70,9 @@ fn parse_compaction_options(options: &Bound<'_, PyDict>) -> PyResult<CompactionO
             "binary_copy_read_batch_bytes" => {
                 opts.binary_copy_read_batch_bytes = value.extract()?;
             }
+            "max_source_fragments" => {
+                opts.max_source_fragments = value.extract()?;
+            }
             _ => {
                 return Err(PyValueError::new_err(format!(
                     "Invalid compaction option: {}",
@@ -485,7 +492,8 @@ impl PyCompaction {
         // Make sure we parse the options within a scoped GIL context, so we
         // aren't holding the GIL while blocking the thread on the operation.
         let options = options.downcast::<PyDict>()?;
-        let opts = parse_compaction_options(options)?;
+        let config = dataset.ds.manifest.config.clone();
+        let opts = parse_compaction_options(options, &config)?;
         let mut new_ds = dataset.ds.as_ref().clone();
         let fut = compact_files(&mut new_ds, opts, None);
         let metrics = rt().block_on(None, async move {
@@ -518,7 +526,8 @@ impl PyCompaction {
         // Make sure we parse the options within a scoped GIL context, so we
         // aren't holding the GIL while blocking the thread on the operation.
         let options = options.downcast::<PyDict>()?;
-        let opts = parse_compaction_options(options)?;
+        let config = dataset.ds.manifest.config.clone();
+        let opts = parse_compaction_options(options, &config)?;
         let plan = rt()
             .block_on(None, async move {
                 plan_compaction(dataset.ds.as_ref(), &opts).await
diff --git a/python/src/debug.rs b/python/src/debug.rs
index 7bc073dff17..bc0e09aba2b 100644
--- a/python/src/debug.rs
+++ b/python/src/debug.rs
@@ -61,8 +61,8 @@ impl PrettyPrintableFragment {
                 let schema = schema.project_by_ids(&file.fields, false);
                 PrettyPrintableDataFile {
                     path: file.path.clone(),
-                    fields: file.fields.clone(),
-                    column_indices: file.column_indices.clone(),
+                    fields: file.fields.to_vec(),
+                    column_indices: file.column_indices.to_vec(),
                     schema,
                     major_version: file.file_major_version,
                     minor_version: file.file_minor_version,
diff --git a/python/src/executor.rs b/python/src/executor.rs
index c3393b0ec72..e2e573a8193 100644
--- a/python/src/executor.rs
+++ b/python/src/executor.rs
@@ -173,6 +173,80 @@ impl BackgroundExecutor {
         }
     }
 
+    /// Block on a future, periodically draining a caller-provided pump.
+    ///
+    /// This is intended for progress callbacks where the future emits events from
+    /// background tasks and the caller needs to run Python code while waiting for
+    /// the future to complete.
+    ///
+    /// Unlike [`block_on`], this method does **not** wrap the future in
+    /// `result_or_interrupt`, so keyboard interrupts are only checked on the
+    /// main thread between poll intervals (`SIGNAL_CHECK_INTERVAL`).  This is
+    /// acceptable because index operations yield frequently to emit progress
+    /// events.
+    pub fn block_on_pumping<F, P>(
+        &self,
+        py: Option<Python<'_>>,
+        future: F,
+        mut pump: P,
+    ) -> PyResult<F::Output>
+    where
+        F: Future + Send,
+        F::Output: Send,
+        P: FnMut() -> PyResult<()>,
+    {
+        let mut future = std::pin::pin!(future);
+
+        loop {
+            pump()?;
+
+            let signal_check = match Python::try_attach(|py| py.check_signals()) {
+                Some(result) => result,
+                None => Ok(()),
+            };
+            signal_check?;
+
+            let maybe_output = if let Some(py) = py {
+                py.detach(|| {
+                    self.runtime.block_on(async {
+                        tokio::select! {
+                            result = &mut future => Some(result),
+                            _ = tokio::time::sleep(SIGNAL_CHECK_INTERVAL) => None,
+                        }
+                    })
+                })
+            } else if let Some(result) = Python::try_attach(|py| {
+                py.detach(|| {
+                    self.runtime.block_on(async {
+                        tokio::select! {
+                            result = &mut future => Some(result),
+                            _ = tokio::time::sleep(SIGNAL_CHECK_INTERVAL) => None,
+                        }
+                    })
+                })
+            }) {
+                result
+            } else {
+                self.runtime.block_on(async {
+                    tokio::select! {
+                        result = &mut future => Some(result),
+                        _ = tokio::time::sleep(SIGNAL_CHECK_INTERVAL) => None,
+                    }
+                })
+            };
+
+            if let Some(output) = maybe_output {
+                if let Err(err) = pump() {
+                    log::warn!(
+                        "Ignoring progress callback error after operation completed successfully: {}",
+                        err
+                    );
+                }
+                return Ok(output);
+            }
+        }
+    }
+
     async fn result_or_interrupt<F>(future: F) -> PyResult<F::Output>
     where
         F: Future + Send,
diff --git a/python/src/file.rs b/python/src/file.rs
index da8ba3e76bb..a25383ccea7 100644
--- a/python/src/file.rs
+++ b/python/src/file.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::namespace::extract_namespace_arc;
 use crate::{error::PythonErrorExt, rt};
 use arrow::pyarrow::PyArrowType;
 use arrow_array::{RecordBatch, RecordBatchReader, UInt32Array};
@@ -28,7 +29,7 @@ use lance_file::reader::{
 };
 use lance_file::writer::{FileWriter, FileWriterOptions};
 use lance_file::{LanceEncodingsIo, version::LanceFileVersion};
-use lance_io::object_store::ObjectStoreParams;
+use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, ObjectStoreParams};
 use lance_io::{
     ReadBatchParams,
     scheduler::{ScanScheduler, SchedulerConfig},
@@ -298,7 +299,7 @@ impl LanceFileWriter {
 #[pymethods]
 impl LanceFileWriter {
     #[new]
-    #[pyo3(signature=(path, schema=None, data_cache_bytes=None, version=None, storage_options=None, storage_options_provider=None, keep_original_array=None, max_page_bytes=None))]
+    #[pyo3(signature=(path, schema=None, data_cache_bytes=None, version=None, storage_options=None, namespace_client=None, table_id=None, keep_original_array=None, max_page_bytes=None))]
     #[allow(clippy::too_many_arguments)]
     pub fn new(
         path: String,
@@ -306,14 +307,22 @@ impl LanceFileWriter {
         data_cache_bytes: Option<u64>,
         version: Option<String>,
         storage_options: Option<HashMap<String, String>>,
-        storage_options_provider: Option<&Bound<'_, PyAny>>,
+        namespace_client: Option<&Bound<'_, PyAny>>,
+        table_id: Option<Vec<String>>,
         keep_original_array: Option<bool>,
         max_page_bytes: Option<u64>,
     ) -> PyResult<Self> {
-        // Convert Python StorageOptionsProvider to Rust trait object
-        let provider = storage_options_provider
-            .map(crate::storage_options::py_object_to_storage_options_provider)
-            .transpose()?;
+        // Create storage options provider from namespace_client and table_id if both are provided
+        let provider = if let (Some(ns_client), Some(tid)) = (&namespace_client, &table_id) {
+            let ns_client = extract_namespace_arc(ns_client.py(), ns_client)?;
+            Some(Arc::new(LanceNamespaceStorageOptionsProvider::new(
+                ns_client,
+                tid.clone(),
+            ))
+                as Arc<dyn lance_io::object_store::StorageOptionsProvider>)
+        } else {
+            None
+        };
 
         rt().block_on(
             None,
@@ -447,15 +456,24 @@ impl LanceFileSession {
 #[pymethods]
 impl LanceFileSession {
     #[new]
-    #[pyo3(signature=(uri_or_path, storage_options=None, storage_options_provider=None))]
+    #[pyo3(signature=(uri_or_path, storage_options=None, namespace_client=None, table_id=None))]
     pub fn new(
         uri_or_path: String,
         storage_options: Option<HashMap<String, String>>,
-        storage_options_provider: Option<&Bound<'_, PyAny>>,
+        namespace_client: Option<&Bound<'_, PyAny>>,
+        table_id: Option<Vec<String>>,
     ) -> PyResult<Self> {
-        let provider = storage_options_provider
-            .map(crate::storage_options::py_object_to_storage_options_provider)
-            .transpose()?;
+        // Create storage options provider from namespace_client and table_id if both are provided
+        let provider = if let (Some(ns_client), Some(tid)) = (&namespace_client, &table_id) {
+            let ns_client = extract_namespace_arc(ns_client.py(), ns_client)?;
+            Some(Arc::new(LanceNamespaceStorageOptionsProvider::new(
+                ns_client,
+                tid.clone(),
+            ))
+                as Arc<dyn lance_io::object_store::StorageOptionsProvider>)
+        } else {
+            None
+        };
         rt().block_on(None, Self::try_new(uri_or_path, storage_options, provider))?
     }
 
@@ -736,16 +754,25 @@ impl LanceFileReader {
 #[pymethods]
 impl LanceFileReader {
     #[new]
-    #[pyo3(signature=(path, storage_options=None, storage_options_provider=None, columns=None))]
+    #[pyo3(signature=(path, storage_options=None, namespace_client=None, table_id=None, columns=None))]
     pub fn new(
         path: String,
         storage_options: Option<HashMap<String, String>>,
-        storage_options_provider: Option<&Bound<'_, PyAny>>,
+        namespace_client: Option<&Bound<'_, PyAny>>,
+        table_id: Option<Vec<String>>,
         columns: Option<Vec<String>>,
     ) -> PyResult<Self> {
-        let provider = storage_options_provider
-            .map(crate::storage_options::py_object_to_storage_options_provider)
-            .transpose()?;
+        // Create storage options provider from namespace_client and table_id if both are provided
+        let provider = if let (Some(ns_client), Some(tid)) = (&namespace_client, &table_id) {
+            let ns_client = extract_namespace_arc(ns_client.py(), ns_client)?;
+            Some(Arc::new(LanceNamespaceStorageOptionsProvider::new(
+                ns_client,
+                tid.clone(),
+            ))
+                as Arc<dyn lance_io::object_store::StorageOptionsProvider>)
+        } else {
+            None
+        };
         rt().block_on(None, Self::open(path, storage_options, provider, columns))?
     }
 
diff --git a/python/src/fragment.rs b/python/src/fragment.rs
index aec4bfac58b..3dd512eb546 100644
--- a/python/src/fragment.rs
+++ b/python/src/fragment.rs
@@ -96,6 +96,20 @@ impl FileFragment {
         Ok(PyLance(metadata))
     }
 
+    #[staticmethod]
+    #[pyo3(signature = (dataset, path, base_id=None))]
+    fn create_data_file(
+        dataset: &Dataset,
+        path: &str,
+        base_id: Option<u32>,
+    ) -> PyResult<PyLance<DataFile>> {
+        let ds = dataset.ds.clone();
+        let data_file = rt()
+            .block_on(None, ds.create_data_file(path, base_id))?
+            .infer_error()?;
+        Ok(PyLance(data_file))
+    }
+
     #[staticmethod]
     #[pyo3(signature = (dataset_uri, fragment_id, reader, **kwargs))]
     fn create(
@@ -627,10 +641,10 @@ pub struct PyRowDatasetVersionMeta(pub RowDatasetVersionMeta);
 
 #[pymethods]
 impl PyRowIdMeta {
-    fn asdict(&self) -> PyResult<Bound<'_, PyDict>> {
-        Err(PyNotImplementedError::new_err(
-            "PyRowIdMeta.asdict is not yet supported.s",
-        ))
+    fn asdict(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
+        pythonize::pythonize(py, &self.0)
+            .map(|b| b.unbind())
+            .map_err(|err| PyValueError::new_err(format!("Could not convert RowIdMeta: {}", err)))
     }
 
     pub fn json(&self) -> PyResult<String> {
@@ -650,6 +664,13 @@ impl PyRowIdMeta {
         Ok(Self(row_id_meta))
     }
 
+    #[staticmethod]
+    pub fn from_dict(dict: &Bound<'_, PyAny>) -> PyResult<Self> {
+        let row_id_meta: RowIdMeta = pythonize::depythonize(dict)
+            .map_err(|err| PyValueError::new_err(format!("Could not load RowIdMeta: {}", err)))?;
+        Ok(Self(row_id_meta))
+    }
+
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, Py<PyAny>)> {
         let state = self.json()?;
         let state = PyTuple::new(py, vec![state])?.extract()?;
@@ -825,10 +846,12 @@ impl FromPyObject<'_> for PyLance<DataFile> {
     fn extract_bound(ob: &pyo3::Bound<'_, PyAny>) -> PyResult<Self> {
         let file_size_bytes: Option<u64> = ob.getattr("file_size_bytes")?.extract()?;
         let file_size_bytes = CachedFileSize::new(file_size_bytes.unwrap_or(0));
+        let fields: Vec<i32> = ob.getattr("fields")?.extract()?;
+        let column_indices: Vec<i32> = ob.getattr("column_indices")?.extract()?;
         Ok(Self(DataFile {
             path: ob.getattr("path")?.extract()?,
-            fields: ob.getattr("fields")?.extract()?,
-            column_indices: ob.getattr("column_indices")?.extract()?,
+            fields: fields.into(),
+            column_indices: column_indices.into(),
             file_major_version: ob.getattr("file_major_version")?.extract()?,
             file_minor_version: ob.getattr("file_minor_version")?.extract()?,
             file_size_bytes,
@@ -851,8 +874,8 @@ impl<'py> IntoPyObject<'py> for PyLance<&DataFile> {
         let file_size_bytes = self.0.file_size_bytes.get().map(u64::from);
         cls.call1((
             &self.0.path,
-            self.0.fields.clone(),
-            self.0.column_indices.clone(),
+            self.0.fields.to_vec(),
+            self.0.column_indices.to_vec(),
             self.0.file_major_version,
             self.0.file_minor_version,
             file_size_bytes,
diff --git a/python/src/indices.rs b/python/src/indices.rs
index 6b8d9e6c001..62f3c0c64ec 100644
--- a/python/src/indices.rs
+++ b/python/src/indices.rs
@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 use std::collections::HashSet;
+use std::fmt::Write;
 use std::sync::Arc;
 
 use arrow::pyarrow::{PyArrowType, ToPyArrow};
@@ -9,7 +10,9 @@ use arrow_array::{Array, FixedSizeListArray};
 use arrow_data::ArrayData;
 use chrono::{DateTime, Utc};
 use lance::dataset::Dataset as LanceDataset;
+use lance::index::DatasetIndexExt;
 use lance::index::vector::ivf::builder::write_vector_storage;
+use lance::index::vector::pq::build_pq_model_in_fragments;
 use lance::io::ObjectStore;
 use lance_index::progress::NoopIndexBuildProgress;
 use lance_index::vector::ivf::shuffler::{IvfShuffler, shuffle_vectors};
@@ -18,6 +21,7 @@ use lance_index::vector::{
     pq::{PQBuildParams, ProductQuantizer},
 };
 use lance_linalg::distance::DistanceType;
+use lance_table::format::{IndexMetadata, list_index_files_with_sizes};
 use pyo3::Bound;
 use pyo3::exceptions::PyValueError;
 use pyo3::prelude::*;
@@ -36,7 +40,7 @@ use crate::{
     dataset::Dataset, error::PythonErrorExt, file::object_store_from_uri_or_path_no_options, rt,
 };
 use lance::index::vector::ivf::write_ivf_pq_file_from_existing_index;
-use lance_index::{DatasetIndexExt, IndexDescription};
+use lance_index::{IndexDescription, IndexType};
 use uuid::Uuid;
 
 #[pyclass(name = "IndexConfig", module = "lance.indices", get_all)]
@@ -120,6 +124,7 @@ fn get_ivf_model(py: Python<'_>, dataset: &Dataset, index_name: &str) -> PyResul
     Py::new(py, PyIvfModel { inner: ivf_model })
 }
 
+#[allow(clippy::too_many_arguments)]
 async fn do_train_ivf_model(
     dataset: &Dataset,
     column: &str,
@@ -128,6 +133,7 @@ async fn do_train_ivf_model(
     distance_type: &str,
     sample_rate: u32,
     max_iters: u32,
+    fragment_ids: Option<Vec<u32>>,
 ) -> PyResult<ArrayData> {
     // We verify distance_type earlier so can unwrap here
     let distance_type = DistanceType::try_from(distance_type).unwrap();
@@ -143,6 +149,7 @@ async fn do_train_ivf_model(
         dimension,
         distance_type,
         &params,
+        fragment_ids.as_deref(),
         Arc::new(NoopIndexBuildProgress),
     )
     .await
@@ -153,6 +160,7 @@ async fn do_train_ivf_model(
 
 #[pyfunction]
 #[allow(clippy::too_many_arguments)]
+#[pyo3(signature=(dataset, column, dimension, num_partitions, distance_type, sample_rate, max_iters, fragment_ids=None))]
 fn train_ivf_model<'py>(
     py: Python<'py>,
     dataset: &Dataset,
@@ -162,6 +170,7 @@ fn train_ivf_model<'py>(
     distance_type: &str,
     sample_rate: u32,
     max_iters: u32,
+    fragment_ids: Option<Vec<u32>>,
 ) -> PyResult<Bound<'py, PyAny>> {
     let centroids = rt().block_on(
         Some(py),
@@ -173,6 +182,7 @@ fn train_ivf_model<'py>(
             distance_type,
             sample_rate,
             max_iters,
+            fragment_ids,
         ),
     )??;
     centroids.to_pyarrow(py)
@@ -188,6 +198,7 @@ async fn do_train_pq_model(
     sample_rate: u32,
     max_iters: u32,
     ivf_model: IvfModel,
+    fragment_ids: Option<Vec<u32>>,
 ) -> PyResult<ArrayData> {
     // We verify distance_type earlier so can unwrap here
     let distance_type = DistanceType::try_from(distance_type).unwrap();
@@ -198,13 +209,14 @@ async fn do_train_pq_model(
         sample_rate: sample_rate as usize,
         ..Default::default()
     };
-    let pq_model = lance::index::vector::pq::build_pq_model(
+    let pq_model = build_pq_model_in_fragments(
         dataset.ds.as_ref(),
         column,
         dimension,
         distance_type,
         &params,
         Some(&ivf_model),
+        fragment_ids.as_deref(),
     )
     .await
     .infer_error()?;
@@ -213,6 +225,7 @@ async fn do_train_pq_model(
 
 #[pyfunction]
 #[allow(clippy::too_many_arguments)]
+#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, sample_rate, max_iters, ivf_centroids, fragment_ids=None))]
 fn train_pq_model<'py>(
     py: Python<'py>,
     dataset: &Dataset,
@@ -223,6 +236,7 @@ fn train_pq_model<'py>(
     sample_rate: u32,
     max_iters: u32,
     ivf_centroids: PyArrowType<ArrayData>,
+    fragment_ids: Option<Vec<u32>>,
 ) -> PyResult<Bound<'py, PyAny>> {
     let ivf_centroids = ivf_centroids.0;
     let ivf_centroids = FixedSizeListArray::from(ivf_centroids);
@@ -243,6 +257,7 @@ fn train_pq_model<'py>(
             sample_rate,
             max_iters,
             ivf_model,
+            fragment_ids,
         ),
     )??;
     codebook.to_pyarrow(py)
@@ -415,7 +430,26 @@ async fn do_load_shuffled_vectors(
     .infer_error()?;
 
     let mut ds = dataset.ds.as_ref().clone();
-    ds.commit_existing_index(index_name, column, index_id)
+    let index_dir = ds.indices_dir().child(index_id.to_string());
+    let files = list_index_files_with_sizes(ds.object_store(), &index_dir)
+        .await
+        .infer_error()?;
+    let metadata = IndexMetadata {
+        uuid: index_id,
+        name: index_name.to_string(),
+        fields: vec![ds.schema().field(column).unwrap().id],
+        dataset_version: ds.manifest.version,
+        fragment_bitmap: Some(ds.fragments().iter().map(|f| f.id as u32).collect()),
+        index_details: Some(Arc::new(
+            prost_types::Any::from_msg(&lance_table::format::pb::VectorIndexDetails::default())
+                .unwrap(),
+        )),
+        index_version: IndexType::IvfPq.version(),
+        created_at: Some(Utc::now()),
+        base_id: None,
+        files: Some(files),
+    };
+    ds.commit_existing_index_segments(index_name, column, vec![metadata])
         .await
         .infer_error()?;
 
@@ -484,17 +518,39 @@ pub struct PyIndexSegmentDescription {
     pub index_version: i32,
     /// The timestamp when the index segment was created
     pub created_at: Option<DateTime<Utc>>,
+    /// The total size in bytes of all files in this segment
+    /// (None for backward compatibility with indices created before file tracking)
+    pub size_bytes: Option<u64>,
 }
 
 impl PyIndexSegmentDescription {
+    pub fn from_metadata(segment: &lance_table::format::IndexMetadata) -> Self {
+        let fragment_ids = segment
+            .fragment_bitmap
+            .as_ref()
+            .map(|bitmap| bitmap.iter().collect::<HashSet<_>>())
+            .unwrap_or_default();
+        let size_bytes = segment.total_size_bytes();
+
+        Self {
+            uuid: segment.uuid.to_string(),
+            dataset_version_at_last_update: segment.dataset_version,
+            fragment_ids,
+            index_version: segment.index_version,
+            created_at: segment.created_at,
+            size_bytes,
+        }
+    }
+
     pub fn __repr__(&self) -> String {
         format!(
-            "IndexSegmentDescription(uuid={}, dataset_version_at_last_update={}, fragment_ids={:?}, index_version={}, created_at={:?})",
+            "IndexSegmentDescription(uuid={}, dataset_version_at_last_update={}, fragment_ids={:?}, index_version={}, created_at={:?}, size_bytes={:?})",
             self.uuid,
             self.dataset_version_at_last_update,
             self.fragment_ids,
             self.index_version,
-            self.created_at
+            self.created_at,
+            self.size_bytes
         )
     }
 }
@@ -517,6 +573,9 @@ pub struct PyIndexDescription {
     pub details: PyJson,
     /// The segments of the index
     pub segments: Vec<PyIndexSegmentDescription>,
+    /// The total size in bytes of all files across all segments
+    /// (None for backward compatibility with indices created before file tracking)
+    pub total_size_bytes: Option<u64>,
 }
 
 impl PyIndexDescription {
@@ -536,20 +595,7 @@ impl PyIndexDescription {
         let segments = index
             .metadata()
             .iter()
-            .map(|segment| {
-                let fragment_ids = segment
-                    .fragment_bitmap
-                    .as_ref()
-                    .map(|bitmap| bitmap.iter().collect::<HashSet<_>>())
-                    .unwrap_or_default();
-                PyIndexSegmentDescription {
-                    uuid: segment.uuid.to_string(),
-                    dataset_version_at_last_update: segment.dataset_version,
-                    fragment_ids,
-                    index_version: segment.index_version,
-                    created_at: segment.created_at,
-                }
-            })
+            .map(PyIndexSegmentDescription::from_metadata)
             .collect();
 
         let details = index.details().unwrap_or_else(|_| "{}".to_string());
@@ -563,6 +609,7 @@ impl PyIndexDescription {
             type_url: index.type_url().to_string(),
             num_rows_indexed: index.rows_indexed(),
             details: PyJson(details),
+            total_size_bytes: index.total_size_bytes(),
         }
     }
 }
@@ -570,15 +617,20 @@ impl PyIndexDescription {
 #[pymethods]
 impl PyIndexDescription {
     pub fn __repr__(&self) -> String {
-        format!(
-            "IndexDescription(name={}, type_url={}, num_rows_indexed={}, fields={:?}, field_names={:?}, num_segments={})",
+        let mut repr = format!(
+            "IndexDescription(name='{}', type_url='{}', num_rows_indexed={}, fields={:?}, field_names={:?}, num_segments={}",
             self.name,
             self.type_url,
             self.num_rows_indexed,
             self.fields,
             self.field_names,
             self.segments.len()
-        )
+        );
+        if let Some(byte_size) = self.total_size_bytes {
+            write!(repr, ", total_size_bytes={}", byte_size).unwrap();
+        }
+        repr.push(')');
+        repr
     }
 }
 
diff --git a/python/src/lib.rs b/python/src/lib.rs
index 8c6f7c186ed..ee5eaf96a16 100644
--- a/python/src/lib.rs
+++ b/python/src/lib.rs
@@ -34,6 +34,7 @@ use ::arrow::pyarrow::PyArrowType;
 use ::arrow_schema::Schema as ArrowSchema;
 use ::lance::arrow::json::ArrowJsonExt;
 use ::lance::datafusion::LanceTableProvider;
+use ::lance::index::DatasetIndexExt;
 use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec;
 use datafusion_ffi::table_provider::FFI_TableProvider;
 #[cfg(feature = "datagen")]
@@ -50,7 +51,6 @@ use file::{
     LanceBufferDescriptor, LanceColumnMetadata, LanceFileMetadata, LanceFileReader,
     LanceFileStatistics, LanceFileWriter, LancePageMetadata, stable_version,
 };
-use lance_index::DatasetIndexExt;
 use log::Level;
 use pyo3::exceptions::PyIOError;
 use pyo3::prelude::*;
diff --git a/python/src/namespace.rs b/python/src/namespace.rs
index b4876ac5559..ee302d12303 100644
--- a/python/src/namespace.rs
+++ b/python/src/namespace.rs
@@ -10,12 +10,24 @@ use async_trait::async_trait;
 use bytes::Bytes;
 use lance_namespace::LanceNamespace as LanceNamespaceTrait;
 use lance_namespace::models::{
-    CreateTableVersionRequest, CreateTableVersionResponse, DescribeTableVersionRequest,
-    DescribeTableVersionResponse, ListTableVersionsRequest, ListTableVersionsResponse,
+    AlterTableAddColumnsRequest, AlterTableAlterColumnsRequest, AlterTableDropColumnsRequest,
+    AlterTransactionRequest, AnalyzeTableQueryPlanRequest, CountTableRowsRequest,
+    CreateTableIndexRequest, CreateTableTagRequest, CreateTableVersionRequest,
+    CreateTableVersionResponse, DeleteFromTableRequest, DeleteTableTagRequest,
+    DescribeTableIndexStatsRequest, DescribeTableRequest, DescribeTableResponse,
+    DescribeTableVersionRequest, DescribeTableVersionResponse, DescribeTransactionRequest,
+    DropTableIndexRequest, ExplainTableQueryPlanRequest, GetTableStatsRequest,
+    GetTableTagVersionRequest, InsertIntoTableRequest, ListTableIndicesRequest,
+    ListTableTagsRequest, ListTableVersionsRequest, ListTableVersionsResponse, ListTablesRequest,
+    MergeInsertIntoTableRequest, QueryTableRequest, RestoreTableRequest, UpdateTableRequest,
+    UpdateTableSchemaMetadataRequest, UpdateTableTagRequest,
 };
 use lance_namespace_impls::RestNamespaceBuilder;
 use lance_namespace_impls::{ConnectBuilder, RestAdapter, RestAdapterConfig, RestAdapterHandle};
-use lance_namespace_impls::{DirectoryNamespaceBuilder, DynamicContextProvider, OperationInfo};
+use lance_namespace_impls::{
+    DirectoryNamespace, DirectoryNamespaceBuilder, DynamicContextProvider, OperationInfo,
+    RestNamespace,
+};
 use pyo3::prelude::*;
 use pyo3::types::{PyBytes, PyDict};
 use pythonize::{depythonize, pythonize};
@@ -104,7 +116,7 @@ fn dict_to_hashmap(dict: &Bound<'_, PyDict>) -> PyResult<HashMap<String, String>
 /// Python wrapper for DirectoryNamespace
 #[pyclass(name = "PyDirectoryNamespace", module = "lance.lance")]
 pub struct PyDirectoryNamespace {
-    pub(crate) inner: Arc<dyn lance_namespace::LanceNamespace>,
+    pub(crate) inner: Arc<DirectoryNamespace>,
 }
 
 #[pymethods]
@@ -316,6 +328,18 @@ impl PyDirectoryNamespace {
         Ok(pythonize(py, &response)?.into())
     }
 
+    fn rename_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.rename_table(request))?
+            .infer_error()?;
+        Ok(pythonize(py, &response)?.into())
+    }
+
     // Table version operations
 
     fn list_table_versions<'py>(
@@ -365,490 +389,1148 @@ impl PyDirectoryNamespace {
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
-}
-
-/// Python wrapper for RestNamespace
-#[pyclass(name = "PyRestNamespace", module = "lance.lance")]
-pub struct PyRestNamespace {
-    pub(crate) inner: Arc<dyn lance_namespace::LanceNamespace>,
-}
-
-#[pymethods]
-impl PyRestNamespace {
-    /// Create a new RestNamespace from properties
-    ///
-    /// # Arguments
-    ///
-    /// * `context_provider` - Optional object with `provide_context(info: dict) -> dict` method
-    ///   for providing dynamic per-request context. Context keys that start with `headers.`
-    ///   are converted to HTTP headers by stripping the prefix. For example,
-    ///   `{"headers.Authorization": "Bearer token"}` becomes the `Authorization` header.
-    /// * `**properties` - Namespace configuration properties (uri, delimiter, header.*, etc.)
-    #[new]
-    #[pyo3(signature = (context_provider = None, **properties))]
-    fn new(
-        context_provider: Option<&Bound<'_, PyAny>>,
-        properties: Option<&Bound<'_, PyDict>>,
-    ) -> PyResult<Self> {
-        let mut props = HashMap::new();
 
-        if let Some(dict) = properties {
-            props = dict_to_hashmap(dict)?;
-        }
+    // Data manipulation operations
 
-        let mut builder = RestNamespaceBuilder::from_properties(props).map_err(|e| {
-            pyo3::exceptions::PyValueError::new_err(format!(
-                "Failed to create RestNamespace: {}",
-                e
-            ))
-        })?;
+    fn count_table_rows(&self, py: Python, request: &Bound<'_, PyAny>) -> PyResult<i64> {
+        let request: CountTableRowsRequest = depythonize(request)?;
+        let count = crate::rt()
+            .block_on(Some(py), self.inner.count_table_rows(request))?
+            .infer_error()?;
+        Ok(count)
+    }
 
-        // Add context provider if provided
-        if let Some(provider) = context_provider {
-            let py_provider = PyDynamicContextProvider::new(provider.clone().unbind());
-            builder = builder.context_provider(Arc::new(py_provider));
-        }
+    fn insert_into_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+        request_data: &Bound<'_, PyBytes>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: InsertIntoTableRequest = depythonize(request)?;
+        let data = Bytes::copy_from_slice(request_data.as_bytes());
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.insert_into_table(request, data))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
 
-        let namespace = builder.build();
+    fn merge_insert_into_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+        request_data: &Bound<'_, PyBytes>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: MergeInsertIntoTableRequest = depythonize(request)?;
+        let data = Bytes::copy_from_slice(request_data.as_bytes());
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.merge_insert_into_table(request, data))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
 
-        Ok(Self {
-            inner: Arc::new(namespace),
-        })
+    fn update_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: UpdateTableRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.update_table(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    /// Get the namespace ID
-    fn namespace_id(&self) -> String {
-        format!("{:?}", self.inner)
+    fn delete_from_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: DeleteFromTableRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.delete_from_table(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn __repr__(&self) -> String {
-        format!("PyRestNamespace({})", self.namespace_id())
+    fn query_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyBytes>> {
+        let request: QueryTableRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.query_table(request))?
+            .infer_error()?;
+        Ok(PyBytes::new(py, &response))
     }
 
-    // Namespace operations
+    // Index operations
 
-    fn list_namespaces<'py>(
+    fn create_table_index<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: CreateTableIndexRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.list_namespaces(request))?
+            .block_on(Some(py), self.inner.create_table_index(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn describe_namespace<'py>(
+    fn list_table_indices<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: ListTableIndicesRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.describe_namespace(request))?
+            .block_on(Some(py), self.inner.list_table_indices(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn create_namespace<'py>(
+    fn describe_table_index_stats<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: DescribeTableIndexStatsRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.create_namespace(request))?
+            .block_on(Some(py), self.inner.describe_table_index_stats(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn drop_namespace<'py>(
+    // Transaction operations
+
+    fn describe_transaction<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: DescribeTransactionRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.drop_namespace(request))?
+            .block_on(Some(py), self.inner.describe_transaction(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn namespace_exists(&self, py: Python, request: &Bound<'_, PyAny>) -> PyResult<()> {
-        let request = depythonize(request)?;
-        crate::rt()
-            .block_on(Some(py), self.inner.namespace_exists(request))?
+    fn alter_transaction<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: AlterTransactionRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.alter_transaction(request))?
             .infer_error()?;
-        Ok(())
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    // Table operations
+    // Additional index operations
 
-    fn list_tables<'py>(
+    fn create_table_scalar_index<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: CreateTableIndexRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.list_tables(request))?
+            .block_on(Some(py), self.inner.create_table_scalar_index(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn describe_table<'py>(
+    fn drop_table_index<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: DropTableIndexRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.describe_table(request))?
+            .block_on(Some(py), self.inner.drop_table_index(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn register_table<'py>(
+    // Additional table operations
+
+    fn list_all_tables<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: ListTablesRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.register_table(request))?
+            .block_on(Some(py), self.inner.list_all_tables(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn table_exists(&self, py: Python, request: &Bound<'_, PyAny>) -> PyResult<()> {
-        let request = depythonize(request)?;
-        crate::rt()
-            .block_on(Some(py), self.inner.table_exists(request))?
+    fn restore_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: RestoreTableRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.restore_table(request))?
             .infer_error()?;
-        Ok(())
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn drop_table<'py>(
+    fn update_table_schema_metadata<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: UpdateTableSchemaMetadataRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.drop_table(request))?
+            .block_on(Some(py), self.inner.update_table_schema_metadata(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn deregister_table<'py>(
+    fn get_table_stats<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: GetTableStatsRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.deregister_table(request))?
+            .block_on(Some(py), self.inner.get_table_stats(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn create_table<'py>(
+    // Query plan operations
+
+    fn explain_table_query_plan(&self, py: Python, request: &Bound<'_, PyAny>) -> PyResult<String> {
+        let request: ExplainTableQueryPlanRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.explain_table_query_plan(request))?
+            .infer_error()?;
+        Ok(response)
+    }
+
+    fn analyze_table_query_plan(&self, py: Python, request: &Bound<'_, PyAny>) -> PyResult<String> {
+        let request: AnalyzeTableQueryPlanRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.analyze_table_query_plan(request))?
+            .infer_error()?;
+        Ok(response)
+    }
+
+    // Column alteration operations
+
+    fn alter_table_add_columns<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
-        request_data: &Bound<'_, PyBytes>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
-        let data = Bytes::copy_from_slice(request_data.as_bytes());
+        let request: AlterTableAddColumnsRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.create_table(request, data))?
+            .block_on(Some(py), self.inner.alter_table_add_columns(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn declare_table<'py>(
+    fn alter_table_alter_columns<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: AlterTableAlterColumnsRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.declare_table(request))?
+            .block_on(Some(py), self.inner.alter_table_alter_columns(request))?
             .infer_error()?;
-        Ok(pythonize(py, &response)?.into())
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn rename_table<'py>(
+    fn alter_table_drop_columns<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: AlterTableDropColumnsRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.rename_table(request))?
+            .block_on(Some(py), self.inner.alter_table_drop_columns(request))?
             .infer_error()?;
-        Ok(pythonize(py, &response)?.into())
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    // Table version operations
+    // Table tag operations
 
-    fn list_table_versions<'py>(
+    fn list_table_tags<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: ListTableTagsRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.list_table_versions(request))?
+            .block_on(Some(py), self.inner.list_table_tags(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn create_table_version<'py>(
+    fn get_table_tag_version<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: GetTableTagVersionRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.create_table_version(request))?
+            .block_on(Some(py), self.inner.get_table_tag_version(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn describe_table_version<'py>(
+    fn create_table_tag<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: CreateTableTagRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.describe_table_version(request))?
+            .block_on(Some(py), self.inner.create_table_tag(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
-    fn batch_delete_table_versions<'py>(
+    fn delete_table_tag<'py>(
         &self,
         py: Python<'py>,
         request: &Bound<'_, PyAny>,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let request = depythonize(request)?;
+        let request: DeleteTableTagRequest = depythonize(request)?;
         let response = crate::rt()
-            .block_on(Some(py), self.inner.batch_delete_table_versions(request))?
+            .block_on(Some(py), self.inner.delete_table_tag(request))?
             .infer_error()?;
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
-}
 
-/// Wrapper that allows any Python object implementing LanceNamespace protocol
-/// to be used as a Rust LanceNamespace.
-///
-/// This is similar to JavaLanceNamespace in the Java bindings - it wraps a Python
-/// object and calls back into Python when namespace methods are invoked.
-///
-/// We use `Arc<Py<PyAny>>` instead of `Py<PyAny>` directly because cloning `Py`
-/// requires the GIL, but cloning `Arc` does not. This allows us to pass the
-/// namespace reference to `spawn_blocking` without holding the GIL.
-pub struct PyLanceNamespace {
-    py_namespace: Arc<Py<PyAny>>,
-    namespace_id: String,
-}
+    fn update_table_tag<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: UpdateTableTagRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.update_table_tag(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
 
-impl PyLanceNamespace {
-    /// Create a new PyLanceNamespace wrapper around a Python namespace object.
-    pub fn new(_py: Python<'_>, py_namespace: &Bound<'_, PyAny>) -> PyResult<Self> {
-        // Get the namespace_id by calling the Python method
-        let namespace_id = py_namespace
-            .call_method0("namespace_id")?
-            .extract::<String>()?;
+    // Operation metrics methods
 
-        Ok(Self {
-            py_namespace: Arc::new(py_namespace.clone().unbind()),
-            namespace_id,
-        })
+    /// Retrieve operation metrics as a dictionary.
+    ///
+    /// Returns a dict where keys are operation names (e.g., "list_tables", "describe_table")
+    /// and values are the number of times each operation was called.
+    ///
+    /// Returns an empty dict if `ops_metrics_enabled` was false when creating the namespace.
+    fn retrieve_ops_metrics(&self) -> HashMap<String, u64> {
+        self.inner.retrieve_ops_metrics()
     }
 
-    /// Create an Arc<dyn LanceNamespace> from a Python namespace object.
-    pub fn create_arc(
-        py: Python<'_>,
-        py_namespace: &Bound<'_, PyAny>,
-    ) -> PyResult<Arc<dyn LanceNamespaceTrait>> {
-        let wrapper = Self::new(py, py_namespace)?;
-        Ok(Arc::new(wrapper))
+    /// Reset all operation metrics counters to zero.
+    ///
+    /// Does nothing if `ops_metrics_enabled` was false when creating the namespace.
+    fn reset_ops_metrics(&self) {
+        self.inner.reset_ops_metrics()
     }
 }
 
-impl std::fmt::Debug for PyLanceNamespace {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "PyLanceNamespace {{ id: {} }}", self.namespace_id)
-    }
+/// Python wrapper for RestNamespace
+#[pyclass(name = "PyRestNamespace", module = "lance.lance")]
+pub struct PyRestNamespace {
+    pub(crate) inner: Arc<RestNamespace>,
 }
 
-#[async_trait]
-impl LanceNamespaceTrait for PyLanceNamespace {
-    fn namespace_id(&self) -> String {
-        self.namespace_id.clone()
-    }
-
-    async fn describe_table_version(
-        &self,
-        request: DescribeTableVersionRequest,
-    ) -> lance_core::Result<DescribeTableVersionResponse> {
-        // Clone the Arc (doesn't need GIL) to pass to spawn_blocking
-        let py_namespace = self.py_namespace.clone();
-        let request_json = serde_json::to_string(&request).map_err(|e| {
-            lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to serialize request: {}",
-                e
-            ))))
-        })?;
-
-        let response_json = tokio::task::spawn_blocking(move || {
-            Python::attach(|py| {
-                let result =
-                    py_namespace.call_method1(py, "describe_table_version_json", (request_json,));
-
-                match result {
-                    Ok(response_py) => {
-                        let response_str: String = response_py.extract(py).map_err(|e| {
-                            lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                                "Failed to extract response string: {}",
-                                e
-                            ))))
-                        })?;
-                        Ok(response_str)
-                    }
-                    Err(e) => Err(lance_core::Error::io_source(Box::new(
-                        std::io::Error::other(format!(
-                            "Failed to call describe_table_version_json: {}",
-                            e
-                        )),
-                    ))),
-                }
-            })
+#[pymethods]
+impl PyRestNamespace {
+    /// Create a new RestNamespace from properties
+    ///
+    /// # Arguments
+    ///
+    /// * `context_provider` - Optional object with `provide_context(info: dict) -> dict` method
+    ///   for providing dynamic per-request context. Context keys that start with `headers.`
+    ///   are converted to HTTP headers by stripping the prefix. For example,
+    ///   `{"headers.Authorization": "Bearer token"}` becomes the `Authorization` header.
+    /// * `**properties` - Namespace configuration properties (uri, delimiter, header.*, etc.)
+    #[new]
+    #[pyo3(signature = (context_provider = None, **properties))]
+    fn new(
+        context_provider: Option<&Bound<'_, PyAny>>,
+        properties: Option<&Bound<'_, PyDict>>,
+    ) -> PyResult<Self> {
+        let mut props = HashMap::new();
+
+        if let Some(dict) = properties {
+            props = dict_to_hashmap(dict)?;
+        }
+
+        let mut builder = RestNamespaceBuilder::from_properties(props).map_err(|e| {
+            pyo3::exceptions::PyValueError::new_err(format!(
+                "Failed to create RestNamespace: {}",
+                e
+            ))
+        })?;
+
+        // Add context provider if provided
+        if let Some(provider) = context_provider {
+            let py_provider = PyDynamicContextProvider::new(provider.clone().unbind());
+            builder = builder.context_provider(Arc::new(py_provider));
+        }
+
+        let namespace = builder.build();
+
+        Ok(Self {
+            inner: Arc::new(namespace),
+        })
+    }
+
+    /// Get the namespace ID
+    fn namespace_id(&self) -> String {
+        format!("{:?}", self.inner)
+    }
+
+    fn __repr__(&self) -> String {
+        format!("PyRestNamespace({})", self.namespace_id())
+    }
+
+    // Namespace operations
+
+    fn list_namespaces<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.list_namespaces(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn describe_namespace<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.describe_namespace(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn create_namespace<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.create_namespace(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn drop_namespace<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.drop_namespace(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn namespace_exists(&self, py: Python, request: &Bound<'_, PyAny>) -> PyResult<()> {
+        let request = depythonize(request)?;
+        crate::rt()
+            .block_on(Some(py), self.inner.namespace_exists(request))?
+            .infer_error()?;
+        Ok(())
+    }
+
+    // Table operations
+
+    fn list_tables<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.list_tables(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn describe_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.describe_table(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn register_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.register_table(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn table_exists(&self, py: Python, request: &Bound<'_, PyAny>) -> PyResult<()> {
+        let request = depythonize(request)?;
+        crate::rt()
+            .block_on(Some(py), self.inner.table_exists(request))?
+            .infer_error()?;
+        Ok(())
+    }
+
+    fn drop_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.drop_table(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn deregister_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.deregister_table(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn create_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+        request_data: &Bound<'_, PyBytes>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let data = Bytes::copy_from_slice(request_data.as_bytes());
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.create_table(request, data))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn declare_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.declare_table(request))?
+            .infer_error()?;
+        Ok(pythonize(py, &response)?.into())
+    }
+
+    fn rename_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.rename_table(request))?
+            .infer_error()?;
+        Ok(pythonize(py, &response)?.into())
+    }
+
+    // Table version operations
+
+    fn list_table_versions<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.list_table_versions(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn create_table_version<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.create_table_version(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn describe_table_version<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.describe_table_version(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn batch_delete_table_versions<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.batch_delete_table_versions(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    // Data manipulation operations
+
+    fn count_table_rows(&self, py: Python, request: &Bound<'_, PyAny>) -> PyResult<i64> {
+        let request: CountTableRowsRequest = depythonize(request)?;
+        let count = crate::rt()
+            .block_on(Some(py), self.inner.count_table_rows(request))?
+            .infer_error()?;
+        Ok(count)
+    }
+
+    fn insert_into_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+        request_data: &Bound<'_, PyBytes>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: InsertIntoTableRequest = depythonize(request)?;
+        let data = Bytes::copy_from_slice(request_data.as_bytes());
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.insert_into_table(request, data))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn merge_insert_into_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+        request_data: &Bound<'_, PyBytes>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: MergeInsertIntoTableRequest = depythonize(request)?;
+        let data = Bytes::copy_from_slice(request_data.as_bytes());
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.merge_insert_into_table(request, data))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn update_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: UpdateTableRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.update_table(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn delete_from_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: DeleteFromTableRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.delete_from_table(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn query_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyBytes>> {
+        let request: QueryTableRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.query_table(request))?
+            .infer_error()?;
+        Ok(PyBytes::new(py, &response))
+    }
+
+    // Index operations
+
+    fn create_table_index<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: CreateTableIndexRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.create_table_index(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn list_table_indices<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: ListTableIndicesRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.list_table_indices(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn describe_table_index_stats<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: DescribeTableIndexStatsRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.describe_table_index_stats(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    // Transaction operations
+
+    fn describe_transaction<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: DescribeTransactionRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.describe_transaction(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn alter_transaction<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: AlterTransactionRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.alter_transaction(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    // Additional index operations
+
+    fn create_table_scalar_index<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: CreateTableIndexRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.create_table_scalar_index(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn drop_table_index<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: DropTableIndexRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.drop_table_index(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    // Additional table operations
+
+    fn list_all_tables<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: ListTablesRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.list_all_tables(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn restore_table<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: RestoreTableRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.restore_table(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn update_table_schema_metadata<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: UpdateTableSchemaMetadataRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.update_table_schema_metadata(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn get_table_stats<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: GetTableStatsRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.get_table_stats(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    // Query plan operations
+
+    fn explain_table_query_plan(&self, py: Python, request: &Bound<'_, PyAny>) -> PyResult<String> {
+        let request: ExplainTableQueryPlanRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.explain_table_query_plan(request))?
+            .infer_error()?;
+        Ok(response)
+    }
+
+    fn analyze_table_query_plan(&self, py: Python, request: &Bound<'_, PyAny>) -> PyResult<String> {
+        let request: AnalyzeTableQueryPlanRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.analyze_table_query_plan(request))?
+            .infer_error()?;
+        Ok(response)
+    }
+
+    // Column alteration operations
+
+    fn alter_table_add_columns<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: AlterTableAddColumnsRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.alter_table_add_columns(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn alter_table_alter_columns<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: AlterTableAlterColumnsRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.alter_table_alter_columns(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn alter_table_drop_columns<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: AlterTableDropColumnsRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.alter_table_drop_columns(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    // Table tag operations
+
+    fn list_table_tags<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: ListTableTagsRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.list_table_tags(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn get_table_tag_version<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: GetTableTagVersionRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.get_table_tag_version(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn create_table_tag<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: CreateTableTagRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.create_table_tag(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn delete_table_tag<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: DeleteTableTagRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.delete_table_tag(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn update_table_tag<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request: UpdateTableTagRequest = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.update_table_tag(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    // Operation metrics methods
+
+    /// Retrieve operation metrics as a dictionary.
+    ///
+    /// Returns a dict where keys are operation names (e.g., "list_tables", "describe_table")
+    /// and values are the number of times each operation was called.
+    ///
+    /// Returns an empty dict if `ops_metrics_enabled` was false when creating the namespace.
+    fn retrieve_ops_metrics(&self) -> HashMap<String, u64> {
+        self.inner.retrieve_ops_metrics()
+    }
+
+    /// Reset all operation metrics counters to zero.
+    ///
+    /// Does nothing if `ops_metrics_enabled` was false when creating the namespace.
+    fn reset_ops_metrics(&self) {
+        self.inner.reset_ops_metrics()
+    }
+}
+
+/// Get or create the DictWithModelDump class in Python.
+/// This class acts like a dict but also has model_dump() method.
+/// This allows it to work with both:
+/// - depythonize (which expects a dict/Mapping)
+/// - Python code that calls .model_dump() (like DirectoryNamespace wrapper)
+fn get_dict_with_model_dump_class(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
+    // Use a module-level cache via __builtins__
+    let builtins = py.import("builtins")?;
+    if builtins.hasattr("_DictWithModelDump")? {
+        return builtins.getattr("_DictWithModelDump");
+    }
+
+    // Create the class using exec
+    let locals = PyDict::new(py);
+    py.run(
+        c"class DictWithModelDump(dict):
+    def model_dump(self):
+        return dict(self)",
+        None,
+        Some(&locals),
+    )?;
+    let class = locals.get_item("DictWithModelDump")?.ok_or_else(|| {
+        pyo3::exceptions::PyRuntimeError::new_err("Failed to create DictWithModelDump class")
+    })?;
+
+    // Cache it
+    builtins.setattr("_DictWithModelDump", &class)?;
+    Ok(class)
+}
+
+/// Helper to call a Python namespace method with JSON serialization.
+/// For methods that take a request and return a response.
+/// Uses DictWithModelDump to pass a dict that also has model_dump() method,
+/// making it compatible with both depythonize and Python wrappers.
+async fn call_py_method<Req, Resp>(
+    py_namespace: Arc<Py<PyAny>>,
+    method_name: &'static str,
+    request: Req,
+) -> lance_core::Result<Resp>
+where
+    Req: serde::Serialize + Send + 'static,
+    Resp: serde::de::DeserializeOwned + Send + 'static,
+{
+    let request_json = serde_json::to_string(&request).map_err(|e| {
+        lance_core::Error::io(format!(
+            "Failed to serialize request for {}: {}",
+            method_name, e
+        ))
+    })?;
+
+    let response_json = tokio::task::spawn_blocking(move || {
+        Python::attach(|py| {
+            let json_module = py.import("json")?;
+            let request_dict = json_module.call_method1("loads", (&request_json,))?;
+
+            // Wrap dict in DictWithModelDump so it works with both depythonize and .model_dump()
+            let dict_class = get_dict_with_model_dump_class(py)?;
+            let request_arg = dict_class.call1((request_dict,))?;
+
+            // Call the Python method
+            let result = py_namespace.call_method1(py, method_name, (request_arg,))?;
+
+            // Convert response to dict, then to JSON
+            // Pydantic models have model_dump() method
+            let result_dict = if result.bind(py).hasattr("model_dump")? {
+                result.call_method0(py, "model_dump")?
+            } else {
+                result
+            };
+            let response_json: String = json_module
+                .call_method1("dumps", (result_dict,))?
+                .extract()?;
+            Ok::<_, PyErr>(response_json)
         })
-        .await
-        .map_err(|e| {
-            lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                "Task join error: {}",
-                e
-            ))))
-        })??;
+    })
+    .await
+    .map_err(|e| lance_core::Error::io(format!("Task join error for {}: {}", method_name, e)))?
+    .map_err(|e: PyErr| lance_core::Error::io(format!("Python error in {}: {}", method_name, e)))?;
+
+    serde_json::from_str(&response_json).map_err(|e| {
+        lance_core::Error::io(format!(
+            "Failed to deserialize response from {}: {}",
+            method_name, e
+        ))
+    })
+}
 
-        serde_json::from_str(&response_json).map_err(|e| {
-            lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to deserialize response: {}",
-                e
-            ))))
+/// Wrapper that allows any Python object implementing LanceNamespace protocol
+/// to be used as a Rust LanceNamespace.
+///
+/// This is similar to JavaLanceNamespace in the Java bindings - it wraps a Python
+/// object and calls back into Python when namespace methods are invoked.
+///
+/// We use `Arc<Py<PyAny>>` instead of `Py<PyAny>` directly because cloning `Py`
+/// requires the GIL, but cloning `Arc` does not. This allows us to pass the
+/// namespace reference to `spawn_blocking` without holding the GIL.
+pub struct PyLanceNamespace {
+    py_namespace: Arc<Py<PyAny>>,
+    namespace_id: String,
+}
+
+impl PyLanceNamespace {
+    /// Create a new PyLanceNamespace wrapper around a Python namespace object.
+    pub fn new(_py: Python<'_>, py_namespace: &Bound<'_, PyAny>) -> PyResult<Self> {
+        // Get the namespace_id by calling the Python method
+        let namespace_id = py_namespace
+            .call_method0("namespace_id")?
+            .extract::<String>()?;
+
+        Ok(Self {
+            py_namespace: Arc::new(py_namespace.clone().unbind()),
+            namespace_id,
         })
     }
 
+    /// Create an Arc<dyn LanceNamespace> from a Python namespace object.
+    pub fn create_arc(
+        py: Python<'_>,
+        py_namespace: &Bound<'_, PyAny>,
+    ) -> PyResult<Arc<dyn LanceNamespaceTrait>> {
+        let wrapper = Self::new(py, py_namespace)?;
+        Ok(Arc::new(wrapper))
+    }
+}
+
+impl std::fmt::Debug for PyLanceNamespace {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "PyLanceNamespace {{ id: {} }}", self.namespace_id)
+    }
+}
+
+#[async_trait]
+impl LanceNamespaceTrait for PyLanceNamespace {
+    fn namespace_id(&self) -> String {
+        self.namespace_id.clone()
+    }
+
+    async fn describe_table(
+        &self,
+        request: DescribeTableRequest,
+    ) -> lance_core::Result<DescribeTableResponse> {
+        call_py_method(self.py_namespace.clone(), "describe_table", request).await
+    }
+
+    async fn describe_table_version(
+        &self,
+        request: DescribeTableVersionRequest,
+    ) -> lance_core::Result<DescribeTableVersionResponse> {
+        call_py_method(self.py_namespace.clone(), "describe_table_version", request).await
+    }
+
     async fn create_table_version(
         &self,
         request: CreateTableVersionRequest,
     ) -> lance_core::Result<CreateTableVersionResponse> {
-        // Clone the Arc (doesn't need GIL) to pass to spawn_blocking
-        let py_namespace = self.py_namespace.clone();
-        let request_json = serde_json::to_string(&request).map_err(|e| {
-            lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to serialize request: {}",
-                e
-            ))))
-        })?;
-
-        let response_json = tokio::task::spawn_blocking(move || {
-            Python::attach(|py| {
-                let result =
-                    py_namespace.call_method1(py, "create_table_version_json", (request_json,));
-
-                match result {
-                    Ok(response_py) => {
-                        let response_str: String = response_py.extract(py).map_err(|e| {
-                            lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                                "Failed to extract response string: {}",
-                                e
-                            ))))
-                        })?;
-                        Ok(response_str)
-                    }
-                    Err(e) => Err(lance_core::Error::io_source(Box::new(
-                        std::io::Error::other(format!(
-                            "Failed to call create_table_version_json: {}",
-                            e
-                        )),
-                    ))),
-                }
-            })
-        })
-        .await
-        .map_err(|e| {
-            lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                "Task join error: {}",
-                e
-            ))))
-        })??;
-
-        serde_json::from_str(&response_json).map_err(|e| {
-            lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to deserialize response: {}",
-                e
-            ))))
-        })
+        call_py_method(self.py_namespace.clone(), "create_table_version", request).await
     }
 
     async fn list_table_versions(
         &self,
         request: ListTableVersionsRequest,
     ) -> lance_core::Result<ListTableVersionsResponse> {
-        // Clone the Arc (doesn't need GIL) to pass to spawn_blocking
-        let py_namespace = self.py_namespace.clone();
-        let request_json = serde_json::to_string(&request).map_err(|e| {
-            lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to serialize request: {}",
-                e
-            ))))
-        })?;
-
-        let response_json = tokio::task::spawn_blocking(move || {
-            Python::attach(|py| {
-                let result =
-                    py_namespace.call_method1(py, "list_table_versions_json", (request_json,));
-
-                match result {
-                    Ok(response_py) => {
-                        let response_str: String = response_py.extract(py).map_err(|e| {
-                            lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                                "Failed to extract response string: {}",
-                                e
-                            ))))
-                        })?;
-                        Ok(response_str)
-                    }
-                    Err(e) => Err(lance_core::Error::io_source(Box::new(
-                        std::io::Error::other(format!(
-                            "Failed to call list_table_versions_json: {}",
-                            e
-                        )),
-                    ))),
-                }
-            })
-        })
-        .await
-        .map_err(|e| {
-            lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                "Task join error: {}",
-                e
-            ))))
-        })??;
-
-        serde_json::from_str(&response_json).map_err(|e| {
-            lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to deserialize response: {}",
-                e
-            ))))
-        })
+        call_py_method(self.py_namespace.clone(), "list_table_versions", request).await
     }
 }
 
@@ -864,37 +1546,39 @@ impl LanceNamespaceTrait for PyLanceNamespace {
 /// are wrapped with PyLanceNamespace to call through Python.
 pub fn extract_namespace_arc(
     py: Python<'_>,
-    ns: &Bound<'_, PyAny>,
+    namespace_client: &Bound<'_, PyAny>,
 ) -> PyResult<Arc<dyn LanceNamespaceTrait>> {
     // Direct PyO3 class
-    if let Ok(dir_ns) = ns.downcast::<PyDirectoryNamespace>() {
-        return Ok(dir_ns.borrow().inner.clone());
+    if let Ok(dir_namespace_client) = namespace_client.downcast::<PyDirectoryNamespace>() {
+        return Ok(dir_namespace_client.borrow().inner.clone() as Arc<dyn LanceNamespaceTrait>);
     }
-    if let Ok(rest_ns) = ns.downcast::<PyRestNamespace>() {
-        return Ok(rest_ns.borrow().inner.clone());
+    if let Ok(rest_namespace_client) = namespace_client.downcast::<PyRestNamespace>() {
+        return Ok(rest_namespace_client.borrow().inner.clone() as Arc<dyn LanceNamespaceTrait>);
     }
 
     // Python wrapper class - check if it's the exact wrapper class
-    if let Ok(inner) = ns.getattr("_inner") {
-        let type_name = ns
+    if let Ok(inner) = namespace_client.getattr("_inner") {
+        let type_name = namespace_client
             .get_type()
             .name()
             .map(|n| n.to_string())
             .unwrap_or_default();
 
         if type_name == "DirectoryNamespace" {
-            if let Ok(dir_ns) = inner.downcast::<PyDirectoryNamespace>() {
-                return Ok(dir_ns.borrow().inner.clone());
+            if let Ok(dir_namespace_client) = inner.downcast::<PyDirectoryNamespace>() {
+                return Ok(
+                    dir_namespace_client.borrow().inner.clone() as Arc<dyn LanceNamespaceTrait>
+                );
             }
         } else if type_name == "RestNamespace"
-            && let Ok(rest_ns) = inner.downcast::<PyRestNamespace>()
+            && let Ok(rest_namespace_client) = inner.downcast::<PyRestNamespace>()
         {
-            return Ok(rest_ns.borrow().inner.clone());
+            return Ok(rest_namespace_client.borrow().inner.clone() as Arc<dyn LanceNamespaceTrait>);
         }
     }
 
     // Custom Python implementation or subclass - wrap with PyLanceNamespace
-    PyLanceNamespace::create_arc(py, ns)
+    PyLanceNamespace::create_arc(py, namespace_client)
 }
 
 /// Python wrapper for REST adapter server
@@ -911,21 +1595,21 @@ impl PyRestAdapter {
     /// Default port is 2333 per REST spec. Use port 0 to let OS assign an ephemeral port.
     /// Use `port` property after `start()` to get the actual port.
     #[new]
-    #[pyo3(signature = (namespace_impl, namespace_properties, session = None, host = None, port = None))]
+    #[pyo3(signature = (namespace_client_impl, namespace_client_properties, session = None, host = None, port = None))]
     fn new(
-        namespace_impl: String,
-        namespace_properties: Option<&Bound<'_, PyDict>>,
+        namespace_client_impl: String,
+        namespace_client_properties: Option<&Bound<'_, PyDict>>,
         session: Option<&Bound<'_, Session>>,
         host: Option<String>,
         port: Option<u16>,
     ) -> PyResult<Self> {
         let mut props = HashMap::new();
 
-        if let Some(dict) = namespace_properties {
+        if let Some(dict) = namespace_client_properties {
             props = dict_to_hashmap(dict)?;
         }
 
-        let mut builder = ConnectBuilder::new(namespace_impl);
+        let mut builder = ConnectBuilder::new(namespace_client_impl);
         for (k, v) in props {
             builder = builder.property(k, v);
         }
diff --git a/python/src/storage_options.rs b/python/src/storage_options.rs
index ca37c984243..15f3aa2ce17 100644
--- a/python/src/storage_options.rs
+++ b/python/src/storage_options.rs
@@ -4,154 +4,11 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use async_trait::async_trait;
-use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
+use lance_io::object_store::StorageOptionsAccessor;
 use pyo3::prelude::*;
-use pyo3::types::PyDict;
 
 use crate::rt;
 
-/// Internal wrapper for Python storage options providers
-///
-/// This is not exposed to Python. Users pass their Python objects directly
-/// to dataset functions, and we wrap them internally with this struct.
-pub struct PyStorageOptionsProvider {
-    /// The Python object implementing get_storage_options()
-    inner: Py<PyAny>,
-}
-
-impl std::fmt::Debug for PyStorageOptionsProvider {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        // Note: We can't call provider_id() here because this is PyStorageOptionsProvider,
-        // not PyStorageOptionsProviderWrapper. Just use a simple format.
-        write!(f, "PyStorageOptionsProvider")
-    }
-}
-
-impl Clone for PyStorageOptionsProvider {
-    fn clone(&self) -> Self {
-        Python::attach(|py| Self {
-            inner: self.inner.clone_ref(py),
-        })
-    }
-}
-
-impl PyStorageOptionsProvider {
-    pub fn new(obj: &Bound<'_, PyAny>) -> PyResult<Self> {
-        // Verify the object has a fetch_storage_options method
-        if !obj.hasattr("fetch_storage_options")? {
-            return Err(pyo3::exceptions::PyTypeError::new_err(
-                "StorageOptionsProvider must implement fetch_storage_options() method",
-            ));
-        }
-        Ok(Self {
-            inner: obj.clone().unbind(),
-        })
-    }
-}
-
-/// Rust wrapper that implements StorageOptionsProvider trait for Python objects
-pub struct PyStorageOptionsProviderWrapper {
-    py_provider: PyStorageOptionsProvider,
-}
-
-impl std::fmt::Debug for PyStorageOptionsProviderWrapper {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.provider_id())
-    }
-}
-
-impl std::fmt::Display for PyStorageOptionsProviderWrapper {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.provider_id())
-    }
-}
-
-impl PyStorageOptionsProviderWrapper {
-    pub fn new(py_provider: PyStorageOptionsProvider) -> Self {
-        Self { py_provider }
-    }
-}
-
-#[async_trait]
-impl StorageOptionsProvider for PyStorageOptionsProviderWrapper {
-    async fn fetch_storage_options(&self) -> lance_core::Result<Option<HashMap<String, String>>> {
-        // Call Python method from async context
-        let py_provider = self.py_provider.clone();
-
-        rt().runtime
-            .spawn_blocking(move || {
-                Python::attach(|py| {
-                    // Call the Python fetch_storage_options method
-                    let result = py_provider
-                        .inner
-                        .bind(py)
-                        .call_method0("fetch_storage_options")
-                        .map_err(|e| lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                            "Failed to call fetch_storage_options: {}",
-                            e
-                        )))))?;
-
-                    // If result is None, return None
-                    if result.is_none() {
-                        return Ok(None);
-                    }
-
-                    // Extract the result dict - should be a flat Map<String, String>
-                    let result_dict = result.downcast::<PyDict>().map_err(|_| {
-                        lance_core::Error::invalid_input_source("fetch_storage_options() must return None or a dict of string key-value pairs"
-                            .into())
-                    })?;
-
-                    // Convert all entries to HashMap<String, String>
-                    let mut storage_options = HashMap::new();
-                    for (key, value) in result_dict.iter() {
-                        let key_str: String =
-                            key.extract().map_err(|e| lance_core::Error::invalid_input_source(format!("storage option keys must be strings: {}", e).into()))?;
-                        let value_str: String =
-                            value
-                                .extract()
-                                .map_err(|e| lance_core::Error::invalid_input_source(format!("storage option values must be strings: {}", e)
-                                    .into()))?;
-                        storage_options.insert(key_str, value_str);
-                    }
-
-                    Ok(Some(storage_options))
-                })
-            })
-            .await
-            .map_err(|e| lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to call Python fetch_storage_options: {}",
-                e
-            )))))?
-    }
-
-    fn provider_id(&self) -> String {
-        Python::attach(|py| {
-            // Call provider_id() method on the Python object
-            // This should always succeed since StorageOptionsProvider.provider_id() has a default implementation
-            let obj = self.py_provider.inner.bind(py);
-            obj.call_method0("provider_id")
-                .and_then(|result| result.extract::<String>())
-                .unwrap_or_else(|e| {
-                    panic!(
-                        "Failed to call provider_id() on Python StorageOptionsProvider: {}",
-                        e
-                    )
-                })
-        })
-    }
-}
-
-/// Convert a Python object to an Arc<dyn StorageOptionsProvider>
-/// This is the main entry point for converting Python storage options providers to Rust
-pub fn py_object_to_storage_options_provider(
-    py_obj: &Bound<'_, PyAny>,
-) -> PyResult<Arc<dyn StorageOptionsProvider>> {
-    let py_provider = PyStorageOptionsProvider::new(py_obj)?;
-    Ok(Arc::new(PyStorageOptionsProviderWrapper::new(py_provider)))
-}
-
 /// Python wrapper for StorageOptionsAccessor
 ///
 /// This wraps a Rust StorageOptionsAccessor and exposes it to Python.
@@ -181,34 +38,6 @@ impl PyStorageOptionsAccessor {
         }
     }
 
-    /// Create an accessor with a dynamic provider (no initial options)
-    ///
-    /// The refresh offset is extracted from storage options using the `refresh_offset_millis` key.
-    #[staticmethod]
-    fn with_provider(provider: &Bound<'_, PyAny>) -> PyResult<Self> {
-        let rust_provider = py_object_to_storage_options_provider(provider)?;
-        Ok(Self {
-            inner: Arc::new(StorageOptionsAccessor::with_provider(rust_provider)),
-        })
-    }
-
-    /// Create an accessor with initial options and a dynamic provider
-    ///
-    /// The refresh offset is extracted from initial_options using the `refresh_offset_millis` key.
-    #[staticmethod]
-    fn with_initial_and_provider(
-        initial_options: HashMap<String, String>,
-        provider: &Bound<'_, PyAny>,
-    ) -> PyResult<Self> {
-        let rust_provider = py_object_to_storage_options_provider(provider)?;
-        Ok(Self {
-            inner: Arc::new(StorageOptionsAccessor::with_initial_and_provider(
-                initial_options,
-                rust_provider,
-            )),
-        })
-    }
-
     /// Get current valid storage options
     fn get_storage_options(&self, py: Python<'_>) -> PyResult<HashMap<String, String>> {
         let accessor = self.inner.clone();
@@ -247,31 +76,17 @@ impl PyStorageOptionsAccessor {
     }
 }
 
-/// Create a StorageOptionsAccessor from Python parameters
+/// Create a StorageOptionsAccessor from storage options
 ///
-/// This handles the conversion from Python types to Rust StorageOptionsAccessor.
-/// The refresh offset is extracted from storage_options using the `refresh_offset_millis` key.
+/// This creates an accessor with static options only.
 #[allow(dead_code)]
-pub fn create_accessor_from_python(
+pub fn create_accessor_from_storage_options(
     storage_options: Option<HashMap<String, String>>,
-    storage_options_provider: Option<&Bound<'_, PyAny>>,
 ) -> PyResult<Option<Arc<StorageOptionsAccessor>>> {
-    match (storage_options, storage_options_provider) {
-        (Some(opts), Some(provider)) => {
-            let rust_provider = py_object_to_storage_options_provider(provider)?;
-            Ok(Some(Arc::new(
-                StorageOptionsAccessor::with_initial_and_provider(opts, rust_provider),
-            )))
-        }
-        (None, Some(provider)) => {
-            let rust_provider = py_object_to_storage_options_provider(provider)?;
-            Ok(Some(Arc::new(StorageOptionsAccessor::with_provider(
-                rust_provider,
-            ))))
-        }
-        (Some(opts), None) => Ok(Some(Arc::new(StorageOptionsAccessor::with_static_options(
+    match storage_options {
+        Some(opts) => Ok(Some(Arc::new(StorageOptionsAccessor::with_static_options(
             opts,
         )))),
-        (None, None) => Ok(None),
+        None => Ok(None),
     }
 }
diff --git a/python/src/transaction.rs b/python/src/transaction.rs
index 1bd0123b405..eae5b49a15d 100644
--- a/python/src/transaction.rs
+++ b/python/src/transaction.rs
@@ -11,7 +11,7 @@ use lance::dataset::transaction::{
     UpdateMapEntry, UpdateMode,
 };
 use lance::datatypes::Schema;
-use lance_table::format::{BasePath, DataFile, Fragment, IndexMetadata};
+use lance_table::format::{BasePath, DataFile, Fragment, IndexFile, IndexMetadata};
 use pyo3::exceptions::PyValueError;
 use pyo3::types::PySet;
 use pyo3::{Bound, FromPyObject, PyAny, PyResult, Python};
@@ -21,7 +21,43 @@ use std::collections::HashMap;
 use std::sync::Arc;
 use uuid::Uuid;
 
-// Add Index bindings
+// IndexFile bindings
+impl FromPyObject<'_> for PyLance<IndexFile> {
+    fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult<Self> {
+        let path = ob.getattr("path")?.extract()?;
+        let size_bytes = ob.getattr("size_bytes")?.extract()?;
+        Ok(Self(IndexFile { path, size_bytes }))
+    }
+}
+
+impl<'py> IntoPyObject<'py> for PyLance<&IndexFile> {
+    type Target = PyAny;
+    type Output = Bound<'py, Self::Target>;
+    type Error = PyErr;
+
+    fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
+        let namespace = py
+            .import(intern!(py, "lance"))
+            .expect("Failed to import lance module");
+
+        let cls = namespace
+            .getattr("IndexFile")
+            .expect("Failed to get IndexFile class");
+        cls.call1((self.0.path.clone(), self.0.size_bytes))
+    }
+}
+
+impl<'py> IntoPyObject<'py> for PyLance<IndexFile> {
+    type Target = PyAny;
+    type Output = Bound<'py, Self::Target>;
+    type Error = PyErr;
+
+    fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
+        PyLance(&self.0).into_pyobject(py)
+    }
+}
+
+// IndexMetadata bindings
 impl FromPyObject<'_> for PyLance<IndexMetadata> {
     fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult<Self> {
         let uuid = ob.getattr("uuid")?.to_string();
@@ -44,16 +80,28 @@ impl FromPyObject<'_> for PyLance<IndexMetadata> {
             .extract::<Option<i64>>()?
             .map(|id| id as u32);
 
+        let files: Option<Vec<IndexFile>> = ob
+            .getattr("files")?
+            .extract::<Option<Vec<PyLance<IndexFile>>>>()?
+            .map(|v| v.into_iter().map(|f| f.0).collect());
+        let index_details = match ob.getattr("index_details") {
+            Ok(details) => details
+                .extract::<Option<(String, Vec<u8>)>>()?
+                .map(|(type_url, value)| Arc::new(prost_types::Any { type_url, value })),
+            Err(_) => None,
+        };
+
         Ok(Self(IndexMetadata {
             uuid: Uuid::parse_str(&uuid).map_err(|e| PyValueError::new_err(e.to_string()))?,
             name,
             fields,
             dataset_version,
             fragment_bitmap,
-            index_details: None,
+            index_details,
             index_version,
             created_at,
             base_id,
+            files,
         }))
     }
 }
@@ -85,6 +133,17 @@ impl<'py> IntoPyObject<'py> for PyLance<&IndexMetadata> {
         );
         let created_at = self.0.created_at;
         let base_id = self.0.base_id.map(|id| id as i64);
+        let files = self
+            .0
+            .files
+            .as_ref()
+            .map(|f| export_vec(py, f.as_slice()))
+            .transpose()?;
+        let index_details = self
+            .0
+            .index_details
+            .as_ref()
+            .map(|details| (details.type_url.clone(), details.value.clone()));
 
         let cls = namespace
             .getattr("Index")
@@ -98,6 +157,8 @@ impl<'py> IntoPyObject<'py> for PyLance<&IndexMetadata> {
             index_version,
             created_at,
             base_id,
+            files,
+            index_details,
         ))
     }
 }
@@ -639,6 +700,7 @@ impl FromPyObject<'_> for PyLance<RewrittenIndex> {
                 value: new_details_value,
             },
             new_index_version,
+            new_index_files: None,
         }))
     }
 }
diff --git a/python/uv.lock b/python/uv.lock
index 351f63aa8ed..55e274b03e9 100644
--- a/python/uv.lock
+++ b/python/uv.lock
@@ -1,115 +1,168 @@
 version = 1
-requires-python = ">=3.10"
+revision = 3
+requires-python = ">=3.9"
 resolution-markers = [
     "python_full_version >= '3.14'",
     "python_full_version == '3.13.*'",
     "python_full_version == '3.12.*'",
     "python_full_version == '3.11.*'",
-    "python_full_version < '3.11'",
+    "python_full_version == '3.10.*'",
+    "python_full_version < '3.10'",
 ]
 
 [[package]]
 name = "absl-py"
 version = "2.3.1"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/10/2a/c93173ffa1b39c1d0395b7e842bbdc62e556ca9d8d3b5572926f3e4ca752/absl_py-2.3.1.tar.gz", hash = "sha256:a97820526f7fbfd2ec1bce83f3f25e3a14840dac0d8e02a0b71cd75db3f77fc9", size = 116588 }
+sdist = { url = "https://files.pythonhosted.org/packages/10/2a/c93173ffa1b39c1d0395b7e842bbdc62e556ca9d8d3b5572926f3e4ca752/absl_py-2.3.1.tar.gz", hash = "sha256:a97820526f7fbfd2ec1bce83f3f25e3a14840dac0d8e02a0b71cd75db3f77fc9", size = 116588, upload-time = "2025-07-03T09:31:44.05Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/8f/aa/ba0014cc4659328dc818a28827be78e6d97312ab0cb98105a770924dc11e/absl_py-2.3.1-py3-none-any.whl", hash = "sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d", size = 135811 },
+    { url = "https://files.pythonhosted.org/packages/8f/aa/ba0014cc4659328dc818a28827be78e6d97312ab0cb98105a770924dc11e/absl_py-2.3.1-py3-none-any.whl", hash = "sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d", size = 135811, upload-time = "2025-07-03T09:31:42.253Z" },
 ]
 
 [[package]]
 name = "aiohappyeyeballs"
 version = "2.6.1"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760 }
+sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265 },
+    { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" },
 ]
 
 [[package]]
 name = "aiohttp"
-version = "3.12.15"
+version = "3.13.4"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "aiohappyeyeballs" },
-    { name = "aiosignal" },
-    { name = "async-timeout", marker = "python_full_version < '3.11'" },
-    { name = "attrs" },
-    { name = "frozenlist" },
-    { name = "multidict" },
-    { name = "propcache" },
-    { name = "yarl" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/9b/e7/d92a237d8802ca88483906c388f7c201bbe96cd80a165ffd0ac2f6a8d59f/aiohttp-3.12.15.tar.gz", hash = "sha256:4fc61385e9c98d72fcdf47e6dd81833f47b2f77c114c29cd64a361be57a763a2", size = 7823716 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/47/dc/ef9394bde9080128ad401ac7ede185267ed637df03b51f05d14d1c99ad67/aiohttp-3.12.15-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b6fc902bff74d9b1879ad55f5404153e2b33a82e72a95c89cec5eb6cc9e92fbc", size = 703921 },
-    { url = "https://files.pythonhosted.org/packages/8f/42/63fccfc3a7ed97eb6e1a71722396f409c46b60a0552d8a56d7aad74e0df5/aiohttp-3.12.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:098e92835b8119b54c693f2f88a1dec690e20798ca5f5fe5f0520245253ee0af", size = 480288 },
-    { url = "https://files.pythonhosted.org/packages/9c/a2/7b8a020549f66ea2a68129db6960a762d2393248f1994499f8ba9728bbed/aiohttp-3.12.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:40b3fee496a47c3b4a39a731954c06f0bd9bd3e8258c059a4beb76ac23f8e421", size = 468063 },
-    { url = "https://files.pythonhosted.org/packages/8f/f5/d11e088da9176e2ad8220338ae0000ed5429a15f3c9dfd983f39105399cd/aiohttp-3.12.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ce13fcfb0bb2f259fb42106cdc63fa5515fb85b7e87177267d89a771a660b79", size = 1650122 },
-    { url = "https://files.pythonhosted.org/packages/b0/6b/b60ce2757e2faed3d70ed45dafee48cee7bfb878785a9423f7e883f0639c/aiohttp-3.12.15-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3beb14f053222b391bf9cf92ae82e0171067cc9c8f52453a0f1ec7c37df12a77", size = 1624176 },
-    { url = "https://files.pythonhosted.org/packages/dd/de/8c9fde2072a1b72c4fadecf4f7d4be7a85b1d9a4ab333d8245694057b4c6/aiohttp-3.12.15-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c39e87afe48aa3e814cac5f535bc6199180a53e38d3f51c5e2530f5aa4ec58c", size = 1696583 },
-    { url = "https://files.pythonhosted.org/packages/0c/ad/07f863ca3d895a1ad958a54006c6dafb4f9310f8c2fdb5f961b8529029d3/aiohttp-3.12.15-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5f1b4ce5bc528a6ee38dbf5f39bbf11dd127048726323b72b8e85769319ffc4", size = 1738896 },
-    { url = "https://files.pythonhosted.org/packages/20/43/2bd482ebe2b126533e8755a49b128ec4e58f1a3af56879a3abdb7b42c54f/aiohttp-3.12.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1004e67962efabbaf3f03b11b4c43b834081c9e3f9b32b16a7d97d4708a9abe6", size = 1643561 },
-    { url = "https://files.pythonhosted.org/packages/23/40/2fa9f514c4cf4cbae8d7911927f81a1901838baf5e09a8b2c299de1acfe5/aiohttp-3.12.15-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8faa08fcc2e411f7ab91d1541d9d597d3a90e9004180edb2072238c085eac8c2", size = 1583685 },
-    { url = "https://files.pythonhosted.org/packages/b8/c3/94dc7357bc421f4fb978ca72a201a6c604ee90148f1181790c129396ceeb/aiohttp-3.12.15-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:fe086edf38b2222328cdf89af0dde2439ee173b8ad7cb659b4e4c6f385b2be3d", size = 1627533 },
-    { url = "https://files.pythonhosted.org/packages/bf/3f/1f8911fe1844a07001e26593b5c255a685318943864b27b4e0267e840f95/aiohttp-3.12.15-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:79b26fe467219add81d5e47b4a4ba0f2394e8b7c7c3198ed36609f9ba161aecb", size = 1638319 },
-    { url = "https://files.pythonhosted.org/packages/4e/46/27bf57a99168c4e145ffee6b63d0458b9c66e58bb70687c23ad3d2f0bd17/aiohttp-3.12.15-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b761bac1192ef24e16706d761aefcb581438b34b13a2f069a6d343ec8fb693a5", size = 1613776 },
-    { url = "https://files.pythonhosted.org/packages/0f/7e/1d2d9061a574584bb4ad3dbdba0da90a27fdc795bc227def3a46186a8bc1/aiohttp-3.12.15-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e153e8adacfe2af562861b72f8bc47f8a5c08e010ac94eebbe33dc21d677cd5b", size = 1693359 },
-    { url = "https://files.pythonhosted.org/packages/08/98/bee429b52233c4a391980a5b3b196b060872a13eadd41c3a34be9b1469ed/aiohttp-3.12.15-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:fc49c4de44977aa8601a00edbf157e9a421f227aa7eb477d9e3df48343311065", size = 1716598 },
-    { url = "https://files.pythonhosted.org/packages/57/39/b0314c1ea774df3392751b686104a3938c63ece2b7ce0ba1ed7c0b4a934f/aiohttp-3.12.15-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2776c7ec89c54a47029940177e75c8c07c29c66f73464784971d6a81904ce9d1", size = 1644940 },
-    { url = "https://files.pythonhosted.org/packages/1b/83/3dacb8d3f8f512c8ca43e3fa8a68b20583bd25636ffa4e56ee841ffd79ae/aiohttp-3.12.15-cp310-cp310-win32.whl", hash = "sha256:2c7d81a277fa78b2203ab626ced1487420e8c11a8e373707ab72d189fcdad20a", size = 429239 },
-    { url = "https://files.pythonhosted.org/packages/eb/f9/470b5daba04d558c9673ca2034f28d067f3202a40e17804425f0c331c89f/aiohttp-3.12.15-cp310-cp310-win_amd64.whl", hash = "sha256:83603f881e11f0f710f8e2327817c82e79431ec976448839f3cd05d7afe8f830", size = 452297 },
-    { url = "https://files.pythonhosted.org/packages/20/19/9e86722ec8e835959bd97ce8c1efa78cf361fa4531fca372551abcc9cdd6/aiohttp-3.12.15-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d3ce17ce0220383a0f9ea07175eeaa6aa13ae5a41f30bc61d84df17f0e9b1117", size = 711246 },
-    { url = "https://files.pythonhosted.org/packages/71/f9/0a31fcb1a7d4629ac9d8f01f1cb9242e2f9943f47f5d03215af91c3c1a26/aiohttp-3.12.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:010cc9bbd06db80fe234d9003f67e97a10fe003bfbedb40da7d71c1008eda0fe", size = 483515 },
-    { url = "https://files.pythonhosted.org/packages/62/6c/94846f576f1d11df0c2e41d3001000527c0fdf63fce7e69b3927a731325d/aiohttp-3.12.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3f9d7c55b41ed687b9d7165b17672340187f87a773c98236c987f08c858145a9", size = 471776 },
-    { url = "https://files.pythonhosted.org/packages/f8/6c/f766d0aaafcee0447fad0328da780d344489c042e25cd58fde566bf40aed/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc4fbc61bb3548d3b482f9ac7ddd0f18c67e4225aaa4e8552b9f1ac7e6bda9e5", size = 1741977 },
-    { url = "https://files.pythonhosted.org/packages/17/e5/fb779a05ba6ff44d7bc1e9d24c644e876bfff5abe5454f7b854cace1b9cc/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7fbc8a7c410bb3ad5d595bb7118147dfbb6449d862cc1125cf8867cb337e8728", size = 1690645 },
-    { url = "https://files.pythonhosted.org/packages/37/4e/a22e799c2035f5d6a4ad2cf8e7c1d1bd0923192871dd6e367dafb158b14c/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:74dad41b3458dbb0511e760fb355bb0b6689e0630de8a22b1b62a98777136e16", size = 1789437 },
-    { url = "https://files.pythonhosted.org/packages/28/e5/55a33b991f6433569babb56018b2fb8fb9146424f8b3a0c8ecca80556762/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b6f0af863cf17e6222b1735a756d664159e58855da99cfe965134a3ff63b0b0", size = 1828482 },
-    { url = "https://files.pythonhosted.org/packages/c6/82/1ddf0ea4f2f3afe79dffed5e8a246737cff6cbe781887a6a170299e33204/aiohttp-3.12.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5b7fe4972d48a4da367043b8e023fb70a04d1490aa7d68800e465d1b97e493b", size = 1730944 },
-    { url = "https://files.pythonhosted.org/packages/1b/96/784c785674117b4cb3877522a177ba1b5e4db9ce0fd519430b5de76eec90/aiohttp-3.12.15-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6443cca89553b7a5485331bc9bedb2342b08d073fa10b8c7d1c60579c4a7b9bd", size = 1668020 },
-    { url = "https://files.pythonhosted.org/packages/12/8a/8b75f203ea7e5c21c0920d84dd24a5c0e971fe1e9b9ebbf29ae7e8e39790/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c5f40ec615e5264f44b4282ee27628cea221fcad52f27405b80abb346d9f3f8", size = 1716292 },
-    { url = "https://files.pythonhosted.org/packages/47/0b/a1451543475bb6b86a5cfc27861e52b14085ae232896a2654ff1231c0992/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2abbb216a1d3a2fe86dbd2edce20cdc5e9ad0be6378455b05ec7f77361b3ab50", size = 1711451 },
-    { url = "https://files.pythonhosted.org/packages/55/fd/793a23a197cc2f0d29188805cfc93aa613407f07e5f9da5cd1366afd9d7c/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:db71ce547012a5420a39c1b744d485cfb823564d01d5d20805977f5ea1345676", size = 1691634 },
-    { url = "https://files.pythonhosted.org/packages/ca/bf/23a335a6670b5f5dfc6d268328e55a22651b440fca341a64fccf1eada0c6/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ced339d7c9b5030abad5854aa5413a77565e5b6e6248ff927d3e174baf3badf7", size = 1785238 },
-    { url = "https://files.pythonhosted.org/packages/57/4f/ed60a591839a9d85d40694aba5cef86dde9ee51ce6cca0bb30d6eb1581e7/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:7c7dd29c7b5bda137464dc9bfc738d7ceea46ff70309859ffde8c022e9b08ba7", size = 1805701 },
-    { url = "https://files.pythonhosted.org/packages/85/e0/444747a9455c5de188c0f4a0173ee701e2e325d4b2550e9af84abb20cdba/aiohttp-3.12.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:421da6fd326460517873274875c6c5a18ff225b40da2616083c5a34a7570b685", size = 1718758 },
-    { url = "https://files.pythonhosted.org/packages/36/ab/1006278d1ffd13a698e5dd4bfa01e5878f6bddefc296c8b62649753ff249/aiohttp-3.12.15-cp311-cp311-win32.whl", hash = "sha256:4420cf9d179ec8dfe4be10e7d0fe47d6d606485512ea2265b0d8c5113372771b", size = 428868 },
-    { url = "https://files.pythonhosted.org/packages/10/97/ad2b18700708452400278039272032170246a1bf8ec5d832772372c71f1a/aiohttp-3.12.15-cp311-cp311-win_amd64.whl", hash = "sha256:edd533a07da85baa4b423ee8839e3e91681c7bfa19b04260a469ee94b778bf6d", size = 453273 },
-    { url = "https://files.pythonhosted.org/packages/63/97/77cb2450d9b35f517d6cf506256bf4f5bda3f93a66b4ad64ba7fc917899c/aiohttp-3.12.15-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:802d3868f5776e28f7bf69d349c26fc0efadb81676d0afa88ed00d98a26340b7", size = 702333 },
-    { url = "https://files.pythonhosted.org/packages/83/6d/0544e6b08b748682c30b9f65640d006e51f90763b41d7c546693bc22900d/aiohttp-3.12.15-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2800614cd560287be05e33a679638e586a2d7401f4ddf99e304d98878c29444", size = 476948 },
-    { url = "https://files.pythonhosted.org/packages/3a/1d/c8c40e611e5094330284b1aea8a4b02ca0858f8458614fa35754cab42b9c/aiohttp-3.12.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8466151554b593909d30a0a125d638b4e5f3836e5aecde85b66b80ded1cb5b0d", size = 469787 },
-    { url = "https://files.pythonhosted.org/packages/38/7d/b76438e70319796bfff717f325d97ce2e9310f752a267bfdf5192ac6082b/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e5a495cb1be69dae4b08f35a6c4579c539e9b5706f606632102c0f855bcba7c", size = 1716590 },
-    { url = "https://files.pythonhosted.org/packages/79/b1/60370d70cdf8b269ee1444b390cbd72ce514f0d1cd1a715821c784d272c9/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6404dfc8cdde35c69aaa489bb3542fb86ef215fc70277c892be8af540e5e21c0", size = 1699241 },
-    { url = "https://files.pythonhosted.org/packages/a3/2b/4968a7b8792437ebc12186db31523f541943e99bda8f30335c482bea6879/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ead1c00f8521a5c9070fcb88f02967b1d8a0544e6d85c253f6968b785e1a2ab", size = 1754335 },
-    { url = "https://files.pythonhosted.org/packages/fb/c1/49524ed553f9a0bec1a11fac09e790f49ff669bcd14164f9fab608831c4d/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6990ef617f14450bc6b34941dba4f12d5613cbf4e33805932f853fbd1cf18bfb", size = 1800491 },
-    { url = "https://files.pythonhosted.org/packages/de/5e/3bf5acea47a96a28c121b167f5ef659cf71208b19e52a88cdfa5c37f1fcc/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd736ed420f4db2b8148b52b46b88ed038d0354255f9a73196b7bbce3ea97545", size = 1719929 },
-    { url = "https://files.pythonhosted.org/packages/39/94/8ae30b806835bcd1cba799ba35347dee6961a11bd507db634516210e91d8/aiohttp-3.12.15-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c5092ce14361a73086b90c6efb3948ffa5be2f5b6fbcf52e8d8c8b8848bb97c", size = 1635733 },
-    { url = "https://files.pythonhosted.org/packages/7a/46/06cdef71dd03acd9da7f51ab3a9107318aee12ad38d273f654e4f981583a/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aaa2234bb60c4dbf82893e934d8ee8dea30446f0647e024074237a56a08c01bd", size = 1696790 },
-    { url = "https://files.pythonhosted.org/packages/02/90/6b4cfaaf92ed98d0ec4d173e78b99b4b1a7551250be8937d9d67ecb356b4/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6d86a2fbdd14192e2f234a92d3b494dd4457e683ba07e5905a0b3ee25389ac9f", size = 1718245 },
-    { url = "https://files.pythonhosted.org/packages/2e/e6/2593751670fa06f080a846f37f112cbe6f873ba510d070136a6ed46117c6/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a041e7e2612041a6ddf1c6a33b883be6a421247c7afd47e885969ee4cc58bd8d", size = 1658899 },
-    { url = "https://files.pythonhosted.org/packages/8f/28/c15bacbdb8b8eb5bf39b10680d129ea7410b859e379b03190f02fa104ffd/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5015082477abeafad7203757ae44299a610e89ee82a1503e3d4184e6bafdd519", size = 1738459 },
-    { url = "https://files.pythonhosted.org/packages/00/de/c269cbc4faa01fb10f143b1670633a8ddd5b2e1ffd0548f7aa49cb5c70e2/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:56822ff5ddfd1b745534e658faba944012346184fbfe732e0d6134b744516eea", size = 1766434 },
-    { url = "https://files.pythonhosted.org/packages/52/b0/4ff3abd81aa7d929b27d2e1403722a65fc87b763e3a97b3a2a494bfc63bc/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b2acbbfff69019d9014508c4ba0401822e8bae5a5fdc3b6814285b71231b60f3", size = 1726045 },
-    { url = "https://files.pythonhosted.org/packages/71/16/949225a6a2dd6efcbd855fbd90cf476052e648fb011aa538e3b15b89a57a/aiohttp-3.12.15-cp312-cp312-win32.whl", hash = "sha256:d849b0901b50f2185874b9a232f38e26b9b3d4810095a7572eacea939132d4e1", size = 423591 },
-    { url = "https://files.pythonhosted.org/packages/2b/d8/fa65d2a349fe938b76d309db1a56a75c4fb8cc7b17a398b698488a939903/aiohttp-3.12.15-cp312-cp312-win_amd64.whl", hash = "sha256:b390ef5f62bb508a9d67cb3bba9b8356e23b3996da7062f1a57ce1a79d2b3d34", size = 450266 },
-    { url = "https://files.pythonhosted.org/packages/f2/33/918091abcf102e39d15aba2476ad9e7bd35ddb190dcdd43a854000d3da0d/aiohttp-3.12.15-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9f922ffd05034d439dde1c77a20461cf4a1b0831e6caa26151fe7aa8aaebc315", size = 696741 },
-    { url = "https://files.pythonhosted.org/packages/b5/2a/7495a81e39a998e400f3ecdd44a62107254803d1681d9189be5c2e4530cd/aiohttp-3.12.15-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2ee8a8ac39ce45f3e55663891d4b1d15598c157b4d494a4613e704c8b43112cd", size = 474407 },
-    { url = "https://files.pythonhosted.org/packages/49/fc/a9576ab4be2dcbd0f73ee8675d16c707cfc12d5ee80ccf4015ba543480c9/aiohttp-3.12.15-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3eae49032c29d356b94eee45a3f39fdf4b0814b397638c2f718e96cfadf4c4e4", size = 466703 },
-    { url = "https://files.pythonhosted.org/packages/09/2f/d4bcc8448cf536b2b54eed48f19682031ad182faa3a3fee54ebe5b156387/aiohttp-3.12.15-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97752ff12cc12f46a9b20327104448042fce5c33a624f88c18f66f9368091c7", size = 1705532 },
-    { url = "https://files.pythonhosted.org/packages/f1/f3/59406396083f8b489261e3c011aa8aee9df360a96ac8fa5c2e7e1b8f0466/aiohttp-3.12.15-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:894261472691d6fe76ebb7fcf2e5870a2ac284c7406ddc95823c8598a1390f0d", size = 1686794 },
-    { url = "https://files.pythonhosted.org/packages/dc/71/164d194993a8d114ee5656c3b7ae9c12ceee7040d076bf7b32fb98a8c5c6/aiohttp-3.12.15-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5fa5d9eb82ce98959fc1031c28198b431b4d9396894f385cb63f1e2f3f20ca6b", size = 1738865 },
-    { url = "https://files.pythonhosted.org/packages/1c/00/d198461b699188a93ead39cb458554d9f0f69879b95078dce416d3209b54/aiohttp-3.12.15-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0fa751efb11a541f57db59c1dd821bec09031e01452b2b6217319b3a1f34f3d", size = 1788238 },
-    { url = "https://files.pythonhosted.org/packages/85/b8/9e7175e1fa0ac8e56baa83bf3c214823ce250d0028955dfb23f43d5e61fd/aiohttp-3.12.15-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5346b93e62ab51ee2a9d68e8f73c7cf96ffb73568a23e683f931e52450e4148d", size = 1710566 },
-    { url = "https://files.pythonhosted.org/packages/59/e4/16a8eac9df39b48ae102ec030fa9f726d3570732e46ba0c592aeeb507b93/aiohttp-3.12.15-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:049ec0360f939cd164ecbfd2873eaa432613d5e77d6b04535e3d1fbae5a9e645", size = 1624270 },
-    { url = "https://files.pythonhosted.org/packages/1f/f8/cd84dee7b6ace0740908fd0af170f9fab50c2a41ccbc3806aabcb1050141/aiohttp-3.12.15-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b52dcf013b57464b6d1e51b627adfd69a8053e84b7103a7cd49c030f9ca44461", size = 1677294 },
-    { url = "https://files.pythonhosted.org/packages/ce/42/d0f1f85e50d401eccd12bf85c46ba84f947a84839c8a1c2c5f6e8ab1eb50/aiohttp-3.12.15-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:9b2af240143dd2765e0fb661fd0361a1b469cab235039ea57663cda087250ea9", size = 1708958 },
-    { url = "https://files.pythonhosted.org/packages/d5/6b/f6fa6c5790fb602538483aa5a1b86fcbad66244997e5230d88f9412ef24c/aiohttp-3.12.15-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ac77f709a2cde2cc71257ab2d8c74dd157c67a0558a0d2799d5d571b4c63d44d", size = 1651553 },
-    { url = "https://files.pythonhosted.org/packages/04/36/a6d36ad545fa12e61d11d1932eef273928b0495e6a576eb2af04297fdd3c/aiohttp-3.12.15-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:47f6b962246f0a774fbd3b6b7be25d59b06fdb2f164cf2513097998fc6a29693", size = 1727688 },
-    { url = "https://files.pythonhosted.org/packages/aa/c8/f195e5e06608a97a4e52c5d41c7927301bf757a8e8bb5bbf8cef6c314961/aiohttp-3.12.15-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:760fb7db442f284996e39cf9915a94492e1896baac44f06ae551974907922b64", size = 1761157 },
-    { url = "https://files.pythonhosted.org/packages/05/6a/ea199e61b67f25ba688d3ce93f63b49b0a4e3b3d380f03971b4646412fc6/aiohttp-3.12.15-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ad702e57dc385cae679c39d318def49aef754455f237499d5b99bea4ef582e51", size = 1710050 },
-    { url = "https://files.pythonhosted.org/packages/b4/2e/ffeb7f6256b33635c29dbed29a22a723ff2dd7401fff42ea60cf2060abfb/aiohttp-3.12.15-cp313-cp313-win32.whl", hash = "sha256:f813c3e9032331024de2eb2e32a88d86afb69291fbc37a3a3ae81cc9917fb3d0", size = 422647 },
-    { url = "https://files.pythonhosted.org/packages/1b/8e/78ee35774201f38d5e1ba079c9958f7629b1fd079459aea9467441dbfbf5/aiohttp-3.12.15-cp313-cp313-win_amd64.whl", hash = "sha256:1a649001580bdb37c6fdb1bebbd7e3bc688e8ec2b5c6f52edbb664662b17dc84", size = 449067 },
+    { name = "aiohappyeyeballs", marker = "python_full_version >= '3.10'" },
+    { name = "aiosignal", marker = "python_full_version >= '3.10'" },
+    { name = "async-timeout", marker = "python_full_version == '3.10.*'" },
+    { name = "attrs", marker = "python_full_version >= '3.10'" },
+    { name = "frozenlist", marker = "python_full_version >= '3.10'" },
+    { name = "multidict", marker = "python_full_version >= '3.10'" },
+    { name = "propcache", marker = "python_full_version >= '3.10'" },
+    { name = "yarl", marker = "python_full_version >= '3.10'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/45/4a/064321452809dae953c1ed6e017504e72551a26b6f5708a5a80e4bf556ff/aiohttp-3.13.4.tar.gz", hash = "sha256:d97a6d09c66087890c2ab5d49069e1e570583f7ac0314ecf98294c1b6aaebd38", size = 7859748, upload-time = "2026-03-28T17:19:40.6Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2c/05/6817e0390eb47b0867cf8efdb535298191662192281bc3ca62a0cb7973eb/aiohttp-3.13.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6290fe12fe8cefa6ea3c1c5b969d32c010dfe191d4392ff9b599a3f473cbe722", size = 753094, upload-time = "2026-03-28T17:14:59.928Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/c1/e5b7f25f6dd1ab57da92aa9d226b2c8b56f223dd20475d3ddfddaba86ab8/aiohttp-3.13.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7520d92c0e8fbbe63f36f20a5762db349ff574ad38ad7bc7732558a650439845", size = 505213, upload-time = "2026-03-28T17:15:01.989Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/e5/8f42033c7ce98b54dfd3791f03e60231cfe4a2db4471b5fc188df2b8a6ad/aiohttp-3.13.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2710ae1e1b81d0f187883b6e9d66cecf8794b50e91aa1e73fc78bfb5503b5d9", size = 498580, upload-time = "2026-03-28T17:15:03.879Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/a4/bbc989f5362066b81930da1a66084a859a971d03faab799dc59a3ce3a220/aiohttp-3.13.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:717d17347567ded1e273aa09918650dfd6fd06f461549204570c7973537d4123", size = 1692718, upload-time = "2026-03-28T17:15:05.541Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/72/3775116969931f151be116689d2ae6ddafff2ec2887d8f9b4e7043f32e74/aiohttp-3.13.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:383880f7b8de5ac208fa829c7038d08e66377283b2de9e791b71e06e803153c2", size = 1660714, upload-time = "2026-03-28T17:15:08.23Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/e8/d2f1a2da2743e32fe348ebf8a4c59caad14a92f5f18af616fd33381275e1/aiohttp-3.13.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1867087e2c1963db1216aedf001efe3b129835ed2b05d97d058176a6d08b5726", size = 1744152, upload-time = "2026-03-28T17:15:10.828Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/a6/575886f417ac3c08e462f2ca237cc49f436bd992ca3f7ff95b7dd9c44205/aiohttp-3.13.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6234bf416a38d687c3ab7f79934d7fb2a42117a5b9813aca07de0a5398489023", size = 1836278, upload-time = "2026-03-28T17:15:12.537Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/4c/0051d4550fb9e8b5ca4e0fe1ccd58652340915180c5164999e6741bf2083/aiohttp-3.13.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3cdd3393130bf6588962441ffd5bde1d3ea2d63a64afa7119b3f3ba349cebbe7", size = 1687953, upload-time = "2026-03-28T17:15:14.248Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/54/841e87b8c51c2adc01a3ceb9919dc45c7899fe4c21deb70aada734ea5a38/aiohttp-3.13.4-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0d0dbc6c76befa76865373d6aa303e480bb8c3486e7763530f7f6e527b471118", size = 1572484, upload-time = "2026-03-28T17:15:15.911Z" },
+    { url = "https://files.pythonhosted.org/packages/da/f1/21cbf5f7fa1e267af6301f886cab9b314f085e4d0097668d189d165cd7da/aiohttp-3.13.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:10fb7b53262cf4144a083c9db0d2b4d22823d6708270a9970c4627b248c6064c", size = 1662851, upload-time = "2026-03-28T17:15:17.822Z" },
+    { url = "https://files.pythonhosted.org/packages/40/15/bcad6b68d7bef27ae7443288215767263c7753ede164267cf6cf63c94a87/aiohttp-3.13.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:eb10ce8c03850e77f4d9518961c227be569e12f71525a7e90d17bca04299921d", size = 1671984, upload-time = "2026-03-28T17:15:19.561Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/fa/ab316931afc7a73c7f493bb1b30fbd61e28ec2d3ea50353336e76293e8ec/aiohttp-3.13.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:7c65738ac5ae32b8feef699a4ed0dc91a0c8618b347781b7461458bbcaaac7eb", size = 1713880, upload-time = "2026-03-28T17:15:21.589Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/45/314e8e64c7f328174964b6db511dd5e9e60c9121ab5457bc2c908b7d03a4/aiohttp-3.13.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:6b335919ffbaf98df8ff3c74f7a6decb8775882632952fd1810a017e38f15aee", size = 1560315, upload-time = "2026-03-28T17:15:23.66Z" },
+    { url = "https://files.pythonhosted.org/packages/18/e7/93d5fa06fe00219a81466577dacae9e3732f3b4f767b12b2e2cc8c35c970/aiohttp-3.13.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:ec75fc18cb9f4aca51c2cbace20cf6716e36850f44189644d2d69a875d5e0532", size = 1735115, upload-time = "2026-03-28T17:15:25.77Z" },
+    { url = "https://files.pythonhosted.org/packages/19/9f/f64b95392ddd4e204fd9ab7cd33dd18d14ac9e4b86866f1f6a69b7cda83d/aiohttp-3.13.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:463fa18a95c5a635d2b8c09babe240f9d7dbf2a2010a6c0b35d8c4dff2a0e819", size = 1673916, upload-time = "2026-03-28T17:15:27.526Z" },
+    { url = "https://files.pythonhosted.org/packages/52/c1/bb33be79fd285c69f32e5b074b299cae8847f748950149c3965c1b3b3adf/aiohttp-3.13.4-cp310-cp310-win32.whl", hash = "sha256:13168f5645d9045522c6cef818f54295376257ed8d02513a37c2ef3046fc7a97", size = 440277, upload-time = "2026-03-28T17:15:29.173Z" },
+    { url = "https://files.pythonhosted.org/packages/23/f9/7cf1688da4dd0885f914ee40bc8e1dce776df98fe6518766de975a570538/aiohttp-3.13.4-cp310-cp310-win_amd64.whl", hash = "sha256:a7058af1f53209fdf07745579ced525d38d481650a989b7aa4a3b484b901cdab", size = 463015, upload-time = "2026-03-28T17:15:30.802Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/7e/cb94129302d78c46662b47f9897d642fd0b33bdfef4b73b20c6ced35aa4c/aiohttp-3.13.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8ea0c64d1bcbf201b285c2246c51a0c035ba3bbd306640007bc5844a3b4658c1", size = 760027, upload-time = "2026-03-28T17:15:33.022Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/cd/2db3c9397c3bd24216b203dd739945b04f8b87bb036c640da7ddb63c75ef/aiohttp-3.13.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6f742e1fa45c0ed522b00ede565e18f97e4cf8d1883a712ac42d0339dfb0cce7", size = 508325, upload-time = "2026-03-28T17:15:34.714Z" },
+    { url = "https://files.pythonhosted.org/packages/36/a3/d28b2722ec13107f2e37a86b8a169897308bab6a3b9e071ecead9d67bd9b/aiohttp-3.13.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dcfb50ee25b3b7a1222a9123be1f9f89e56e67636b561441f0b304e25aaef8f", size = 502402, upload-time = "2026-03-28T17:15:36.409Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/d6/acd47b5f17c4430e555590990a4746efbcb2079909bb865516892bf85f37/aiohttp-3.13.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3262386c4ff370849863ea93b9ea60fd59c6cf56bf8f93beac625cf4d677c04d", size = 1771224, upload-time = "2026-03-28T17:15:38.223Z" },
+    { url = "https://files.pythonhosted.org/packages/98/af/af6e20113ba6a48fd1cd9e5832c4851e7613ef50c7619acdaee6ec5f1aff/aiohttp-3.13.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:473bb5aa4218dd254e9ae4834f20e31f5a0083064ac0136a01a62ddbae2eaa42", size = 1731530, upload-time = "2026-03-28T17:15:39.988Z" },
+    { url = "https://files.pythonhosted.org/packages/81/16/78a2f5d9c124ad05d5ce59a9af94214b6466c3491a25fb70760e98e9f762/aiohttp-3.13.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e56423766399b4c77b965f6aaab6c9546617b8994a956821cc507d00b91d978c", size = 1827925, upload-time = "2026-03-28T17:15:41.944Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/1f/79acf0974ced805e0e70027389fccbb7d728e6f30fcac725fb1071e63075/aiohttp-3.13.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8af249343fafd5ad90366a16d230fc265cf1149f26075dc9fe93cfd7c7173942", size = 1923579, upload-time = "2026-03-28T17:15:44.071Z" },
+    { url = "https://files.pythonhosted.org/packages/af/53/29f9e2054ea6900413f3b4c3eb9d8331f60678ec855f13ba8714c47fd48d/aiohttp-3.13.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bc0a5cf4f10ef5a2c94fdde488734b582a3a7a000b131263e27c9295bd682d9", size = 1767655, upload-time = "2026-03-28T17:15:45.911Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/57/462fe1d3da08109ba4aa8590e7aed57c059af2a7e80ec21f4bac5cfe1094/aiohttp-3.13.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5c7ff1028e3c9fc5123a865ce17df1cb6424d180c503b8517afbe89aa566e6be", size = 1630439, upload-time = "2026-03-28T17:15:48.11Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/4b/4813344aacdb8127263e3eec343d24e973421143826364fa9fc847f6283f/aiohttp-3.13.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ba5cf98b5dcb9bddd857da6713a503fa6d341043258ca823f0f5ab7ab4a94ee8", size = 1745557, upload-time = "2026-03-28T17:15:50.13Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/01/1ef1adae1454341ec50a789f03cfafe4c4ac9c003f6a64515ecd32fe4210/aiohttp-3.13.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d85965d3ba21ee4999e83e992fecb86c4614d6920e40705501c0a1f80a583c12", size = 1741796, upload-time = "2026-03-28T17:15:52.351Z" },
+    { url = "https://files.pythonhosted.org/packages/22/04/8cdd99af988d2aa6922714d957d21383c559835cbd43fbf5a47ddf2e0f05/aiohttp-3.13.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:49f0b18a9b05d79f6f37ddd567695943fcefb834ef480f17a4211987302b2dc7", size = 1805312, upload-time = "2026-03-28T17:15:54.407Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/7f/b48d5577338d4b25bbdbae35c75dbfd0493cb8886dc586fbfb2e90862239/aiohttp-3.13.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7f78cb080c86fbf765920e5f1ef35af3f24ec4314d6675d0a21eaf41f6f2679c", size = 1621751, upload-time = "2026-03-28T17:15:56.564Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/89/4eecad8c1858e6d0893c05929e22343e0ebe3aec29a8a399c65c3cc38311/aiohttp-3.13.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:67a3ec705534a614b68bbf1c70efa777a21c3da3895d1c44510a41f5a7ae0453", size = 1826073, upload-time = "2026-03-28T17:15:58.489Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/5c/9dc8293ed31b46c39c9c513ac7ca152b3c3d38e0ea111a530ad12001b827/aiohttp-3.13.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6630ec917e85c5356b2295744c8a97d40f007f96a1c76bf1928dc2e27465393", size = 1760083, upload-time = "2026-03-28T17:16:00.677Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/19/8bbf6a4994205d96831f97b7d21a0feed120136e6267b5b22d229c6dc4dc/aiohttp-3.13.4-cp311-cp311-win32.whl", hash = "sha256:54049021bc626f53a5394c29e8c444f726ee5a14b6e89e0ad118315b1f90f5e3", size = 439690, upload-time = "2026-03-28T17:16:02.902Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/f5/ac409ecd1007528d15c3e8c3a57d34f334c70d76cfb7128a28cffdebd4c1/aiohttp-3.13.4-cp311-cp311-win_amd64.whl", hash = "sha256:c033f2bc964156030772d31cbf7e5defea181238ce1f87b9455b786de7d30145", size = 463824, upload-time = "2026-03-28T17:16:05.058Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/bd/ede278648914cabbabfdf95e436679b5d4156e417896a9b9f4587169e376/aiohttp-3.13.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ee62d4471ce86b108b19c3364db4b91180d13fe3510144872d6bad5401957360", size = 752158, upload-time = "2026-03-28T17:16:06.901Z" },
+    { url = "https://files.pythonhosted.org/packages/90/de/581c053253c07b480b03785196ca5335e3c606a37dc73e95f6527f1591fe/aiohttp-3.13.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c0fd8f41b54b58636402eb493afd512c23580456f022c1ba2db0f810c959ed0d", size = 501037, upload-time = "2026-03-28T17:16:08.82Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/f9/a5ede193c08f13cc42c0a5b50d1e246ecee9115e4cf6e900d8dbd8fd6acb/aiohttp-3.13.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4baa48ce49efd82d6b1a0be12d6a36b35e5594d1dd42f8bfba96ea9f8678b88c", size = 501556, upload-time = "2026-03-28T17:16:10.63Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/10/88ff67cd48a6ec36335b63a640abe86135791544863e0cfe1f065d6cef7a/aiohttp-3.13.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d738ebab9f71ee652d9dbd0211057690022201b11197f9a7324fd4dba128aa97", size = 1757314, upload-time = "2026-03-28T17:16:12.498Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/15/fdb90a5cf5a1f52845c276e76298c75fbbcc0ac2b4a86551906d54529965/aiohttp-3.13.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0ce692c3468fa831af7dceed52edf51ac348cebfc8d3feb935927b63bd3e8576", size = 1731819, upload-time = "2026-03-28T17:16:14.558Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/df/28146785a007f7820416be05d4f28cc207493efd1e8c6c1068e9bdc29198/aiohttp-3.13.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8e08abcfe752a454d2cb89ff0c08f2d1ecd057ae3e8cc6d84638de853530ebab", size = 1793279, upload-time = "2026-03-28T17:16:16.594Z" },
+    { url = "https://files.pythonhosted.org/packages/10/47/689c743abf62ea7a77774d5722f220e2c912a77d65d368b884d9779ef41b/aiohttp-3.13.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5977f701b3fff36367a11087f30ea73c212e686d41cd363c50c022d48b011d8d", size = 1891082, upload-time = "2026-03-28T17:16:18.71Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/b6/f7f4f318c7e58c23b761c9b13b9a3c9b394e0f9d5d76fbc6622fa98509f6/aiohttp-3.13.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54203e10405c06f8b6020bd1e076ae0fe6c194adcee12a5a78af3ffa3c57025e", size = 1773938, upload-time = "2026-03-28T17:16:21.125Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/06/f207cb3121852c989586a6fc16ff854c4fcc8651b86c5d3bd1fc83057650/aiohttp-3.13.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:358a6af0145bc4dda037f13167bef3cce54b132087acc4c295c739d05d16b1c3", size = 1579548, upload-time = "2026-03-28T17:16:23.588Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/58/e1289661a32161e24c1fe479711d783067210d266842523752869cc1d9c2/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:898ea1850656d7d61832ef06aa9846ab3ddb1621b74f46de78fbc5e1a586ba83", size = 1714669, upload-time = "2026-03-28T17:16:25.713Z" },
+    { url = "https://files.pythonhosted.org/packages/96/0a/3e86d039438a74a86e6a948a9119b22540bae037d6ba317a042ae3c22711/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7bc30cceb710cf6a44e9617e43eebb6e3e43ad855a34da7b4b6a73537d8a6763", size = 1754175, upload-time = "2026-03-28T17:16:28.18Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/30/e717fc5df83133ba467a560b6d8ef20197037b4bb5d7075b90037de1018e/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4a31c0c587a8a038f19a4c7e60654a6c899c9de9174593a13e7cc6e15ff271f9", size = 1762049, upload-time = "2026-03-28T17:16:30.941Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/28/8f7a2d4492e336e40005151bdd94baf344880a4707573378579f833a64c1/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2062f675f3fe6e06d6113eb74a157fb9df58953ffed0cdb4182554b116545758", size = 1570861, upload-time = "2026-03-28T17:16:32.953Z" },
+    { url = "https://files.pythonhosted.org/packages/78/45/12e1a3d0645968b1c38de4b23fdf270b8637735ea057d4f84482ff918ad9/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d1ba8afb847ff80626d5e408c1fdc99f942acc877d0702fe137015903a220a9", size = 1790003, upload-time = "2026-03-28T17:16:35.468Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/0f/60374e18d590de16dcb39d6ff62f39c096c1b958e6f37727b5870026ea30/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b08149419994cdd4d5eecf7fd4bc5986b5a9380285bcd01ab4c0d6bfca47b79d", size = 1737289, upload-time = "2026-03-28T17:16:38.187Z" },
+    { url = "https://files.pythonhosted.org/packages/02/bf/535e58d886cfbc40a8b0013c974afad24ef7632d645bca0b678b70033a60/aiohttp-3.13.4-cp312-cp312-win32.whl", hash = "sha256:fc432f6a2c4f720180959bc19aa37259651c1a4ed8af8afc84dd41c60f15f791", size = 434185, upload-time = "2026-03-28T17:16:40.735Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/1a/d92e3325134ebfff6f4069f270d3aac770d63320bd1fcd0eca023e74d9a8/aiohttp-3.13.4-cp312-cp312-win_amd64.whl", hash = "sha256:6148c9ae97a3e8bff9a1fc9c757fa164116f86c100468339730e717590a3fb77", size = 461285, upload-time = "2026-03-28T17:16:42.713Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/ac/892f4162df9b115b4758d615f32ec63d00f3084c705ff5526630887b9b42/aiohttp-3.13.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:63dd5e5b1e43b8fb1e91b79b7ceba1feba588b317d1edff385084fcc7a0a4538", size = 745744, upload-time = "2026-03-28T17:16:44.67Z" },
+    { url = "https://files.pythonhosted.org/packages/97/a9/c5b87e4443a2f0ea88cb3000c93a8fdad1ee63bffc9ded8d8c8e0d66efc6/aiohttp-3.13.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:746ac3cc00b5baea424dacddea3ec2c2702f9590de27d837aa67004db1eebc6e", size = 498178, upload-time = "2026-03-28T17:16:46.766Z" },
+    { url = "https://files.pythonhosted.org/packages/94/42/07e1b543a61250783650df13da8ddcdc0d0a5538b2bd15cef6e042aefc61/aiohttp-3.13.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bda8f16ea99d6a6705e5946732e48487a448be874e54a4f73d514660ff7c05d3", size = 498331, upload-time = "2026-03-28T17:16:48.9Z" },
+    { url = "https://files.pythonhosted.org/packages/20/d6/492f46bf0328534124772d0cf58570acae5b286ea25006900650f69dae0e/aiohttp-3.13.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b061e7b5f840391e3f64d0ddf672973e45c4cfff7a0feea425ea24e51530fc2", size = 1744414, upload-time = "2026-03-28T17:16:50.968Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/4d/e02627b2683f68051246215d2d62b2d2f249ff7a285e7a858dc47d6b6a14/aiohttp-3.13.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b252e8d5cd66184b570d0d010de742736e8a4fab22c58299772b0c5a466d4b21", size = 1719226, upload-time = "2026-03-28T17:16:53.173Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/6c/5d0a3394dd2b9f9aeba6e1b6065d0439e4b75d41f1fb09a3ec010b43552b/aiohttp-3.13.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:20af8aad61d1803ff11152a26146d8d81c266aa8c5aa9b4504432abb965c36a0", size = 1782110, upload-time = "2026-03-28T17:16:55.362Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/2d/c20791e3437700a7441a7edfb59731150322424f5aadf635602d1d326101/aiohttp-3.13.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:13a5cc924b59859ad2adb1478e31f410a7ed46e92a2a619d6d1dd1a63c1a855e", size = 1884809, upload-time = "2026-03-28T17:16:57.734Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/94/d99dbfbd1924a87ef643833932eb2a3d9e5eee87656efea7d78058539eff/aiohttp-3.13.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:534913dfb0a644d537aebb4123e7d466d94e3be5549205e6a31f72368980a81a", size = 1764938, upload-time = "2026-03-28T17:17:00.221Z" },
+    { url = "https://files.pythonhosted.org/packages/49/61/3ce326a1538781deb89f6cf5e094e2029cd308ed1e21b2ba2278b08426f6/aiohttp-3.13.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:320e40192a2dcc1cf4b5576936e9652981ab596bf81eb309535db7e2f5b5672f", size = 1570697, upload-time = "2026-03-28T17:17:02.985Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/77/4ab5a546857bb3028fbaf34d6eea180267bdab022ee8b1168b1fcde4bfdd/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9e587fcfce2bcf06526a43cb705bdee21ac089096f2e271d75de9c339db3100c", size = 1702258, upload-time = "2026-03-28T17:17:05.28Z" },
+    { url = "https://files.pythonhosted.org/packages/79/63/d8f29021e39bc5af8e5d5e9da1b07976fb9846487a784e11e4f4eeda4666/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:9eb9c2eea7278206b5c6c1441fdd9dc420c278ead3f3b2cc87f9b693698cc500", size = 1740287, upload-time = "2026-03-28T17:17:07.712Z" },
+    { url = "https://files.pythonhosted.org/packages/55/3a/cbc6b3b124859a11bc8055d3682c26999b393531ef926754a3445b99dfef/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:29be00c51972b04bf9d5c8f2d7f7314f48f96070ca40a873a53056e652e805f7", size = 1753011, upload-time = "2026-03-28T17:17:10.053Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/30/836278675205d58c1368b21520eab9572457cf19afd23759216c04483048/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:90c06228a6c3a7c9f776fe4fc0b7ff647fffd3bed93779a6913c804ae00c1073", size = 1566359, upload-time = "2026-03-28T17:17:12.433Z" },
+    { url = "https://files.pythonhosted.org/packages/50/b4/8032cc9b82d17e4277704ba30509eaccb39329dc18d6a35f05e424439e32/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a533ec132f05fd9a1d959e7f34184cd7d5e8511584848dab85faefbaac573069", size = 1785537, upload-time = "2026-03-28T17:17:14.721Z" },
+    { url = "https://files.pythonhosted.org/packages/17/7d/5873e98230bde59f493bf1f7c3e327486a4b5653fa401144704df5d00211/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1c946f10f413836f82ea4cfb90200d2a59578c549f00857e03111cf45ad01ca5", size = 1740752, upload-time = "2026-03-28T17:17:17.387Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/f2/13e46e0df051494d7d3c68b7f72d071f48c384c12716fc294f75d5b1a064/aiohttp-3.13.4-cp313-cp313-win32.whl", hash = "sha256:48708e2706106da6967eff5908c78ca3943f005ed6bcb75da2a7e4da94ef8c70", size = 433187, upload-time = "2026-03-28T17:17:19.523Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/c0/649856ee655a843c8f8664592cfccb73ac80ede6a8c8db33a25d810c12db/aiohttp-3.13.4-cp313-cp313-win_amd64.whl", hash = "sha256:74a2eb058da44fa3a877a49e2095b591d4913308bb424c418b77beb160c55ce3", size = 459778, upload-time = "2026-03-28T17:17:21.964Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/29/6657cc37ae04cacc2dbf53fb730a06b6091cc4cbe745028e047c53e6d840/aiohttp-3.13.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:e0a2c961fc92abeff61d6444f2ce6ad35bb982db9fc8ff8a47455beacf454a57", size = 749363, upload-time = "2026-03-28T17:17:24.044Z" },
+    { url = "https://files.pythonhosted.org/packages/90/7f/30ccdf67ca3d24b610067dc63d64dcb91e5d88e27667811640644aa4a85d/aiohttp-3.13.4-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:153274535985a0ff2bff1fb6c104ed547cec898a09213d21b0f791a44b14d933", size = 499317, upload-time = "2026-03-28T17:17:26.199Z" },
+    { url = "https://files.pythonhosted.org/packages/93/13/e372dd4e68ad04ee25dafb050c7f98b0d91ea643f7352757e87231102555/aiohttp-3.13.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:351f3171e2458da3d731ce83f9e6b9619e325c45cbd534c7759750cabf453ad7", size = 500477, upload-time = "2026-03-28T17:17:28.279Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/fe/ee6298e8e586096fb6f5eddd31393d8544f33ae0792c71ecbb4c2bef98ac/aiohttp-3.13.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f989ac8bc5595ff761a5ccd32bdb0768a117f36dd1504b1c2c074ed5d3f4df9c", size = 1737227, upload-time = "2026-03-28T17:17:30.587Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/b9/a7a0463a09e1a3fe35100f74324f23644bfc3383ac5fd5effe0722a5f0b7/aiohttp-3.13.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d36fc1709110ec1e87a229b201dd3ddc32aa01e98e7868083a794609b081c349", size = 1694036, upload-time = "2026-03-28T17:17:33.29Z" },
+    { url = "https://files.pythonhosted.org/packages/57/7c/8972ae3fb7be00a91aee6b644b2a6a909aedb2c425269a3bfd90115e6f8f/aiohttp-3.13.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:42adaeea83cbdf069ab94f5103ce0787c21fb1a0153270da76b59d5578302329", size = 1786814, upload-time = "2026-03-28T17:17:36.035Z" },
+    { url = "https://files.pythonhosted.org/packages/93/01/c81e97e85c774decbaf0d577de7d848934e8166a3a14ad9f8aa5be329d28/aiohttp-3.13.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:92deb95469928cc41fd4b42a95d8012fa6df93f6b1c0a83af0ffbc4a5e218cde", size = 1866676, upload-time = "2026-03-28T17:17:38.441Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/5f/5b46fe8694a639ddea2cd035bf5729e4677ea882cb251396637e2ef1590d/aiohttp-3.13.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c0c7c07c4257ef3a1df355f840bc62d133bcdef5c1c5ba75add3c08553e2eed", size = 1740842, upload-time = "2026-03-28T17:17:40.783Z" },
+    { url = "https://files.pythonhosted.org/packages/20/a2/0d4b03d011cca6b6b0acba8433193c1e484efa8d705ea58295590fe24203/aiohttp-3.13.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f062c45de8a1098cb137a1898819796a2491aec4e637a06b03f149315dff4d8f", size = 1566508, upload-time = "2026-03-28T17:17:43.235Z" },
+    { url = "https://files.pythonhosted.org/packages/98/17/e689fd500da52488ec5f889effd6404dece6a59de301e380f3c64f167beb/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:76093107c531517001114f0ebdb4f46858ce818590363e3e99a4a2280334454a", size = 1700569, upload-time = "2026-03-28T17:17:46.165Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/0d/66402894dbcf470ef7db99449e436105ea862c24f7ea4c95c683e635af35/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:6f6ec32162d293b82f8b63a16edc80769662fbd5ae6fbd4936d3206a2c2cc63b", size = 1707407, upload-time = "2026-03-28T17:17:48.825Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/eb/af0ab1a3650092cbd8e14ef29e4ab0209e1460e1c299996c3f8288b3f1ff/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5903e2db3d202a00ad9f0ec35a122c005e85d90c9836ab4cda628f01edf425e2", size = 1752214, upload-time = "2026-03-28T17:17:51.206Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/bf/72326f8a98e4c666f292f03c385545963cc65e358835d2a7375037a97b57/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2d5bea57be7aca98dbbac8da046d99b5557c5cf4e28538c4c786313078aca09e", size = 1562162, upload-time = "2026-03-28T17:17:53.634Z" },
+    { url = "https://files.pythonhosted.org/packages/67/9f/13b72435f99151dd9a5469c96b3b5f86aa29b7e785ca7f35cf5e538f74c0/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:bcf0c9902085976edc0232b75006ef38f89686901249ce14226b6877f88464fb", size = 1768904, upload-time = "2026-03-28T17:17:55.991Z" },
+    { url = "https://files.pythonhosted.org/packages/18/bc/28d4970e7d5452ac7776cdb5431a1164a0d9cf8bd2fffd67b4fb463aa56d/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c3295f98bfeed2e867cab588f2a146a9db37a85e3ae9062abf46ba062bd29165", size = 1723378, upload-time = "2026-03-28T17:17:58.348Z" },
+    { url = "https://files.pythonhosted.org/packages/53/74/b32458ca1a7f34d65bdee7aef2036adbe0438123d3d53e2b083c453c24dd/aiohttp-3.13.4-cp314-cp314-win32.whl", hash = "sha256:a598a5c5767e1369d8f5b08695cab1d8160040f796c4416af76fd773d229b3c9", size = 438711, upload-time = "2026-03-28T17:18:00.728Z" },
+    { url = "https://files.pythonhosted.org/packages/40/b2/54b487316c2df3e03a8f3435e9636f8a81a42a69d942164830d193beb56a/aiohttp-3.13.4-cp314-cp314-win_amd64.whl", hash = "sha256:c555db4bc7a264bead5a7d63d92d41a1122fcd39cc62a4db815f45ad46f9c2c8", size = 464977, upload-time = "2026-03-28T17:18:03.367Z" },
+    { url = "https://files.pythonhosted.org/packages/47/fb/e41b63c6ce71b07a59243bb8f3b457ee0c3402a619acb9d2c0d21ef0e647/aiohttp-3.13.4-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:45abbbf09a129825d13c18c7d3182fecd46d9da3cfc383756145394013604ac1", size = 781549, upload-time = "2026-03-28T17:18:05.779Z" },
+    { url = "https://files.pythonhosted.org/packages/97/53/532b8d28df1e17e44c4d9a9368b78dcb6bf0b51037522136eced13afa9e8/aiohttp-3.13.4-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:74c80b2bc2c2adb7b3d1941b2b60701ee2af8296fc8aad8b8bc48bc25767266c", size = 514383, upload-time = "2026-03-28T17:18:08.096Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/1f/62e5d400603e8468cd635812d99cb81cfdc08127a3dc474c647615f31339/aiohttp-3.13.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c97989ae40a9746650fa196894f317dafc12227c808c774929dda0ff873a5954", size = 518304, upload-time = "2026-03-28T17:18:10.642Z" },
+    { url = "https://files.pythonhosted.org/packages/90/57/2326b37b10896447e3c6e0cbef4fe2486d30913639a5cfd1332b5d870f82/aiohttp-3.13.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dae86be9811493f9990ef44fff1685f5c1a3192e9061a71a109d527944eed551", size = 1893433, upload-time = "2026-03-28T17:18:13.121Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/b4/a24d82112c304afdb650167ef2fe190957d81cbddac7460bedd245f765aa/aiohttp-3.13.4-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1db491abe852ca2fa6cc48a3341985b0174b3741838e1341b82ac82c8bd9e871", size = 1755901, upload-time = "2026-03-28T17:18:16.21Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/2d/0883ef9d878d7846287f036c162a951968f22aabeef3ac97b0bea6f76d5d/aiohttp-3.13.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e5d701c0aad02a7dce72eef6b93226cf3734330f1a31d69ebbf69f33b86666e", size = 1876093, upload-time = "2026-03-28T17:18:18.703Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/52/9204bb59c014869b71971addad6778f005daa72a96eed652c496789d7468/aiohttp-3.13.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8ac32a189081ae0a10ba18993f10f338ec94341f0d5df8fff348043962f3c6f8", size = 1970815, upload-time = "2026-03-28T17:18:21.858Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/b5/e4eb20275a866dde0f570f411b36c6b48f7b53edfe4f4071aa1b0728098a/aiohttp-3.13.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98e968cdaba43e45c73c3f306fca418c8009a957733bac85937c9f9cf3f4de27", size = 1816223, upload-time = "2026-03-28T17:18:24.729Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/23/e98075c5bb146aa61a1239ee1ac7714c85e814838d6cebbe37d3fe19214a/aiohttp-3.13.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ca114790c9144c335d538852612d3e43ea0f075288f4849cf4b05d6cd2238ce7", size = 1649145, upload-time = "2026-03-28T17:18:27.269Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/c1/7bad8be33bb06c2bb224b6468874346026092762cbec388c3bdb65a368ee/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ea2e071661ba9cfe11eabbc81ac5376eaeb3061f6e72ec4cc86d7cdd1ffbdbbb", size = 1816562, upload-time = "2026-03-28T17:18:29.847Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/10/c00323348695e9a5e316825969c88463dcc24c7e9d443244b8a2c9cf2eae/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:34e89912b6c20e0fd80e07fa401fd218a410aa1ce9f1c2f1dad6db1bd0ce0927", size = 1800333, upload-time = "2026-03-28T17:18:32.269Z" },
+    { url = "https://files.pythonhosted.org/packages/84/43/9b2147a1df3559f49bd723e22905b46a46c068a53adb54abdca32c4de180/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0e217cf9f6a42908c52b46e42c568bd57adc39c9286ced31aaace614b6087965", size = 1820617, upload-time = "2026-03-28T17:18:35.238Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/7f/b3481a81e7a586d02e99387b18c6dafff41285f6efd3daa2124c01f87eae/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:0c296f1221e21ba979f5ac1964c3b78cfde15c5c5f855ffd2caab337e9cd9182", size = 1643417, upload-time = "2026-03-28T17:18:37.949Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/72/07181226bc99ce1124e0f89280f5221a82d3ae6a6d9d1973ce429d48e52b/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:d99a9d168ebaffb74f36d011750e490085ac418f4db926cce3989c8fe6cb6b1b", size = 1849286, upload-time = "2026-03-28T17:18:40.534Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/e6/1b3566e103eca6da5be4ae6713e112a053725c584e96574caf117568ffef/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cb19177205d93b881f3f89e6081593676043a6828f59c78c17a0fd6c1fbed2ba", size = 1782635, upload-time = "2026-03-28T17:18:43.073Z" },
+    { url = "https://files.pythonhosted.org/packages/37/58/1b11c71904b8d079eb0c39fe664180dd1e14bebe5608e235d8bfbadc8929/aiohttp-3.13.4-cp314-cp314t-win32.whl", hash = "sha256:c606aa5656dab6552e52ca368e43869c916338346bfaf6304e15c58fb113ea30", size = 472537, upload-time = "2026-03-28T17:18:46.286Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/8f/87c56a1a1977d7dddea5b31e12189665a140fdb48a71e9038ff90bb564ec/aiohttp-3.13.4-cp314-cp314t-win_amd64.whl", hash = "sha256:014dcc10ec8ab8db681f0d68e939d1e9286a5aa2b993cbbdb0db130853e02144", size = 506381, upload-time = "2026-03-28T17:18:48.74Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/f9/17e8a70abe874ec694395119338fde2f13ee1903bd14f3fd5b310b77a1ea/aiohttp-3.13.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b3f00bb9403728b08eb3951e982ca0a409c7a871d709684623daeab79465b181", size = 755716, upload-time = "2026-03-28T17:18:51.918Z" },
+    { url = "https://files.pythonhosted.org/packages/27/b3/fdb36e59b9fb37297b1651248d3d84e61faa49af2faabc1e243d3f75585f/aiohttp-3.13.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cb15595eb52870f84248d7cc97013a76f52ab02ff74d394be093b1d9b8b82bc0", size = 506500, upload-time = "2026-03-28T17:18:54.755Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/fb/dacf759c43cfb5fa32568bd369f054eeb23906ab23f4e3663e01e04c7988/aiohttp-3.13.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:907ad36b6a65cff7d88d7aca0f77c650546ba850a4f92c92ecb83590d4613249", size = 499881, upload-time = "2026-03-28T17:18:57.302Z" },
+    { url = "https://files.pythonhosted.org/packages/52/cd/7824ee57dde8ca7f62e7fbc247ebe1aa3b5495d3598f0c516f06de1ef7ab/aiohttp-3.13.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5539ec0d6a3a5c6799b661b7e79166ad1b7ae71ccb59a92fcb6b4ef89295bc94", size = 1681734, upload-time = "2026-03-28T17:19:00.057Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/40/6f4ca61736a16deed2d2762a8dbeaaa48ad292974489be2a2f32f62a4e0b/aiohttp-3.13.4-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3b4e07d8803a70dd886b5f38588e5b49f894995ca8e132b06c31a2583ae2ef6e", size = 1653787, upload-time = "2026-03-28T17:19:03.026Z" },
+    { url = "https://files.pythonhosted.org/packages/89/80/3793f0a1148a42190f6824ce9a0af79910cd3df8dfc58fa784234a7d9e41/aiohttp-3.13.4-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ce7320a945aac4bf0bb8901600e4f9409eb602f25ce3ef4d275b48f6d704a862", size = 1737964, upload-time = "2026-03-28T17:19:05.77Z" },
+    { url = "https://files.pythonhosted.org/packages/15/fd/e41981d0f9e0dccfb8f2580d4e64e6c59d293b9b0815849950cc499fe53a/aiohttp-3.13.4-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:26ed03f7d3d6453634729e2c7600d7255d65e879559c5a48fe1bb78355cde74b", size = 1832226, upload-time = "2026-03-28T17:19:08.809Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/69/e6b566c638b37bfa14b98c2c429fcdba3b097a990acc9845fcc779ce39cc/aiohttp-3.13.4-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c3f733916e85506b8000dddc071c6b82f8c68f56c99adb328d6550017db062d", size = 1681476, upload-time = "2026-03-28T17:19:11.502Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/8c/f1b7f03e745fa6281dd949673297c7ac54d7cc54d2e58beb5135ac5c6204/aiohttp-3.13.4-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3d525648fe7c8b4977e460c18098f9f81d7991d72edfdc2f13cf96068f279bc", size = 1573061, upload-time = "2026-03-28T17:19:14.437Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/56/e7e972f1bed922297d72cc1d27bae6b2e28fdc2d6a895320e396a93c0f8a/aiohttp-3.13.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4e2e68085730a03704beb2cff035fa8648f62c9f93758d7e6d70add7f7bb5b3b", size = 1653248, upload-time = "2026-03-28T17:19:17.432Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/98/3d63d2f2e06808911e103d6d47c400548cf26a23dd3275de594339ff8e96/aiohttp-3.13.4-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:797613182ffaaca0b9ad5f3b3d3ce5d21242c768f75e66c750b8292bd97c9de3", size = 1666599, upload-time = "2026-03-28T17:19:20.17Z" },
+    { url = "https://files.pythonhosted.org/packages/da/c8/31e487fb16d37c89cc6ee190a424b218471750ac48a227e042e200a17687/aiohttp-3.13.4-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:2d15e7e4f1099d9e4d863eaf77a8eee5dcb002b7d7188061b0fbee37f845899e", size = 1709919, upload-time = "2026-03-28T17:19:22.872Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/86/3b742bd9204b7deb4f61e6723b1f42a8211ccc60dfddb3e52a6cd4329d46/aiohttp-3.13.4-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:19f60011ad60e40a01d242238bb335399e3a4d8df958c63cbb835add8d5c3b5a", size = 1560523, upload-time = "2026-03-28T17:19:25.879Z" },
+    { url = "https://files.pythonhosted.org/packages/72/63/6b80cef343a0527690588808d02aad7604cc4e23eaab207179e77dd607be/aiohttp-3.13.4-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:c344c47e85678e410b064fc2ace14db86bb69db7ed5520c234bf13aed603ec30", size = 1731336, upload-time = "2026-03-28T17:19:29.02Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/3c/9b39bc9609cac87e19b3394b7ed4bbab3787b434b14e012b9e16be64e9d5/aiohttp-3.13.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:d904084985ca66459e93797e5e05985c048a9c0633655331144c089943e53d12", size = 1667646, upload-time = "2026-03-28T17:19:31.797Z" },
+    { url = "https://files.pythonhosted.org/packages/21/72/3fb0ea857c891de89f6914f737f7423b7fa4dd1f46d8ce621eb07595ff4c/aiohttp-3.13.4-cp39-cp39-win32.whl", hash = "sha256:1746338dc2a33cf706cd7446575d13d451f28f9860bebc908c7632b22e71ae3f", size = 441019, upload-time = "2026-03-28T17:19:34.79Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/61/8a7191782a31ae3c7f7cee2cd2e37b3ee5849666767db116d449cfe20b88/aiohttp-3.13.4-cp39-cp39-win_amd64.whl", hash = "sha256:a5444dce2e6fba0a1dc2d58d026e674f25f21de178c6f844342629bcef019f2f", size = 464025, upload-time = "2026-03-28T17:19:37.362Z" },
 ]
 
 [[package]]
@@ -117,21 +170,21 @@ name = "aiosignal"
 version = "1.4.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "frozenlist" },
-    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+    { name = "frozenlist", marker = "python_full_version >= '3.10'" },
+    { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007 }
+sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490 },
+    { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" },
 ]
 
 [[package]]
 name = "annotated-types"
 version = "0.7.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081 }
+sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643 },
+    { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" },
 ]
 
 [[package]]
@@ -141,59 +194,72 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "typing-extensions", marker = "python_full_version < '3.12'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/2a/01/f06342d2eb822153f63d188153e41fbeabb29b48247f7a11ce76c538f7d1/arro3_core-0.6.5.tar.gz", hash = "sha256:768078887cd7ac82de4736f94bbd91f6d660f10779848bd5b019f511badd9d75", size = 107522 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/7a/8a/24b35cf01a68621f5f07e3191ca96f70a145022ca367347266901eb504a7/arro3_core-0.6.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:da193dc2fb8c2005d0b3887b09d1a90d42cec1f59f17a8a1a5791f0de90946ae", size = 2678116 },
-    { url = "https://files.pythonhosted.org/packages/5a/7a/4398bb0582fb22d575f256f2b9ac7be735c765222cc61fb214d606bdb77c/arro3_core-0.6.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed1a760ec39fe19c65e98f45515582408002d0212df5db227a5959ffeb07ad4a", size = 2383214 },
-    { url = "https://files.pythonhosted.org/packages/82/3f/a321501c5da4bf3ff7438c3e5eb6e63bcecb5630c0f4a89a017cbfa8e4a0/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6584a3d28007740afcef1e301332876e2b785bd8edd59a458a6bc9b051bce052", size = 2883536 },
-    { url = "https://files.pythonhosted.org/packages/0d/50/1d1e55b9a8c4cf2fdeb954947aa135010554a3333b709e8cad3d5d084be2/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8e0af4789618f02bead4a0cd4d0a54abd9c8aa4fcedf9872b4891d2e3e984161", size = 2908828 },
-    { url = "https://files.pythonhosted.org/packages/12/75/b4b1de1ccb17890bada9a3f4131cf3137f145d5d10490db51de6b8799926/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c73f212e549e9b6d11cfe3f14bbf3fba9d0891426afb5916688d16d0df724085", size = 3145458 },
-    { url = "https://files.pythonhosted.org/packages/08/4f/f42ce1840490fd0863bfbc56f28eaaec3bcb4eb322079af9c070111657e5/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f88f62e4e276a9e84f250722d2e5ffc078af9a3f67ac691f572a0e05dd6095", size = 2775793 },
-    { url = "https://files.pythonhosted.org/packages/2b/aa/9637efc8d8733c34bedef44e5b2c170dea14d15ab56b3566d8d7963c2616/arro3_core-0.6.5-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:b2635e4c227f25ff8784dc8efb38cb7c1674646cfdc68ded53f2426289885f0e", size = 2516697 },
-    { url = "https://files.pythonhosted.org/packages/60/84/1fcfadf956bc25eb5251b1ea7a7099f05198a55764635d2fc9ceafdbdbd1/arro3_core-0.6.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a5f3e936686bcd8542fafc94c68fdb23ec42d1d51a4777967ae815c90aff7296", size = 3023625 },
-    { url = "https://files.pythonhosted.org/packages/58/d0/52d0cb3c0dfa8e94ba2118b7e91a70da76d6ede9de4e70374f831f38cfdf/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:705c32fec03dadc08f807d69ce557882005d43eb20ec62699f7036340f0d580f", size = 2701346 },
-    { url = "https://files.pythonhosted.org/packages/69/bf/42a6f6501805c31cb65d8a6e3379eeec4fa6c26dc07c9ce894f363ccad1c/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:56d8166235a4c54e4f7ba082ec76890c820fa8c1b6c995ec59cead62a9698e59", size = 3153207 },
-    { url = "https://files.pythonhosted.org/packages/4f/e5/41fdee468b33759b42958347c2d70b0461bf8f70ba1762a94cdf2e9b0142/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:1ba43ba9081c00767083195222b6be74913de668296f55599658c4b0bb7cd327", size = 3105033 },
-    { url = "https://files.pythonhosted.org/packages/03/e0/b6d733b4540c05bac546162e045b547031f4d88c67b7c864929d9bce29ad/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4f5df13c6742e3f0b494cfe9025dccdc8426a74cc9e3e5a1239311e07a4b24e0", size = 2954793 },
-    { url = "https://files.pythonhosted.org/packages/c0/34/8353ba79c8d0498eaacc077d58b384ef785e0b69c9cbff7c2580136b8fe3/arro3_core-0.6.5-cp310-cp310-win_amd64.whl", hash = "sha256:34676b728178236df63c9ea10b21432392d4b5bb51e2030e77c68eed4dede2ad", size = 2837495 },
-    { url = "https://files.pythonhosted.org/packages/78/85/20e46d3ed59d2f93be4a4d1abea4f6bef3e96acd59bf5a50726f84303c51/arro3_core-0.6.5-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9d5999506daec1ab31096b3deb1e3573041d6ecadb4ca99c96f7ab26720c592c", size = 2685615 },
-    { url = "https://files.pythonhosted.org/packages/d0/9c/427d578f7d2bf3149515a8b75217e7189e7b1d74e5c5609e1a7e7f0f8d3c/arro3_core-0.6.5-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:bd3e251184c2dd6ade81c5613256b6d85ab3ddbd5af838b1de657e0ddec017f8", size = 2391944 },
-    { url = "https://files.pythonhosted.org/packages/90/24/7e4af478eb889bfa401e1c1b8868048ca692e6205affbf81cf3666347852/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7cadb29349960d3821b0515d9df80f2725cea155ad966c699f6084de32e313cb", size = 2888376 },
-    { url = "https://files.pythonhosted.org/packages/70/3b/01006a96bc980275aa4d2eb759c5f10afb7c85fcdce3c36ddb18635ad23b/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a922e560ed2ccee3293d51b39e013b51cc233895d25ddafcacfb83c540a19e6f", size = 2916568 },
-    { url = "https://files.pythonhosted.org/packages/a2/2f/4e04c7f5687de6fb6f88aa7590b16bcf507ba17ddbd268525f27b70b7a68/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68fe6672bf51f039b12046a209cba0a9405e10ae44e5a0d557f091b356a62051", size = 3144223 },
-    { url = "https://files.pythonhosted.org/packages/31/4a/72dc383d1a0d14f1d453e334e3461e229762edb1bf3f75b3ab977e9386ed/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c3ee95603e375401a58ff763ce2c8aa858e0c4f757c1fb719f48fb070f540b2", size = 2781862 },
-    { url = "https://files.pythonhosted.org/packages/14/dc/0df7684b683114eaf8e57989b4230edb359cbfb6e98b8770d69128b27572/arro3_core-0.6.5-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:fbaf6b65213630007b798b565e0701c2092a330deeba16bd3d896d401f7e9f28", size = 2522442 },
-    { url = "https://files.pythonhosted.org/packages/c9/04/75f8627cd7fe4d103eca51760d50269cfbc0bf6beaf83a3cdefb4ebd37c7/arro3_core-0.6.5-cp311-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:20679f874558bb2113e96325522625ec64a72687000b7a9578031a4d082c6ef5", size = 3033454 },
-    { url = "https://files.pythonhosted.org/packages/ea/19/f2d54985da65bf6d3da76218bee56383285035541c8d0cadb53095845b3e/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d82d6ec32d5c7c73057fb9c528390289fd5bc94b8d8f28fca9c56fc8e41c412c", size = 2705984 },
-    { url = "https://files.pythonhosted.org/packages/6c/53/b1d7742d6db7b4aa44d3785956955d651b3ac36db321625fd15466be1aca/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4cba4db0a4203a3ccf131c3fb7804d77f0740d6165ec9efa3aa3acbca87c43a3", size = 3157472 },
-    { url = "https://files.pythonhosted.org/packages/05/31/68711327dbdd480aed54158fc1c46ab245e860ab0286e0916ce788f9889e/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:e358affc4a0fe5c1b5dccf4f92c43a836aaa4c4eab0906c83b00b60275de3b6d", size = 3117099 },
-    { url = "https://files.pythonhosted.org/packages/31/e3/15ffca0797d9500b23759ae4477cf052fde8dd47a3890f4e4e1d04639016/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:324e43f07b7681846d00a8995b78bdc4b4a719047aa0d34426b462b8f208ee98", size = 2963677 },
-    { url = "https://files.pythonhosted.org/packages/bc/02/69e60dbe3bbe2bfc8b6dfa4f4bfcb8d1dd240a137bf2a5f7bcc84703f05c/arro3_core-0.6.5-cp311-abi3-win_amd64.whl", hash = "sha256:285f802c8a42fe29ecb84584d1700bc4c4f974552b75f805e1f4362d28b97080", size = 2850445 },
-    { url = "https://files.pythonhosted.org/packages/b1/29/2e5b091f6b5cffb6489dbe7ed353841568dde8ac4d1232c77321da1d0925/arro3_core-0.6.5-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:8c20e69c3b3411fd6ed56091f388e699072651e880e682be5bd14f3a392ed3e8", size = 2671985 },
-    { url = "https://files.pythonhosted.org/packages/30/74/764ac4b58fef3fdfc655416c42349206156db5c687fa24a0674acaeaadbb/arro3_core-0.6.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:92211f1d03221ff74d0b535a576b39601083d8e98e9d47228314573f9d4f9ae2", size = 2382931 },
-    { url = "https://files.pythonhosted.org/packages/6a/07/bd8c92e218240ae8a30150a5d7a2dab359b452ab54a8bb7b90effe806e3d/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:280d933b75f2649779d76e32a07f91d2352a952f2c97ddf7b320e267f440cd42", size = 2879900 },
-    { url = "https://files.pythonhosted.org/packages/0f/d4/253725019fe2ae5f5fde87928118ffa568cc59f07b2d6a0e90620938c537/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfc3f6b93b924f43fb7985b06202343c30b43da6bd5055ba8b84eda431e494d4", size = 2904149 },
-    { url = "https://files.pythonhosted.org/packages/f0/b0/7a3dea641ac8de041c1a34859a2f2a82d3cdf3c3360872101c1d198a1e24/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5963635eb698ebc7da689e641f68b3998864bab894cf0ca84bd058b8c60d97f", size = 3143477 },
-    { url = "https://files.pythonhosted.org/packages/a7/05/1a50575be33fe9240898a1b5a8574658a905b5675865285585e070dcf7e2/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac291b3e74b57e56e03373d57530540cbbbfd92e4219fe2778ea531006673fe9", size = 2776522 },
-    { url = "https://files.pythonhosted.org/packages/2e/bd/e7b03207e7906e94e327cd4190fdb2d26ae52bc4ee1edeb057fed760796b/arro3_core-0.6.5-cp313-cp313t-manylinux_2_24_aarch64.whl", hash = "sha256:5d3f4cc58a654037d61f61ba230419da2c8f88a0ac82b9d41fe307f7cf9fda97", size = 2515426 },
-    { url = "https://files.pythonhosted.org/packages/f9/ed/82d1febd5c104eccdfb82434e3619125c328c36da143e19dfa3c86de4a81/arro3_core-0.6.5-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:93cddac90238d64451f5e66c630ded89d0b5fd6d2c099bf3a5151dde2c1ddf1d", size = 3024759 },
-    { url = "https://files.pythonhosted.org/packages/da/cd/00e06907e42e404c21eb08282dee94ac7a1961facfa9a96d116829031721/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:1fa7ac10db5846c33f4e8b66a6eaa705d84998e38575a835acac9a6a6649933d", size = 2700191 },
-    { url = "https://files.pythonhosted.org/packages/a3/11/a4bb9a900f456a6905d481bd2289f7a2371dcde024de56779621fd6a92c3/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:ca69f698a065cdbf845d59d412bc204e8f8af12f93737d82e6a18f3cff812349", size = 3149963 },
-    { url = "https://files.pythonhosted.org/packages/28/8a/79c76ad88b16f2fac25684f7313593738f353355eb1af2307e43efd7b1ca/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:de74a2512e2e2366d4b064c498c38672bf6ddea38acec8b1999b4e66182dd001", size = 3104663 },
-    { url = "https://files.pythonhosted.org/packages/20/66/9152feaa87f851a37c1a2bd74fb89d7e82e4c76447ee590bf8e6fff5e9d8/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:806ca8e20507675b2de68b3d009f76e898cc3c3e441c834ea5220866f68aac50", size = 2956440 },
-    { url = "https://files.pythonhosted.org/packages/ad/66/f4179ef64d5c18fe76ec93cfbff42c0f401438ef771c6766b880044d7e13/arro3_core-0.6.5-cp313-cp313t-win_amd64.whl", hash = "sha256:8f6f0cc78877ade7ad6e678a4671b191406547e7b407bc9637436869c017ed47", size = 2845345 },
-    { url = "https://files.pythonhosted.org/packages/10/ca/b2139dbb25f9fefb9b1cdce8a73785615de6763af6a16bf6ff96a3b630f2/arro3_core-0.6.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:26d5b50139f1a96727fa1760b4d70393acf5ee0fba45346ad2d4f69824d3bdc2", size = 2676788 },
-    { url = "https://files.pythonhosted.org/packages/34/a1/c68dde2944f493c8ccfcb91bf6da6d27a27c3674316dd09c9560f9e6ab1a/arro3_core-0.6.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b65b3d8d7f65f2f3c36002dc467380d7a31ea771132986dddc6341c5a9dc726f", size = 2382809 },
-    { url = "https://files.pythonhosted.org/packages/c6/fc/2fb81d42a3cecd632deace97dc23ac74083d60d158106440c783bae4ff01/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c3442a79a757ed3fbd7793de180019ae3201f04237537c2e2e3f1e3dd99b31c", size = 2882818 },
-    { url = "https://files.pythonhosted.org/packages/58/7f/16f741e1d49ba5c5a893ce6f8eb0283d64bc68d6cc9e07ac62f96eaadfae/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:def7b0065a684d6f903a658d2567da47e2fcecde716e0b34eff4d899c6468c8d", size = 2907503 },
-    { url = "https://files.pythonhosted.org/packages/eb/45/2eb7972e0bbec0ee0ab22b0f166ec1ea74b53bd76c93a18ced434713e495/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbfe2f2d4d0d393833cd6a4bd9c15266a02307a3028f159155a1c536469c3ae7", size = 3143706 },
-    { url = "https://files.pythonhosted.org/packages/2d/af/b78e28842faa675e4e6c4d82e861accf21ac08bbab80a65fa80c578f80a1/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a191a3e4f72c34f7ace7724a94f2d90b06c804a6cbece4ae0f18d36325479cf3", size = 2775462 },
-    { url = "https://files.pythonhosted.org/packages/45/df/950e57e4915e0457acadaaca13c4423d5e2652e403135eb7606d5e6e5443/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_24_aarch64.whl", hash = "sha256:e3f6ab4c6ea96c451eff72aa6c5b9835a0ea8a9847cfe3995c88cce0c7701fb5", size = 2516212 },
-    { url = "https://files.pythonhosted.org/packages/07/73/821640d0827a829ed2565c2d4812080ab7fb86f0d271b462f9b37e6d946e/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:27df5239835330299636a02977f2cb34d5c460cc03b2ae1d6ab6a03d28051b08", size = 3023342 },
-    { url = "https://files.pythonhosted.org/packages/fd/30/51302d2f4d1b627dd11e2be979f2c48550b782d8d58d0378316342e284a8/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:71dce89c0e91be4cfb42591f03809235bbc374c396e08acdf93c4d85b09e40f5", size = 2700740 },
-    { url = "https://files.pythonhosted.org/packages/1d/e8/0c8a345a013bb64abea60b4864bacc01e43b8699b8874794baec9c8a7e76/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:d380c28f85568ed99c1686fb9d64b5a811d76d569f367cbec8ef7e58f6e2fdf9", size = 3152749 },
-    { url = "https://files.pythonhosted.org/packages/6a/42/003b30c4da394366d5967a5b993f7471a74182c983d8f757891b3dd5d594/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:8e359c0c4fe9992f5a863a4a31502ea58eb2f92988fc2e501850540b3eff0328", size = 3104676 },
-    { url = "https://files.pythonhosted.org/packages/0b/fd/4f8dac58ea17e05978bf35cb9a3e485b1ff3cdd6e2cc29deb08f54080de4/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:9a58acbc61480b533aa84d735db04b1e68fc7f6807ab694d606c03b5e694d83d", size = 2954405 },
+sdist = { url = "https://files.pythonhosted.org/packages/2a/01/f06342d2eb822153f63d188153e41fbeabb29b48247f7a11ce76c538f7d1/arro3_core-0.6.5.tar.gz", hash = "sha256:768078887cd7ac82de4736f94bbd91f6d660f10779848bd5b019f511badd9d75", size = 107522, upload-time = "2025-10-13T23:12:38.872Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7a/8a/24b35cf01a68621f5f07e3191ca96f70a145022ca367347266901eb504a7/arro3_core-0.6.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:da193dc2fb8c2005d0b3887b09d1a90d42cec1f59f17a8a1a5791f0de90946ae", size = 2678116, upload-time = "2025-10-13T23:09:04.198Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/7a/4398bb0582fb22d575f256f2b9ac7be735c765222cc61fb214d606bdb77c/arro3_core-0.6.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed1a760ec39fe19c65e98f45515582408002d0212df5db227a5959ffeb07ad4a", size = 2383214, upload-time = "2025-10-13T23:09:06.841Z" },
+    { url = "https://files.pythonhosted.org/packages/82/3f/a321501c5da4bf3ff7438c3e5eb6e63bcecb5630c0f4a89a017cbfa8e4a0/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6584a3d28007740afcef1e301332876e2b785bd8edd59a458a6bc9b051bce052", size = 2883536, upload-time = "2025-10-13T23:09:08.877Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/50/1d1e55b9a8c4cf2fdeb954947aa135010554a3333b709e8cad3d5d084be2/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8e0af4789618f02bead4a0cd4d0a54abd9c8aa4fcedf9872b4891d2e3e984161", size = 2908828, upload-time = "2025-10-13T23:09:10.958Z" },
+    { url = "https://files.pythonhosted.org/packages/12/75/b4b1de1ccb17890bada9a3f4131cf3137f145d5d10490db51de6b8799926/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c73f212e549e9b6d11cfe3f14bbf3fba9d0891426afb5916688d16d0df724085", size = 3145458, upload-time = "2025-10-13T23:09:13.275Z" },
+    { url = "https://files.pythonhosted.org/packages/08/4f/f42ce1840490fd0863bfbc56f28eaaec3bcb4eb322079af9c070111657e5/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f88f62e4e276a9e84f250722d2e5ffc078af9a3f67ac691f572a0e05dd6095", size = 2775793, upload-time = "2025-10-13T23:09:15.342Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/aa/9637efc8d8733c34bedef44e5b2c170dea14d15ab56b3566d8d7963c2616/arro3_core-0.6.5-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:b2635e4c227f25ff8784dc8efb38cb7c1674646cfdc68ded53f2426289885f0e", size = 2516697, upload-time = "2025-10-13T23:09:17.584Z" },
+    { url = "https://files.pythonhosted.org/packages/60/84/1fcfadf956bc25eb5251b1ea7a7099f05198a55764635d2fc9ceafdbdbd1/arro3_core-0.6.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a5f3e936686bcd8542fafc94c68fdb23ec42d1d51a4777967ae815c90aff7296", size = 3023625, upload-time = "2025-10-13T23:09:21.556Z" },
+    { url = "https://files.pythonhosted.org/packages/58/d0/52d0cb3c0dfa8e94ba2118b7e91a70da76d6ede9de4e70374f831f38cfdf/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:705c32fec03dadc08f807d69ce557882005d43eb20ec62699f7036340f0d580f", size = 2701346, upload-time = "2025-10-13T23:09:25.031Z" },
+    { url = "https://files.pythonhosted.org/packages/69/bf/42a6f6501805c31cb65d8a6e3379eeec4fa6c26dc07c9ce894f363ccad1c/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:56d8166235a4c54e4f7ba082ec76890c820fa8c1b6c995ec59cead62a9698e59", size = 3153207, upload-time = "2025-10-13T23:09:28.254Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/e5/41fdee468b33759b42958347c2d70b0461bf8f70ba1762a94cdf2e9b0142/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:1ba43ba9081c00767083195222b6be74913de668296f55599658c4b0bb7cd327", size = 3105033, upload-time = "2025-10-13T23:09:31.545Z" },
+    { url = "https://files.pythonhosted.org/packages/03/e0/b6d733b4540c05bac546162e045b547031f4d88c67b7c864929d9bce29ad/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4f5df13c6742e3f0b494cfe9025dccdc8426a74cc9e3e5a1239311e07a4b24e0", size = 2954793, upload-time = "2025-10-13T23:09:34.988Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/34/8353ba79c8d0498eaacc077d58b384ef785e0b69c9cbff7c2580136b8fe3/arro3_core-0.6.5-cp310-cp310-win_amd64.whl", hash = "sha256:34676b728178236df63c9ea10b21432392d4b5bb51e2030e77c68eed4dede2ad", size = 2837495, upload-time = "2025-10-13T23:09:38.539Z" },
+    { url = "https://files.pythonhosted.org/packages/78/85/20e46d3ed59d2f93be4a4d1abea4f6bef3e96acd59bf5a50726f84303c51/arro3_core-0.6.5-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9d5999506daec1ab31096b3deb1e3573041d6ecadb4ca99c96f7ab26720c592c", size = 2685615, upload-time = "2025-10-13T23:09:41.793Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/9c/427d578f7d2bf3149515a8b75217e7189e7b1d74e5c5609e1a7e7f0f8d3c/arro3_core-0.6.5-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:bd3e251184c2dd6ade81c5613256b6d85ab3ddbd5af838b1de657e0ddec017f8", size = 2391944, upload-time = "2025-10-13T23:09:45.266Z" },
+    { url = "https://files.pythonhosted.org/packages/90/24/7e4af478eb889bfa401e1c1b8868048ca692e6205affbf81cf3666347852/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7cadb29349960d3821b0515d9df80f2725cea155ad966c699f6084de32e313cb", size = 2888376, upload-time = "2025-10-13T23:09:48.737Z" },
+    { url = "https://files.pythonhosted.org/packages/70/3b/01006a96bc980275aa4d2eb759c5f10afb7c85fcdce3c36ddb18635ad23b/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a922e560ed2ccee3293d51b39e013b51cc233895d25ddafcacfb83c540a19e6f", size = 2916568, upload-time = "2025-10-13T23:09:51.95Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/2f/4e04c7f5687de6fb6f88aa7590b16bcf507ba17ddbd268525f27b70b7a68/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68fe6672bf51f039b12046a209cba0a9405e10ae44e5a0d557f091b356a62051", size = 3144223, upload-time = "2025-10-13T23:09:55.387Z" },
+    { url = "https://files.pythonhosted.org/packages/31/4a/72dc383d1a0d14f1d453e334e3461e229762edb1bf3f75b3ab977e9386ed/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c3ee95603e375401a58ff763ce2c8aa858e0c4f757c1fb719f48fb070f540b2", size = 2781862, upload-time = "2025-10-13T23:09:59.035Z" },
+    { url = "https://files.pythonhosted.org/packages/14/dc/0df7684b683114eaf8e57989b4230edb359cbfb6e98b8770d69128b27572/arro3_core-0.6.5-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:fbaf6b65213630007b798b565e0701c2092a330deeba16bd3d896d401f7e9f28", size = 2522442, upload-time = "2025-10-13T23:10:02.134Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/04/75f8627cd7fe4d103eca51760d50269cfbc0bf6beaf83a3cdefb4ebd37c7/arro3_core-0.6.5-cp311-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:20679f874558bb2113e96325522625ec64a72687000b7a9578031a4d082c6ef5", size = 3033454, upload-time = "2025-10-13T23:10:05.192Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/19/f2d54985da65bf6d3da76218bee56383285035541c8d0cadb53095845b3e/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d82d6ec32d5c7c73057fb9c528390289fd5bc94b8d8f28fca9c56fc8e41c412c", size = 2705984, upload-time = "2025-10-13T23:10:08.518Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/53/b1d7742d6db7b4aa44d3785956955d651b3ac36db321625fd15466be1aca/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4cba4db0a4203a3ccf131c3fb7804d77f0740d6165ec9efa3aa3acbca87c43a3", size = 3157472, upload-time = "2025-10-13T23:10:11.976Z" },
+    { url = "https://files.pythonhosted.org/packages/05/31/68711327dbdd480aed54158fc1c46ab245e860ab0286e0916ce788f9889e/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:e358affc4a0fe5c1b5dccf4f92c43a836aaa4c4eab0906c83b00b60275de3b6d", size = 3117099, upload-time = "2025-10-13T23:10:15.374Z" },
+    { url = "https://files.pythonhosted.org/packages/31/e3/15ffca0797d9500b23759ae4477cf052fde8dd47a3890f4e4e1d04639016/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:324e43f07b7681846d00a8995b78bdc4b4a719047aa0d34426b462b8f208ee98", size = 2963677, upload-time = "2025-10-13T23:10:18.828Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/02/69e60dbe3bbe2bfc8b6dfa4f4bfcb8d1dd240a137bf2a5f7bcc84703f05c/arro3_core-0.6.5-cp311-abi3-win_amd64.whl", hash = "sha256:285f802c8a42fe29ecb84584d1700bc4c4f974552b75f805e1f4362d28b97080", size = 2850445, upload-time = "2025-10-13T23:10:22.345Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/29/2e5b091f6b5cffb6489dbe7ed353841568dde8ac4d1232c77321da1d0925/arro3_core-0.6.5-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:8c20e69c3b3411fd6ed56091f388e699072651e880e682be5bd14f3a392ed3e8", size = 2671985, upload-time = "2025-10-13T23:10:25.515Z" },
+    { url = "https://files.pythonhosted.org/packages/30/74/764ac4b58fef3fdfc655416c42349206156db5c687fa24a0674acaeaadbb/arro3_core-0.6.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:92211f1d03221ff74d0b535a576b39601083d8e98e9d47228314573f9d4f9ae2", size = 2382931, upload-time = "2025-10-13T23:10:29.893Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/07/bd8c92e218240ae8a30150a5d7a2dab359b452ab54a8bb7b90effe806e3d/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:280d933b75f2649779d76e32a07f91d2352a952f2c97ddf7b320e267f440cd42", size = 2879900, upload-time = "2025-10-13T23:10:33.238Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/d4/253725019fe2ae5f5fde87928118ffa568cc59f07b2d6a0e90620938c537/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfc3f6b93b924f43fb7985b06202343c30b43da6bd5055ba8b84eda431e494d4", size = 2904149, upload-time = "2025-10-13T23:10:36.547Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/b0/7a3dea641ac8de041c1a34859a2f2a82d3cdf3c3360872101c1d198a1e24/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5963635eb698ebc7da689e641f68b3998864bab894cf0ca84bd058b8c60d97f", size = 3143477, upload-time = "2025-10-13T23:10:40.232Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/05/1a50575be33fe9240898a1b5a8574658a905b5675865285585e070dcf7e2/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac291b3e74b57e56e03373d57530540cbbbfd92e4219fe2778ea531006673fe9", size = 2776522, upload-time = "2025-10-13T23:10:43.413Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/bd/e7b03207e7906e94e327cd4190fdb2d26ae52bc4ee1edeb057fed760796b/arro3_core-0.6.5-cp313-cp313t-manylinux_2_24_aarch64.whl", hash = "sha256:5d3f4cc58a654037d61f61ba230419da2c8f88a0ac82b9d41fe307f7cf9fda97", size = 2515426, upload-time = "2025-10-13T23:10:46.926Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/ed/82d1febd5c104eccdfb82434e3619125c328c36da143e19dfa3c86de4a81/arro3_core-0.6.5-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:93cddac90238d64451f5e66c630ded89d0b5fd6d2c099bf3a5151dde2c1ddf1d", size = 3024759, upload-time = "2025-10-13T23:10:50.281Z" },
+    { url = "https://files.pythonhosted.org/packages/da/cd/00e06907e42e404c21eb08282dee94ac7a1961facfa9a96d116829031721/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:1fa7ac10db5846c33f4e8b66a6eaa705d84998e38575a835acac9a6a6649933d", size = 2700191, upload-time = "2025-10-13T23:10:53.776Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/11/a4bb9a900f456a6905d481bd2289f7a2371dcde024de56779621fd6a92c3/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:ca69f698a065cdbf845d59d412bc204e8f8af12f93737d82e6a18f3cff812349", size = 3149963, upload-time = "2025-10-13T23:10:57.163Z" },
+    { url = "https://files.pythonhosted.org/packages/28/8a/79c76ad88b16f2fac25684f7313593738f353355eb1af2307e43efd7b1ca/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:de74a2512e2e2366d4b064c498c38672bf6ddea38acec8b1999b4e66182dd001", size = 3104663, upload-time = "2025-10-13T23:11:00.582Z" },
+    { url = "https://files.pythonhosted.org/packages/20/66/9152feaa87f851a37c1a2bd74fb89d7e82e4c76447ee590bf8e6fff5e9d8/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:806ca8e20507675b2de68b3d009f76e898cc3c3e441c834ea5220866f68aac50", size = 2956440, upload-time = "2025-10-13T23:11:03.769Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/66/f4179ef64d5c18fe76ec93cfbff42c0f401438ef771c6766b880044d7e13/arro3_core-0.6.5-cp313-cp313t-win_amd64.whl", hash = "sha256:8f6f0cc78877ade7ad6e678a4671b191406547e7b407bc9637436869c017ed47", size = 2845345, upload-time = "2025-10-13T23:11:07.447Z" },
+    { url = "https://files.pythonhosted.org/packages/07/c2/407d6bc19813fb74cc2b087ad3e959e102b29ff81e35dcc0ad0dfb5b946c/arro3_core-0.6.5-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:dfac7fac3c6a302399d94644d48682a19488a5b67bd1ccbdf6c560a7ffabde6d", size = 2680237, upload-time = "2025-10-13T23:11:10.876Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/73/c67156794d7e9734f4cc03d2eca7e44a1cc014686e6b7663f5110f58581d/arro3_core-0.6.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9fc70042e558d1cd5fbe917b58e8ef52701441e38ff30b1912858050f796a62c", size = 2386228, upload-time = "2025-10-13T23:11:14.02Z" },
+    { url = "https://files.pythonhosted.org/packages/79/e8/817ee1abb0cfa7e266ef00749b144553d2bb9c4679ca932ecbca9dc7dea9/arro3_core-0.6.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1583b29b2ba83927a33e5435e5d9d134114c45a6360a8bb4db4beda13dab4fd8", size = 2886476, upload-time = "2025-10-13T23:11:17.579Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/d6/1b9beceab797c4510abfc25ef6e657e4c940d06a9615927ce506463691dd/arro3_core-0.6.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6a170fe53f18dda4a4647fd3b8b4a9373fc11ac42c41a4b65f55d79ad531a33e", size = 2911941, upload-time = "2025-10-13T23:11:21.131Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/ed/4fe1fb9a24698fe6189111836d22c9582cbc92fa159b24b8664e924738dc/arro3_core-0.6.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:83047b4e6e18835c91c8d12c5494e6ababc7c185c5a772d3429e8f9b0c185894", size = 3150419, upload-time = "2025-10-13T23:11:24.503Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/91/d6215b782fa91493f504ae13623db889beeaf0519037c28fc6744464439a/arro3_core-0.6.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3d4393d281d1ef18927915a11187da27287d279f99d5325bc9afb417f76084f", size = 2777891, upload-time = "2025-10-13T23:11:28.11Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/de/0aa3504e6cbf406086de49b59cb0dcb3ab11f64acbb38602143e479831dc/arro3_core-0.6.5-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:f0c88d8babcf51affdd69390882e2f0ecb1890a1b8a5abfc087d003e7181eb6e", size = 2519673, upload-time = "2025-10-13T23:11:31.426Z" },
+    { url = "https://files.pythonhosted.org/packages/05/69/47bf9c9ab66bafc7056a41f6db9d2149639eea6417299e3fe6c01ef99b6c/arro3_core-0.6.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:36424e1d62212466a5cacdc27d414e99bf0fdab1544cc2b7e5b81e41437e5970", size = 3026254, upload-time = "2025-10-13T23:11:36.199Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/e8/638582437ab41ba52d3c7f2a1b0a98e4a05a51e3f660985e594b4f6c18d5/arro3_core-0.6.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4eb4d96f7db618f100758a8b7ec1b221c8737d543073701b7ffee74bc5019d46", size = 2704582, upload-time = "2025-10-13T23:11:39.408Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/0a/7bc46ee799459cce72a2e15b0eb184170f26cac37eace0b813e855fbc4d8/arro3_core-0.6.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:2cfe9b4b1dd663d256754f1aa7aae783a1cddd3eb5698892b9caf381431f0af7", size = 3155815, upload-time = "2025-10-13T23:11:43.304Z" },
+    { url = "https://files.pythonhosted.org/packages/99/8a/f20eff8f4ff5bd7db9b37b70ea058b37375a930a10e03d584a7597b6b740/arro3_core-0.6.5-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:a3b2621505f97eb5ce80f1c6fa8c77d18d757ab48d1f11d33a805e9ccbcd6fb6", size = 3107791, upload-time = "2025-10-13T23:11:46.735Z" },
+    { url = "https://files.pythonhosted.org/packages/79/da/60c66f0cc4a6af7f54e57973190540f77b84da1218fad2a9917e17bd897b/arro3_core-0.6.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6c1becbb96ceba0b20f3d4318dd35f3417ee9a49065813d99f52b0fa285fc569", size = 2957730, upload-time = "2025-10-13T23:11:49.875Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/8d/6e3235894196e1fd2be34e01ac2d4280dd24e6c9019e3b12603858651e91/arro3_core-0.6.5-cp39-cp39-win_amd64.whl", hash = "sha256:5459e7bd39bb9dd8c57aa06856d2bebc5c1ca782cbccab0e186c6c89530e4ca9", size = 2839298, upload-time = "2025-10-13T23:11:53.566Z" },
+    { url = "https://files.pythonhosted.org/packages/10/ca/b2139dbb25f9fefb9b1cdce8a73785615de6763af6a16bf6ff96a3b630f2/arro3_core-0.6.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:26d5b50139f1a96727fa1760b4d70393acf5ee0fba45346ad2d4f69824d3bdc2", size = 2676788, upload-time = "2025-10-13T23:11:56.965Z" },
+    { url = "https://files.pythonhosted.org/packages/34/a1/c68dde2944f493c8ccfcb91bf6da6d27a27c3674316dd09c9560f9e6ab1a/arro3_core-0.6.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b65b3d8d7f65f2f3c36002dc467380d7a31ea771132986dddc6341c5a9dc726f", size = 2382809, upload-time = "2025-10-13T23:12:00.175Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/fc/2fb81d42a3cecd632deace97dc23ac74083d60d158106440c783bae4ff01/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c3442a79a757ed3fbd7793de180019ae3201f04237537c2e2e3f1e3dd99b31c", size = 2882818, upload-time = "2025-10-13T23:12:03.721Z" },
+    { url = "https://files.pythonhosted.org/packages/58/7f/16f741e1d49ba5c5a893ce6f8eb0283d64bc68d6cc9e07ac62f96eaadfae/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:def7b0065a684d6f903a658d2567da47e2fcecde716e0b34eff4d899c6468c8d", size = 2907503, upload-time = "2025-10-13T23:12:07.066Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/45/2eb7972e0bbec0ee0ab22b0f166ec1ea74b53bd76c93a18ced434713e495/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbfe2f2d4d0d393833cd6a4bd9c15266a02307a3028f159155a1c536469c3ae7", size = 3143706, upload-time = "2025-10-13T23:12:10.492Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/af/b78e28842faa675e4e6c4d82e861accf21ac08bbab80a65fa80c578f80a1/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a191a3e4f72c34f7ace7724a94f2d90b06c804a6cbece4ae0f18d36325479cf3", size = 2775462, upload-time = "2025-10-13T23:12:14.026Z" },
+    { url = "https://files.pythonhosted.org/packages/45/df/950e57e4915e0457acadaaca13c4423d5e2652e403135eb7606d5e6e5443/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_24_aarch64.whl", hash = "sha256:e3f6ab4c6ea96c451eff72aa6c5b9835a0ea8a9847cfe3995c88cce0c7701fb5", size = 2516212, upload-time = "2025-10-13T23:12:17.548Z" },
+    { url = "https://files.pythonhosted.org/packages/07/73/821640d0827a829ed2565c2d4812080ab7fb86f0d271b462f9b37e6d946e/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:27df5239835330299636a02977f2cb34d5c460cc03b2ae1d6ab6a03d28051b08", size = 3023342, upload-time = "2025-10-13T23:12:21.308Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/30/51302d2f4d1b627dd11e2be979f2c48550b782d8d58d0378316342e284a8/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:71dce89c0e91be4cfb42591f03809235bbc374c396e08acdf93c4d85b09e40f5", size = 2700740, upload-time = "2025-10-13T23:12:24.968Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/e8/0c8a345a013bb64abea60b4864bacc01e43b8699b8874794baec9c8a7e76/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:d380c28f85568ed99c1686fb9d64b5a811d76d569f367cbec8ef7e58f6e2fdf9", size = 3152749, upload-time = "2025-10-13T23:12:28.393Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/42/003b30c4da394366d5967a5b993f7471a74182c983d8f757891b3dd5d594/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:8e359c0c4fe9992f5a863a4a31502ea58eb2f92988fc2e501850540b3eff0328", size = 3104676, upload-time = "2025-10-13T23:12:31.711Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/fd/4f8dac58ea17e05978bf35cb9a3e485b1ff3cdd6e2cc29deb08f54080de4/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:9a58acbc61480b533aa84d735db04b1e68fc7f6807ab694d606c03b5e694d83d", size = 2954405, upload-time = "2025-10-13T23:12:35.328Z" },
 ]
 
 [[package]]
@@ -204,27 +270,27 @@ dependencies = [
     { name = "six" },
     { name = "wheel" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/f3/af/4182184d3c338792894f34a62672919db7ca008c89abee9b564dd34d8029/astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872", size = 18290 }
+sdist = { url = "https://files.pythonhosted.org/packages/f3/af/4182184d3c338792894f34a62672919db7ca008c89abee9b564dd34d8029/astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872", size = 18290, upload-time = "2019-12-22T18:12:13.129Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/2b/03/13dde6512ad7b4557eb792fbcf0c653af6076b81e5941d36ec61f7ce6028/astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8", size = 12732 },
+    { url = "https://files.pythonhosted.org/packages/2b/03/13dde6512ad7b4557eb792fbcf0c653af6076b81e5941d36ec61f7ce6028/astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8", size = 12732, upload-time = "2019-12-22T18:12:11.297Z" },
 ]
 
 [[package]]
 name = "async-timeout"
 version = "5.0.1"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274 }
+sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233 },
+    { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" },
 ]
 
 [[package]]
 name = "attrs"
 version = "25.3.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032 }
+sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815 },
+    { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" },
 ]
 
 [[package]]
@@ -236,9 +302,9 @@ dependencies = [
     { name = "jmespath" },
     { name = "s3transfer" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/1e/43/0ef93cd27a8e753e66d93d7b94f686315384ab6cd63f065a14a4a6c9ee20/boto3-1.40.43.tar.gz", hash = "sha256:9ad9190672ce8736898bec2d94875aea6ae1ead2ac6d158e01d820f3ff9c23e0", size = 111552 }
+sdist = { url = "https://files.pythonhosted.org/packages/1e/43/0ef93cd27a8e753e66d93d7b94f686315384ab6cd63f065a14a4a6c9ee20/boto3-1.40.43.tar.gz", hash = "sha256:9ad9190672ce8736898bec2d94875aea6ae1ead2ac6d158e01d820f3ff9c23e0", size = 111552, upload-time = "2025-10-01T19:38:26.089Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f5/86/377e2b9aeddfdb7468223c7b48e29a1697b86c200c44916ddfb8dae05a68/boto3-1.40.43-py3-none-any.whl", hash = "sha256:c5d64ba2fb2d90c33c3969f3751869c45746d5efb5136e4cc619e3630ece89a3", size = 139344 },
+    { url = "https://files.pythonhosted.org/packages/f5/86/377e2b9aeddfdb7468223c7b48e29a1697b86c200c44916ddfb8dae05a68/boto3-1.40.43-py3-none-any.whl", hash = "sha256:c5d64ba2fb2d90c33c3969f3751869c45746d5efb5136e4cc619e3630ece89a3", size = 139344, upload-time = "2025-10-01T19:38:25Z" },
 ]
 
 [[package]]
@@ -248,93 +314,105 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "jmespath" },
     { name = "python-dateutil" },
-    { name = "urllib3" },
+    { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "urllib3", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/49/d0/3888673417202262ddd7e6361cab8e01ee2705e39643af8445e2eb276eab/botocore-1.40.43.tar.gz", hash = "sha256:d87412dc1ea785df156f412627d3417c9f9eb45601fd0846d8fe96fe3c78b630", size = 14389164 }
+sdist = { url = "https://files.pythonhosted.org/packages/49/d0/3888673417202262ddd7e6361cab8e01ee2705e39643af8445e2eb276eab/botocore-1.40.43.tar.gz", hash = "sha256:d87412dc1ea785df156f412627d3417c9f9eb45601fd0846d8fe96fe3c78b630", size = 14389164, upload-time = "2025-10-01T19:38:16.06Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/79/46/2eb4802e15e38befbea6cab7dafa1ab796722ab6f0833991c2a05e9f8ef0/botocore-1.40.43-py3-none-any.whl", hash = "sha256:1639f38999fc0cf42c92c5c83c5fbe189a4857a86f55b842be868e3283c6d3bb", size = 14057986 },
+    { url = "https://files.pythonhosted.org/packages/79/46/2eb4802e15e38befbea6cab7dafa1ab796722ab6f0833991c2a05e9f8ef0/botocore-1.40.43-py3-none-any.whl", hash = "sha256:1639f38999fc0cf42c92c5c83c5fbe189a4857a86f55b842be868e3283c6d3bb", size = 14057986, upload-time = "2025-10-01T19:38:13.714Z" },
 ]
 
 [[package]]
 name = "certifi"
 version = "2025.8.3"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386 }
+sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216 },
+    { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" },
 ]
 
 [[package]]
 name = "charset-normalizer"
 version = "3.4.3"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/d6/98/f3b8013223728a99b908c9344da3aa04ee6e3fa235f19409033eda92fb78/charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72", size = 207695 },
-    { url = "https://files.pythonhosted.org/packages/21/40/5188be1e3118c82dcb7c2a5ba101b783822cfb413a0268ed3be0468532de/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe", size = 147153 },
-    { url = "https://files.pythonhosted.org/packages/37/60/5d0d74bc1e1380f0b72c327948d9c2aca14b46a9efd87604e724260f384c/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601", size = 160428 },
-    { url = "https://files.pythonhosted.org/packages/85/9a/d891f63722d9158688de58d050c59dc3da560ea7f04f4c53e769de5140f5/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c", size = 157627 },
-    { url = "https://files.pythonhosted.org/packages/65/1a/7425c952944a6521a9cfa7e675343f83fd82085b8af2b1373a2409c683dc/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2", size = 152388 },
-    { url = "https://files.pythonhosted.org/packages/f0/c9/a2c9c2a355a8594ce2446085e2ec97fd44d323c684ff32042e2a6b718e1d/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0", size = 150077 },
-    { url = "https://files.pythonhosted.org/packages/3b/38/20a1f44e4851aa1c9105d6e7110c9d020e093dfa5836d712a5f074a12bf7/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0", size = 161631 },
-    { url = "https://files.pythonhosted.org/packages/a4/fa/384d2c0f57edad03d7bec3ebefb462090d8905b4ff5a2d2525f3bb711fac/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0", size = 159210 },
-    { url = "https://files.pythonhosted.org/packages/33/9e/eca49d35867ca2db336b6ca27617deed4653b97ebf45dfc21311ce473c37/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a", size = 153739 },
-    { url = "https://files.pythonhosted.org/packages/2a/91/26c3036e62dfe8de8061182d33be5025e2424002125c9500faff74a6735e/charset_normalizer-3.4.3-cp310-cp310-win32.whl", hash = "sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f", size = 99825 },
-    { url = "https://files.pythonhosted.org/packages/e2/c6/f05db471f81af1fa01839d44ae2a8bfeec8d2a8b4590f16c4e7393afd323/charset_normalizer-3.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669", size = 107452 },
-    { url = "https://files.pythonhosted.org/packages/7f/b5/991245018615474a60965a7c9cd2b4efbaabd16d582a5547c47ee1c7730b/charset_normalizer-3.4.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", size = 204483 },
-    { url = "https://files.pythonhosted.org/packages/c7/2a/ae245c41c06299ec18262825c1569c5d3298fc920e4ddf56ab011b417efd/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", size = 145520 },
-    { url = "https://files.pythonhosted.org/packages/3a/a4/b3b6c76e7a635748c4421d2b92c7b8f90a432f98bda5082049af37ffc8e3/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", size = 158876 },
-    { url = "https://files.pythonhosted.org/packages/e2/e6/63bb0e10f90a8243c5def74b5b105b3bbbfb3e7bb753915fe333fb0c11ea/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", size = 156083 },
-    { url = "https://files.pythonhosted.org/packages/87/df/b7737ff046c974b183ea9aa111b74185ac8c3a326c6262d413bd5a1b8c69/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", size = 150295 },
-    { url = "https://files.pythonhosted.org/packages/61/f1/190d9977e0084d3f1dc169acd060d479bbbc71b90bf3e7bf7b9927dec3eb/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", size = 148379 },
-    { url = "https://files.pythonhosted.org/packages/4c/92/27dbe365d34c68cfe0ca76f1edd70e8705d82b378cb54ebbaeabc2e3029d/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", size = 160018 },
-    { url = "https://files.pythonhosted.org/packages/99/04/baae2a1ea1893a01635d475b9261c889a18fd48393634b6270827869fa34/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", size = 157430 },
-    { url = "https://files.pythonhosted.org/packages/2f/36/77da9c6a328c54d17b960c89eccacfab8271fdaaa228305330915b88afa9/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", size = 151600 },
-    { url = "https://files.pythonhosted.org/packages/64/d4/9eb4ff2c167edbbf08cdd28e19078bf195762e9bd63371689cab5ecd3d0d/charset_normalizer-3.4.3-cp311-cp311-win32.whl", hash = "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", size = 99616 },
-    { url = "https://files.pythonhosted.org/packages/f4/9c/996a4a028222e7761a96634d1820de8a744ff4327a00ada9c8942033089b/charset_normalizer-3.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", size = 107108 },
-    { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655 },
-    { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223 },
-    { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366 },
-    { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104 },
-    { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830 },
-    { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854 },
-    { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670 },
-    { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501 },
-    { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173 },
-    { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822 },
-    { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543 },
-    { url = "https://files.pythonhosted.org/packages/65/ca/2135ac97709b400c7654b4b764daf5c5567c2da45a30cdd20f9eefe2d658/charset_normalizer-3.4.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", size = 205326 },
-    { url = "https://files.pythonhosted.org/packages/71/11/98a04c3c97dd34e49c7d247083af03645ca3730809a5509443f3c37f7c99/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", size = 146008 },
-    { url = "https://files.pythonhosted.org/packages/60/f5/4659a4cb3c4ec146bec80c32d8bb16033752574c20b1252ee842a95d1a1e/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", size = 159196 },
-    { url = "https://files.pythonhosted.org/packages/86/9e/f552f7a00611f168b9a5865a1414179b2c6de8235a4fa40189f6f79a1753/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", size = 156819 },
-    { url = "https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", size = 151350 },
-    { url = "https://files.pythonhosted.org/packages/c2/a9/3865b02c56f300a6f94fc631ef54f0a8a29da74fb45a773dfd3dcd380af7/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", size = 148644 },
-    { url = "https://files.pythonhosted.org/packages/77/d9/cbcf1a2a5c7d7856f11e7ac2d782aec12bdfea60d104e60e0aa1c97849dc/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9", size = 160468 },
-    { url = "https://files.pythonhosted.org/packages/f6/42/6f45efee8697b89fda4d50580f292b8f7f9306cb2971d4b53f8914e4d890/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", size = 158187 },
-    { url = "https://files.pythonhosted.org/packages/70/99/f1c3bdcfaa9c45b3ce96f70b14f070411366fa19549c1d4832c935d8e2c3/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", size = 152699 },
-    { url = "https://files.pythonhosted.org/packages/a3/ad/b0081f2f99a4b194bcbb1934ef3b12aa4d9702ced80a37026b7607c72e58/charset_normalizer-3.4.3-cp313-cp313-win32.whl", hash = "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", size = 99580 },
-    { url = "https://files.pythonhosted.org/packages/9a/8f/ae790790c7b64f925e5c953b924aaa42a243fb778fed9e41f147b2a5715a/charset_normalizer-3.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", size = 107366 },
-    { url = "https://files.pythonhosted.org/packages/8e/91/b5a06ad970ddc7a0e513112d40113e834638f4ca1120eb727a249fb2715e/charset_normalizer-3.4.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", size = 204342 },
-    { url = "https://files.pythonhosted.org/packages/ce/ec/1edc30a377f0a02689342f214455c3f6c2fbedd896a1d2f856c002fc3062/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", size = 145995 },
-    { url = "https://files.pythonhosted.org/packages/17/e5/5e67ab85e6d22b04641acb5399c8684f4d37caf7558a53859f0283a650e9/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", size = 158640 },
-    { url = "https://files.pythonhosted.org/packages/f1/e5/38421987f6c697ee3722981289d554957c4be652f963d71c5e46a262e135/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", size = 156636 },
-    { url = "https://files.pythonhosted.org/packages/a0/e4/5a075de8daa3ec0745a9a3b54467e0c2967daaaf2cec04c845f73493e9a1/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", size = 150939 },
-    { url = "https://files.pythonhosted.org/packages/02/f7/3611b32318b30974131db62b4043f335861d4d9b49adc6d57c1149cc49d4/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", size = 148580 },
-    { url = "https://files.pythonhosted.org/packages/7e/61/19b36f4bd67f2793ab6a99b979b4e4f3d8fc754cbdffb805335df4337126/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", size = 159870 },
-    { url = "https://files.pythonhosted.org/packages/06/57/84722eefdd338c04cf3030ada66889298eaedf3e7a30a624201e0cbe424a/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", size = 157797 },
-    { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224 },
-    { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086 },
-    { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400 },
-    { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175 },
+sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d6/98/f3b8013223728a99b908c9344da3aa04ee6e3fa235f19409033eda92fb78/charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72", size = 207695, upload-time = "2025-08-09T07:55:36.452Z" },
+    { url = "https://files.pythonhosted.org/packages/21/40/5188be1e3118c82dcb7c2a5ba101b783822cfb413a0268ed3be0468532de/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe", size = 147153, upload-time = "2025-08-09T07:55:38.467Z" },
+    { url = "https://files.pythonhosted.org/packages/37/60/5d0d74bc1e1380f0b72c327948d9c2aca14b46a9efd87604e724260f384c/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601", size = 160428, upload-time = "2025-08-09T07:55:40.072Z" },
+    { url = "https://files.pythonhosted.org/packages/85/9a/d891f63722d9158688de58d050c59dc3da560ea7f04f4c53e769de5140f5/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c", size = 157627, upload-time = "2025-08-09T07:55:41.706Z" },
+    { url = "https://files.pythonhosted.org/packages/65/1a/7425c952944a6521a9cfa7e675343f83fd82085b8af2b1373a2409c683dc/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2", size = 152388, upload-time = "2025-08-09T07:55:43.262Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/c9/a2c9c2a355a8594ce2446085e2ec97fd44d323c684ff32042e2a6b718e1d/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0", size = 150077, upload-time = "2025-08-09T07:55:44.903Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/38/20a1f44e4851aa1c9105d6e7110c9d020e093dfa5836d712a5f074a12bf7/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0", size = 161631, upload-time = "2025-08-09T07:55:46.346Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/fa/384d2c0f57edad03d7bec3ebefb462090d8905b4ff5a2d2525f3bb711fac/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0", size = 159210, upload-time = "2025-08-09T07:55:47.539Z" },
+    { url = "https://files.pythonhosted.org/packages/33/9e/eca49d35867ca2db336b6ca27617deed4653b97ebf45dfc21311ce473c37/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a", size = 153739, upload-time = "2025-08-09T07:55:48.744Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/91/26c3036e62dfe8de8061182d33be5025e2424002125c9500faff74a6735e/charset_normalizer-3.4.3-cp310-cp310-win32.whl", hash = "sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f", size = 99825, upload-time = "2025-08-09T07:55:50.305Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/c6/f05db471f81af1fa01839d44ae2a8bfeec8d2a8b4590f16c4e7393afd323/charset_normalizer-3.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669", size = 107452, upload-time = "2025-08-09T07:55:51.461Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/b5/991245018615474a60965a7c9cd2b4efbaabd16d582a5547c47ee1c7730b/charset_normalizer-3.4.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", size = 204483, upload-time = "2025-08-09T07:55:53.12Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/2a/ae245c41c06299ec18262825c1569c5d3298fc920e4ddf56ab011b417efd/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", size = 145520, upload-time = "2025-08-09T07:55:54.712Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/a4/b3b6c76e7a635748c4421d2b92c7b8f90a432f98bda5082049af37ffc8e3/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", size = 158876, upload-time = "2025-08-09T07:55:56.024Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/e6/63bb0e10f90a8243c5def74b5b105b3bbbfb3e7bb753915fe333fb0c11ea/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", size = 156083, upload-time = "2025-08-09T07:55:57.582Z" },
+    { url = "https://files.pythonhosted.org/packages/87/df/b7737ff046c974b183ea9aa111b74185ac8c3a326c6262d413bd5a1b8c69/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", size = 150295, upload-time = "2025-08-09T07:55:59.147Z" },
+    { url = "https://files.pythonhosted.org/packages/61/f1/190d9977e0084d3f1dc169acd060d479bbbc71b90bf3e7bf7b9927dec3eb/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", size = 148379, upload-time = "2025-08-09T07:56:00.364Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/92/27dbe365d34c68cfe0ca76f1edd70e8705d82b378cb54ebbaeabc2e3029d/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", size = 160018, upload-time = "2025-08-09T07:56:01.678Z" },
+    { url = "https://files.pythonhosted.org/packages/99/04/baae2a1ea1893a01635d475b9261c889a18fd48393634b6270827869fa34/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", size = 157430, upload-time = "2025-08-09T07:56:02.87Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/36/77da9c6a328c54d17b960c89eccacfab8271fdaaa228305330915b88afa9/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", size = 151600, upload-time = "2025-08-09T07:56:04.089Z" },
+    { url = "https://files.pythonhosted.org/packages/64/d4/9eb4ff2c167edbbf08cdd28e19078bf195762e9bd63371689cab5ecd3d0d/charset_normalizer-3.4.3-cp311-cp311-win32.whl", hash = "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", size = 99616, upload-time = "2025-08-09T07:56:05.658Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/9c/996a4a028222e7761a96634d1820de8a744ff4327a00ada9c8942033089b/charset_normalizer-3.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", size = 107108, upload-time = "2025-08-09T07:56:07.176Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" },
+    { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" },
+    { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" },
+    { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" },
+    { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" },
+    { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" },
+    { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" },
+    { url = "https://files.pythonhosted.org/packages/65/ca/2135ac97709b400c7654b4b764daf5c5567c2da45a30cdd20f9eefe2d658/charset_normalizer-3.4.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", size = 205326, upload-time = "2025-08-09T07:56:24.721Z" },
+    { url = "https://files.pythonhosted.org/packages/71/11/98a04c3c97dd34e49c7d247083af03645ca3730809a5509443f3c37f7c99/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", size = 146008, upload-time = "2025-08-09T07:56:26.004Z" },
+    { url = "https://files.pythonhosted.org/packages/60/f5/4659a4cb3c4ec146bec80c32d8bb16033752574c20b1252ee842a95d1a1e/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", size = 159196, upload-time = "2025-08-09T07:56:27.25Z" },
+    { url = "https://files.pythonhosted.org/packages/86/9e/f552f7a00611f168b9a5865a1414179b2c6de8235a4fa40189f6f79a1753/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", size = 156819, upload-time = "2025-08-09T07:56:28.515Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", size = 151350, upload-time = "2025-08-09T07:56:29.716Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/a9/3865b02c56f300a6f94fc631ef54f0a8a29da74fb45a773dfd3dcd380af7/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", size = 148644, upload-time = "2025-08-09T07:56:30.984Z" },
+    { url = "https://files.pythonhosted.org/packages/77/d9/cbcf1a2a5c7d7856f11e7ac2d782aec12bdfea60d104e60e0aa1c97849dc/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9", size = 160468, upload-time = "2025-08-09T07:56:32.252Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/42/6f45efee8697b89fda4d50580f292b8f7f9306cb2971d4b53f8914e4d890/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", size = 158187, upload-time = "2025-08-09T07:56:33.481Z" },
+    { url = "https://files.pythonhosted.org/packages/70/99/f1c3bdcfaa9c45b3ce96f70b14f070411366fa19549c1d4832c935d8e2c3/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", size = 152699, upload-time = "2025-08-09T07:56:34.739Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/ad/b0081f2f99a4b194bcbb1934ef3b12aa4d9702ced80a37026b7607c72e58/charset_normalizer-3.4.3-cp313-cp313-win32.whl", hash = "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", size = 99580, upload-time = "2025-08-09T07:56:35.981Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/8f/ae790790c7b64f925e5c953b924aaa42a243fb778fed9e41f147b2a5715a/charset_normalizer-3.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", size = 107366, upload-time = "2025-08-09T07:56:37.339Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/91/b5a06ad970ddc7a0e513112d40113e834638f4ca1120eb727a249fb2715e/charset_normalizer-3.4.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", size = 204342, upload-time = "2025-08-09T07:56:38.687Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/ec/1edc30a377f0a02689342f214455c3f6c2fbedd896a1d2f856c002fc3062/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", size = 145995, upload-time = "2025-08-09T07:56:40.048Z" },
+    { url = "https://files.pythonhosted.org/packages/17/e5/5e67ab85e6d22b04641acb5399c8684f4d37caf7558a53859f0283a650e9/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", size = 158640, upload-time = "2025-08-09T07:56:41.311Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/e5/38421987f6c697ee3722981289d554957c4be652f963d71c5e46a262e135/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", size = 156636, upload-time = "2025-08-09T07:56:43.195Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/e4/5a075de8daa3ec0745a9a3b54467e0c2967daaaf2cec04c845f73493e9a1/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", size = 150939, upload-time = "2025-08-09T07:56:44.819Z" },
+    { url = "https://files.pythonhosted.org/packages/02/f7/3611b32318b30974131db62b4043f335861d4d9b49adc6d57c1149cc49d4/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", size = 148580, upload-time = "2025-08-09T07:56:46.684Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/61/19b36f4bd67f2793ab6a99b979b4e4f3d8fc754cbdffb805335df4337126/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", size = 159870, upload-time = "2025-08-09T07:56:47.941Z" },
+    { url = "https://files.pythonhosted.org/packages/06/57/84722eefdd338c04cf3030ada66889298eaedf3e7a30a624201e0cbe424a/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", size = 157797, upload-time = "2025-08-09T07:56:49.756Z" },
+    { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224, upload-time = "2025-08-09T07:56:51.369Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086, upload-time = "2025-08-09T07:56:52.722Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400, upload-time = "2025-08-09T07:56:55.172Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/ca/9a0983dd5c8e9733565cf3db4df2b0a2e9a82659fd8aa2a868ac6e4a991f/charset_normalizer-3.4.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:70bfc5f2c318afece2f5838ea5e4c3febada0be750fcf4775641052bbba14d05", size = 207520, upload-time = "2025-08-09T07:57:11.026Z" },
+    { url = "https://files.pythonhosted.org/packages/39/c6/99271dc37243a4f925b09090493fb96c9333d7992c6187f5cfe5312008d2/charset_normalizer-3.4.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23b6b24d74478dc833444cbd927c338349d6ae852ba53a0d02a2de1fce45b96e", size = 147307, upload-time = "2025-08-09T07:57:12.4Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/69/132eab043356bba06eb333cc2cc60c6340857d0a2e4ca6dc2b51312886b3/charset_normalizer-3.4.3-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:34a7f768e3f985abdb42841e20e17b330ad3aaf4bb7e7aeeb73db2e70f077b99", size = 160448, upload-time = "2025-08-09T07:57:13.712Z" },
+    { url = "https://files.pythonhosted.org/packages/04/9a/914d294daa4809c57667b77470533e65def9c0be1ef8b4c1183a99170e9d/charset_normalizer-3.4.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb731e5deb0c7ef82d698b0f4c5bb724633ee2a489401594c5c88b02e6cb15f7", size = 157758, upload-time = "2025-08-09T07:57:14.979Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/a8/6f5bcf1bcf63cb45625f7c5cadca026121ff8a6c8a3256d8d8cd59302663/charset_normalizer-3.4.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:257f26fed7d7ff59921b78244f3cd93ed2af1800ff048c33f624c87475819dd7", size = 152487, upload-time = "2025-08-09T07:57:16.332Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/72/d3d0e9592f4e504f9dea08b8db270821c909558c353dc3b457ed2509f2fb/charset_normalizer-3.4.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1ef99f0456d3d46a50945c98de1774da86f8e992ab5c77865ea8b8195341fc19", size = 150054, upload-time = "2025-08-09T07:57:17.576Z" },
+    { url = "https://files.pythonhosted.org/packages/20/30/5f64fe3981677fe63fa987b80e6c01042eb5ff653ff7cec1b7bd9268e54e/charset_normalizer-3.4.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:2c322db9c8c89009a990ef07c3bcc9f011a3269bc06782f916cd3d9eed7c9312", size = 161703, upload-time = "2025-08-09T07:57:20.012Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/ef/dd08b2cac9284fd59e70f7d97382c33a3d0a926e45b15fc21b3308324ffd/charset_normalizer-3.4.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:511729f456829ef86ac41ca78c63a5cb55240ed23b4b737faca0eb1abb1c41bc", size = 159096, upload-time = "2025-08-09T07:57:21.329Z" },
+    { url = "https://files.pythonhosted.org/packages/45/8c/dcef87cfc2b3f002a6478f38906f9040302c68aebe21468090e39cde1445/charset_normalizer-3.4.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:88ab34806dea0671532d3f82d82b85e8fc23d7b2dd12fa837978dad9bb392a34", size = 153852, upload-time = "2025-08-09T07:57:22.608Z" },
+    { url = "https://files.pythonhosted.org/packages/63/86/9cbd533bd37883d467fcd1bd491b3547a3532d0fbb46de2b99feeebf185e/charset_normalizer-3.4.3-cp39-cp39-win32.whl", hash = "sha256:16a8770207946ac75703458e2c743631c79c59c5890c80011d536248f8eaa432", size = 99840, upload-time = "2025-08-09T07:57:23.883Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/d6/7e805c8e5c46ff9729c49950acc4ee0aeb55efb8b3a56687658ad10c3216/charset_normalizer-3.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:d22dbedd33326a4a5190dd4fe9e9e693ef12160c77382d9e87919bce54f3d4ca", size = 107438, upload-time = "2025-08-09T07:57:25.287Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" },
 ]
 
 [[package]]
 name = "colorama"
 version = "0.4.6"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 }
+sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 },
+    { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
 ]
 
 [[package]]
@@ -342,84 +420,106 @@ name = "datafusion"
 version = "52.0.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" },
-    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" },
-    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+    { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/58/04/4dabd255e04801b942221bf7eeea661f540d8c116e6b4a783fe2479410f0/datafusion-52.0.0.tar.gz", hash = "sha256:842cf9cdb523d04a053c5408da24645e3b2adce5d6c42ddc80a8c5edf9013ff3", size = 204988 }
+sdist = { url = "https://files.pythonhosted.org/packages/58/04/4dabd255e04801b942221bf7eeea661f540d8c116e6b4a783fe2479410f0/datafusion-52.0.0.tar.gz", hash = "sha256:842cf9cdb523d04a053c5408da24645e3b2adce5d6c42ddc80a8c5edf9013ff3", size = 204988, upload-time = "2026-02-23T12:22:50.919Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/77/38/66b2f2fd77d3fb66ff48a8922130379dece3ba6d2e29fc86fbb4298a874b/datafusion-52.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:999881df12ab78b6c8f04dd2056b24389374e93775a649ed20c5e35db2f42f65", size = 31473623 },
-    { url = "https://files.pythonhosted.org/packages/d0/b5/ce6c6030fa8e4fc38d10d5c4aa9cc6fe1cda625e409a18eb08ea09a87c8d/datafusion-52.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd58e64158152f5c4a5836a3ce3bcca2a109d600c9ce7efdcf82e61c1ab0fbc8", size = 28108736 },
-    { url = "https://files.pythonhosted.org/packages/d8/c1/d7ac9ddc9f54a8f178900f529a723d6121361111f0d0d2527bb47f86f6ce/datafusion-52.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ab3591904f32ce290ff7161fb804e1c7bf323de16e3ddc8cf1f76310e994208e", size = 30699663 },
-    { url = "https://files.pythonhosted.org/packages/b0/2f/14cffc5305abe05d56f3e99e8054c96bd94411185de059a98fc1ca0e5ec0/datafusion-52.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ac4b364937c277bbfcac032dbc49d08c078b13ba3f8bfda117da5fda4ea328bc", size = 33050161 },
-    { url = "https://files.pythonhosted.org/packages/24/ae/3fdea50fa88f304db96728a67deb6e07bb0d9a02f665ca09db4237a9a199/datafusion-52.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:67e252ef20b918537c8fdb47e6c825c0bd639795e19715a85fedde331a83d2e1", size = 33717685 },
+    { url = "https://files.pythonhosted.org/packages/77/38/66b2f2fd77d3fb66ff48a8922130379dece3ba6d2e29fc86fbb4298a874b/datafusion-52.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:999881df12ab78b6c8f04dd2056b24389374e93775a649ed20c5e35db2f42f65", size = 31473623, upload-time = "2026-02-23T12:22:30.437Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/b5/ce6c6030fa8e4fc38d10d5c4aa9cc6fe1cda625e409a18eb08ea09a87c8d/datafusion-52.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd58e64158152f5c4a5836a3ce3bcca2a109d600c9ce7efdcf82e61c1ab0fbc8", size = 28108736, upload-time = "2026-02-23T12:22:33.5Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/c1/d7ac9ddc9f54a8f178900f529a723d6121361111f0d0d2527bb47f86f6ce/datafusion-52.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ab3591904f32ce290ff7161fb804e1c7bf323de16e3ddc8cf1f76310e994208e", size = 30699663, upload-time = "2026-02-23T12:22:37.193Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/2f/14cffc5305abe05d56f3e99e8054c96bd94411185de059a98fc1ca0e5ec0/datafusion-52.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ac4b364937c277bbfcac032dbc49d08c078b13ba3f8bfda117da5fda4ea328bc", size = 33050161, upload-time = "2026-02-23T12:22:40.858Z" },
+    { url = "https://files.pythonhosted.org/packages/24/ae/3fdea50fa88f304db96728a67deb6e07bb0d9a02f665ca09db4237a9a199/datafusion-52.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:67e252ef20b918537c8fdb47e6c825c0bd639795e19715a85fedde331a83d2e1", size = 33717685, upload-time = "2026-02-23T12:22:44.463Z" },
 ]
 
+[[package]]
+name = "datasets"
+version = "0.0.9"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/cd/fe/4d2874473a753d59c83335691bd9532704f2605418a0d288a1d70fa003fc/datasets-0.0.9.zip", hash = "sha256:86d54441bab87aebb2aa3bf0853aa7fb7abed8c708f9bb08a88e86a498972010", size = 4013, upload-time = "2015-08-18T00:07:40.556Z" }
+
 [[package]]
 name = "datasets"
 version = "4.1.1"
 source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version >= '3.14'",
+    "python_full_version == '3.13.*'",
+    "python_full_version == '3.12.*'",
+    "python_full_version == '3.11.*'",
+    "python_full_version == '3.10.*'",
+]
 dependencies = [
-    { name = "dill" },
-    { name = "filelock" },
-    { name = "fsspec", extra = ["http"] },
-    { name = "huggingface-hub" },
-    { name = "multiprocess" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "dill", marker = "python_full_version >= '3.10'" },
+    { name = "filelock", marker = "python_full_version >= '3.10'" },
+    { name = "fsspec", extra = ["http"], marker = "python_full_version >= '3.10'" },
+    { name = "huggingface-hub", marker = "python_full_version >= '3.10'" },
+    { name = "multiprocess", marker = "python_full_version >= '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "packaging" },
-    { name = "pandas" },
-    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" },
-    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" },
-    { name = "pyyaml" },
-    { name = "requests" },
-    { name = "tqdm" },
-    { name = "xxhash" },
+    { name = "packaging", marker = "python_full_version >= '3.10'" },
+    { name = "pandas", marker = "python_full_version >= '3.10'" },
+    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+    { name = "pyyaml", marker = "python_full_version >= '3.10'" },
+    { name = "requests", marker = "python_full_version >= '3.10'" },
+    { name = "tqdm", marker = "python_full_version >= '3.10'" },
+    { name = "xxhash", marker = "python_full_version >= '3.10'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/91/a4/73f8e6ef52c535e1d20d5b2ca83bfe6de399d8b8b8a61ccc8d63d60735aa/datasets-4.1.1.tar.gz", hash = "sha256:7d8d5ba8b12861d2c44bfff9c83484ebfafff1ff553371e5901a8d3aab5450e2", size = 579324 }
+sdist = { url = "https://files.pythonhosted.org/packages/91/a4/73f8e6ef52c535e1d20d5b2ca83bfe6de399d8b8b8a61ccc8d63d60735aa/datasets-4.1.1.tar.gz", hash = "sha256:7d8d5ba8b12861d2c44bfff9c83484ebfafff1ff553371e5901a8d3aab5450e2", size = 579324, upload-time = "2025-09-18T13:14:27.108Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f4/c8/09012ac195a0aab58755800d2efdc0e7d5905053509f12cb5d136c911cda/datasets-4.1.1-py3-none-any.whl", hash = "sha256:62e4f6899a36be9ec74a7e759a6951253cc85b3fcfa0a759b0efa8353b149dac", size = 503623 },
+    { url = "https://files.pythonhosted.org/packages/f4/c8/09012ac195a0aab58755800d2efdc0e7d5905053509f12cb5d136c911cda/datasets-4.1.1-py3-none-any.whl", hash = "sha256:62e4f6899a36be9ec74a7e759a6951253cc85b3fcfa0a759b0efa8353b149dac", size = 503623, upload-time = "2025-09-18T13:14:25.111Z" },
 ]
 
 [[package]]
 name = "dill"
 version = "0.4.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976 }
+sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976, upload-time = "2025-04-16T00:41:48.867Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668 },
+    { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668, upload-time = "2025-04-16T00:41:47.671Z" },
 ]
 
 [[package]]
 name = "duckdb"
 version = "1.4.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/82/93/adc0d183642fc9a602ca9b97cb16754c84b8c1d92e5b99aec412e0c419a8/duckdb-1.4.0.tar.gz", hash = "sha256:bd5edee8bd5a73b5822f2b390668597b5fcdc2d3292c244d8d933bb87ad6ac4c", size = 18453175 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/0f/4a/b2e17dbe2953481b084f355f162ed319a67ef760e28794c6870058583aec/duckdb-1.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e24e981a6c87e299201694b9bb24fff0beb04ccad399fca6f13072a59814488f", size = 31293005 },
-    { url = "https://files.pythonhosted.org/packages/a9/89/e34ed03cce7e35b83c1f056126aa4e8e8097eb93e7324463020f85d5cbfa/duckdb-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db500ef2c8cb7dc1ca078740ecf1dceaa20d3f5dc5bce269be45d5cff4170c0f", size = 17288207 },
-    { url = "https://files.pythonhosted.org/packages/f8/17/7ff24799ee98c4dbb177c3ec6c93e38e9513828785c31757c727b47ad71e/duckdb-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a65739b8a7106634e6e77d0e110fc5e057b88edc9df6cb1683d499a1e5aa3177", size = 14817523 },
-    { url = "https://files.pythonhosted.org/packages/fc/ab/7a482a76ff75212b5cf4f2172a802f2a59b4ab096416e5821aa62a305bc4/duckdb-1.4.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d59f7be24862adb803a1ddfc9c3b8cb09e6005bca0c9c6f7c631a1da1c3aa0c", size = 18410654 },
-    { url = "https://files.pythonhosted.org/packages/1e/f6/a235233b973652b31448b6d600604620d02fc552b90ab94ca7f645fd5ac0/duckdb-1.4.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d052a87e9edf4eb3bab0b7a6ac995676018c6083b8049421628dfa3b983a2d4", size = 20399121 },
-    { url = "https://files.pythonhosted.org/packages/b1/cf/63fedb74d00d7c4e19ffc73a1d8d98ee8d3d6498cf2865509c104aa8e799/duckdb-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:0329b81e587f745b2fc6f3a488ea3188b0f029c3b5feef43792a25eaac84ac01", size = 12283288 },
-    { url = "https://files.pythonhosted.org/packages/60/e9/b29cc5bceac52e049b20d613551a2171a092df07f26d4315f3f9651c80d4/duckdb-1.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6505fed1ccae8df9f574e744c48fa32ee2feaeebe5346c2daf4d4d10a8dac5aa", size = 31290878 },
-    { url = "https://files.pythonhosted.org/packages/1f/68/d88a15dba48bf6a4b33f1be5097ef45c83f7b9e97c854cc638a85bb07d70/duckdb-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:36974a04b29c74ac2143457e95420a7422016d050e28573060b89a90b9cf2b57", size = 17288823 },
-    { url = "https://files.pythonhosted.org/packages/8c/7e/e3d2101dc6bbd60f2b3c1d748351ff541fc8c48790ac1218c0199cb930f6/duckdb-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90484b896e5059f145d1facfabea38e22c54a2dcc2bd62dd6c290423f0aee258", size = 14819684 },
-    { url = "https://files.pythonhosted.org/packages/c4/bb/4ec8e4d03cb5b77d75b9ee0057c2c714cffaa9bda1e55ffec833458af0a3/duckdb-1.4.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a969d624b385853b31a43b0a23089683297da2f14846243921c6dbec8382d659", size = 18410075 },
-    { url = "https://files.pythonhosted.org/packages/ec/21/e896616d892d50dc1e0c142428e9359b483d4dd6e339231d822e57834ad3/duckdb-1.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5935644f96a75e9f6f3c3eeb3da14cdcaf7bad14d1199c08439103decb29466a", size = 20402984 },
-    { url = "https://files.pythonhosted.org/packages/c4/c0/b5eb9497e4a9167d23fbad745969eaa36e28d346648e17565471892d1b33/duckdb-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:300aa0e963af97969c38440877fffd576fc1f49c1f5914789a9d01f2fe7def91", size = 12282971 },
-    { url = "https://files.pythonhosted.org/packages/e8/6d/0c774d6af1aed82dbe855d266cb000a1c09ea31ed7d6c3a79e2167a38e7a/duckdb-1.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:18b3a048fca6cc7bafe08b10e1b0ab1509d7a0381ffb2c70359e7dc56d8a705d", size = 31307425 },
-    { url = "https://files.pythonhosted.org/packages/d3/c0/1fd7b7b2c0c53d8d748d2f28ea9096df5ee9dc39fa736cca68acabe69656/duckdb-1.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c1271cb85aeacccfd0b1284e816280a7450df1dd4dd85ccb2848563cfdf90e9", size = 17295727 },
-    { url = "https://files.pythonhosted.org/packages/98/d3/4d4c4bd667b7ada5f6c207c2f127591ebb8468333f207f8f10ff0532578e/duckdb-1.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55064dd2e25711eeaa6a72c25405bdd7994c81a3221657e94309a2faf65d25a6", size = 14826879 },
-    { url = "https://files.pythonhosted.org/packages/b0/48/e0c1b97d76fb7567c53db5739931323238fad54a642707008104f501db37/duckdb-1.4.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0536d7c81bc506532daccf373ddbc8c6add46aeb70ef3cd5ee70ad5c2b3165ea", size = 18417856 },
-    { url = "https://files.pythonhosted.org/packages/12/78/297b838f3b9511589badc8f472f70b31cf3bbf9eb99fa0a4d6e911d3114a/duckdb-1.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:784554e3ddfcfc5c5c7b1aa1f9925fedb7938f6628729adba48f7ea37554598f", size = 20427154 },
-    { url = "https://files.pythonhosted.org/packages/ea/57/500d251b886494f6c52d56eeab8a1860572ee62aed05d7d50c71ba2320f3/duckdb-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:c5d2aa4d6981f525ada95e6db41bb929403632bb5ff24bd6d6dd551662b1b613", size = 12290108 },
-    { url = "https://files.pythonhosted.org/packages/2f/64/ee22b2b8572746e1523143b9f28d606575782e0204de5020656a1d15dd14/duckdb-1.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1d94d010a09b1a62d9021a2a71cf266188750f3c9b1912ccd6afe104a6ce8010", size = 31307662 },
-    { url = "https://files.pythonhosted.org/packages/76/2e/4241cd00046ca6b781bd1d9002e8223af061e85d1cc21830aa63e7a7db7c/duckdb-1.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c61756fa8b3374627e5fa964b8e0d5b58e364dce59b87dba7fb7bc6ede196b26", size = 17295617 },
-    { url = "https://files.pythonhosted.org/packages/f7/98/5ab136bc7b12ac18580350a220db7c00606be9eac2d89de259cce733f64c/duckdb-1.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e70d7d9881ea2c0836695de70ea68c970e18a2856ba3d6502e276c85bd414ae7", size = 14826727 },
-    { url = "https://files.pythonhosted.org/packages/23/32/57866cf8881288b3dfb9212720221fb890daaa534dbdc6fe3fff3979ecd1/duckdb-1.4.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2de258a93435c977a0ec3a74ec8f60c2f215ddc73d427ee49adc4119558facd3", size = 18421289 },
-    { url = "https://files.pythonhosted.org/packages/a0/83/7438fb43be451a7d4a04650aaaf662b2ff2d95895bbffe3e0e28cbe030c9/duckdb-1.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a6d3659641d517dd9ed1ab66f110cdbdaa6900106f116effaf2dbedd83c38de3", size = 20426547 },
-    { url = "https://files.pythonhosted.org/packages/21/b2/98fb89ae81611855f35984e96f648d871f3967bb3f524b51d1372d052f0c/duckdb-1.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:07fcc612ea5f0fe6032b92bcc93693034eb00e7a23eb9146576911d5326af4f7", size = 12290467 },
+sdist = { url = "https://files.pythonhosted.org/packages/82/93/adc0d183642fc9a602ca9b97cb16754c84b8c1d92e5b99aec412e0c419a8/duckdb-1.4.0.tar.gz", hash = "sha256:bd5edee8bd5a73b5822f2b390668597b5fcdc2d3292c244d8d933bb87ad6ac4c", size = 18453175, upload-time = "2025-09-16T10:22:41.509Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0f/4a/b2e17dbe2953481b084f355f162ed319a67ef760e28794c6870058583aec/duckdb-1.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e24e981a6c87e299201694b9bb24fff0beb04ccad399fca6f13072a59814488f", size = 31293005, upload-time = "2025-09-16T10:21:28.296Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/89/e34ed03cce7e35b83c1f056126aa4e8e8097eb93e7324463020f85d5cbfa/duckdb-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db500ef2c8cb7dc1ca078740ecf1dceaa20d3f5dc5bce269be45d5cff4170c0f", size = 17288207, upload-time = "2025-09-16T10:21:31.129Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/17/7ff24799ee98c4dbb177c3ec6c93e38e9513828785c31757c727b47ad71e/duckdb-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a65739b8a7106634e6e77d0e110fc5e057b88edc9df6cb1683d499a1e5aa3177", size = 14817523, upload-time = "2025-09-16T10:21:33.397Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/ab/7a482a76ff75212b5cf4f2172a802f2a59b4ab096416e5821aa62a305bc4/duckdb-1.4.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d59f7be24862adb803a1ddfc9c3b8cb09e6005bca0c9c6f7c631a1da1c3aa0c", size = 18410654, upload-time = "2025-09-16T10:21:35.864Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/f6/a235233b973652b31448b6d600604620d02fc552b90ab94ca7f645fd5ac0/duckdb-1.4.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d052a87e9edf4eb3bab0b7a6ac995676018c6083b8049421628dfa3b983a2d4", size = 20399121, upload-time = "2025-09-16T10:21:38.524Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/cf/63fedb74d00d7c4e19ffc73a1d8d98ee8d3d6498cf2865509c104aa8e799/duckdb-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:0329b81e587f745b2fc6f3a488ea3188b0f029c3b5feef43792a25eaac84ac01", size = 12283288, upload-time = "2025-09-16T10:21:40.732Z" },
+    { url = "https://files.pythonhosted.org/packages/60/e9/b29cc5bceac52e049b20d613551a2171a092df07f26d4315f3f9651c80d4/duckdb-1.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6505fed1ccae8df9f574e744c48fa32ee2feaeebe5346c2daf4d4d10a8dac5aa", size = 31290878, upload-time = "2025-09-16T10:21:43.256Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/68/d88a15dba48bf6a4b33f1be5097ef45c83f7b9e97c854cc638a85bb07d70/duckdb-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:36974a04b29c74ac2143457e95420a7422016d050e28573060b89a90b9cf2b57", size = 17288823, upload-time = "2025-09-16T10:21:45.716Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/7e/e3d2101dc6bbd60f2b3c1d748351ff541fc8c48790ac1218c0199cb930f6/duckdb-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90484b896e5059f145d1facfabea38e22c54a2dcc2bd62dd6c290423f0aee258", size = 14819684, upload-time = "2025-09-16T10:21:48.117Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/bb/4ec8e4d03cb5b77d75b9ee0057c2c714cffaa9bda1e55ffec833458af0a3/duckdb-1.4.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a969d624b385853b31a43b0a23089683297da2f14846243921c6dbec8382d659", size = 18410075, upload-time = "2025-09-16T10:21:50.517Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/21/e896616d892d50dc1e0c142428e9359b483d4dd6e339231d822e57834ad3/duckdb-1.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5935644f96a75e9f6f3c3eeb3da14cdcaf7bad14d1199c08439103decb29466a", size = 20402984, upload-time = "2025-09-16T10:21:52.808Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/c0/b5eb9497e4a9167d23fbad745969eaa36e28d346648e17565471892d1b33/duckdb-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:300aa0e963af97969c38440877fffd576fc1f49c1f5914789a9d01f2fe7def91", size = 12282971, upload-time = "2025-09-16T10:21:55.314Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/6d/0c774d6af1aed82dbe855d266cb000a1c09ea31ed7d6c3a79e2167a38e7a/duckdb-1.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:18b3a048fca6cc7bafe08b10e1b0ab1509d7a0381ffb2c70359e7dc56d8a705d", size = 31307425, upload-time = "2025-09-16T10:21:57.83Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/c0/1fd7b7b2c0c53d8d748d2f28ea9096df5ee9dc39fa736cca68acabe69656/duckdb-1.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c1271cb85aeacccfd0b1284e816280a7450df1dd4dd85ccb2848563cfdf90e9", size = 17295727, upload-time = "2025-09-16T10:22:02.242Z" },
+    { url = "https://files.pythonhosted.org/packages/98/d3/4d4c4bd667b7ada5f6c207c2f127591ebb8468333f207f8f10ff0532578e/duckdb-1.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55064dd2e25711eeaa6a72c25405bdd7994c81a3221657e94309a2faf65d25a6", size = 14826879, upload-time = "2025-09-16T10:22:05.162Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/48/e0c1b97d76fb7567c53db5739931323238fad54a642707008104f501db37/duckdb-1.4.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0536d7c81bc506532daccf373ddbc8c6add46aeb70ef3cd5ee70ad5c2b3165ea", size = 18417856, upload-time = "2025-09-16T10:22:07.919Z" },
+    { url = "https://files.pythonhosted.org/packages/12/78/297b838f3b9511589badc8f472f70b31cf3bbf9eb99fa0a4d6e911d3114a/duckdb-1.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:784554e3ddfcfc5c5c7b1aa1f9925fedb7938f6628729adba48f7ea37554598f", size = 20427154, upload-time = "2025-09-16T10:22:10.216Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/57/500d251b886494f6c52d56eeab8a1860572ee62aed05d7d50c71ba2320f3/duckdb-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:c5d2aa4d6981f525ada95e6db41bb929403632bb5ff24bd6d6dd551662b1b613", size = 12290108, upload-time = "2025-09-16T10:22:12.668Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/64/ee22b2b8572746e1523143b9f28d606575782e0204de5020656a1d15dd14/duckdb-1.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1d94d010a09b1a62d9021a2a71cf266188750f3c9b1912ccd6afe104a6ce8010", size = 31307662, upload-time = "2025-09-16T10:22:14.9Z" },
+    { url = "https://files.pythonhosted.org/packages/76/2e/4241cd00046ca6b781bd1d9002e8223af061e85d1cc21830aa63e7a7db7c/duckdb-1.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c61756fa8b3374627e5fa964b8e0d5b58e364dce59b87dba7fb7bc6ede196b26", size = 17295617, upload-time = "2025-09-16T10:22:17.239Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/98/5ab136bc7b12ac18580350a220db7c00606be9eac2d89de259cce733f64c/duckdb-1.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e70d7d9881ea2c0836695de70ea68c970e18a2856ba3d6502e276c85bd414ae7", size = 14826727, upload-time = "2025-09-16T10:22:19.415Z" },
+    { url = "https://files.pythonhosted.org/packages/23/32/57866cf8881288b3dfb9212720221fb890daaa534dbdc6fe3fff3979ecd1/duckdb-1.4.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2de258a93435c977a0ec3a74ec8f60c2f215ddc73d427ee49adc4119558facd3", size = 18421289, upload-time = "2025-09-16T10:22:21.564Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/83/7438fb43be451a7d4a04650aaaf662b2ff2d95895bbffe3e0e28cbe030c9/duckdb-1.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a6d3659641d517dd9ed1ab66f110cdbdaa6900106f116effaf2dbedd83c38de3", size = 20426547, upload-time = "2025-09-16T10:22:23.759Z" },
+    { url = "https://files.pythonhosted.org/packages/21/b2/98fb89ae81611855f35984e96f648d871f3967bb3f524b51d1372d052f0c/duckdb-1.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:07fcc612ea5f0fe6032b92bcc93693034eb00e7a23eb9146576911d5326af4f7", size = 12290467, upload-time = "2025-09-16T10:22:25.923Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/42/0f355319b3e8ee1703d0e17378dd829db391434306621f85c110134f2763/duckdb-1.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1c97ee61c582002b654331f7fd967d6b1e83bf7fdb0772f409dfd4b6af3a70f4", size = 31292373, upload-time = "2025-09-16T10:22:28.118Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/52/091dbef5eb2ac4e60a9c6d38fcc7c7530a75fafa0f37658450e8731a265b/duckdb-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:74e3d6295355160df5d3588b880e8bcae23fdd6f573f538793a8a1abf4c2c29d", size = 17288145, upload-time = "2025-09-16T10:22:30.346Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/6c/879317d9c3ac7a2a1f0618ca536a48ebfa4b9fe202f9783e07070e168192/duckdb-1.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d0c76425e4ffe98069dd4fc4752ab919a4125dc0d176bb676b3065fdea152c42", size = 14816258, upload-time = "2025-09-16T10:22:32.442Z" },
+    { url = "https://files.pythonhosted.org/packages/95/87/83ac8e67c0530b69fe39f91bbb7f3bd0a49b0c24216cffa9c5561fb2845c/duckdb-1.4.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c122bd7d80ab5057f53024ee3922d7612a5cdc99583fae730990964aebc3fd4", size = 18391043, upload-time = "2025-09-16T10:22:34.616Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/01/1d70bd6c594ef915c004edc0f1119d1602173dc5ce91c1eed7368f6aab34/duckdb-1.4.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:30689c1436bca723526be6102fe1f4f82ea6d4780fb9ca196bda7ed5ec227950", size = 20385348, upload-time = "2025-09-16T10:22:36.982Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/04/0650128cdcdc5208c4f51341a0a3f8db436ecaba51032c6065e20ea0baae/duckdb-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:4c55a367c1296617cff89c5e1c7153f1dc3c3b556ef70711a45b0236515f80c2", size = 12283322, upload-time = "2025-09-16T10:22:39.388Z" },
 ]
 
 [[package]]
@@ -429,144 +529,199 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "typing-extensions", marker = "python_full_version < '3.11'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749 }
+sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674 },
+    { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" },
 ]
 
 [[package]]
 name = "filelock"
 version = "3.19.1"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/40/bb/0ab3e58d22305b6f5440629d20683af28959bf793d98d11950e305c1c326/filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58", size = 17687 }
+sdist = { url = "https://files.pythonhosted.org/packages/40/bb/0ab3e58d22305b6f5440629d20683af28959bf793d98d11950e305c1c326/filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58", size = 17687, upload-time = "2025-08-14T16:56:03.016Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988 },
+    { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" },
+]
+
+[[package]]
+name = "flatbuffers"
+version = "2.0.7"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d1/90/0532e737a11e1dc50e9e352c3ccc97338cb75991f83279c2edbc9234e022/flatbuffers-2.0.7.tar.gz", hash = "sha256:0ae7d69c5b82bf41962ca5fde9cc43033bc9501311d975fd5a25e8a7d29c1245", size = 22686, upload-time = "2022-08-23T22:50:07.903Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d7/0d/b5bfb553a6ac66d6ec2b6d7f1e814a908fba7188356ac94bb36ae3d905c3/flatbuffers-2.0.7-py2.py3-none-any.whl", hash = "sha256:71e135d533be527192819aaab757c5e3d109cb10fbb01e687f6bdb7a61ad39d1", size = 26562, upload-time = "2022-08-23T22:50:56.342Z" },
 ]
 
 [[package]]
 name = "flatbuffers"
 version = "25.9.23"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/9d/1f/3ee70b0a55137442038f2a33469cc5fddd7e0ad2abf83d7497c18a2b6923/flatbuffers-25.9.23.tar.gz", hash = "sha256:676f9fa62750bb50cf531b42a0a2a118ad8f7f797a511eda12881c016f093b12", size = 22067 }
+resolution-markers = [
+    "python_full_version >= '3.14'",
+    "python_full_version == '3.13.*'",
+    "python_full_version == '3.12.*'",
+    "python_full_version == '3.11.*'",
+    "python_full_version == '3.10.*'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9d/1f/3ee70b0a55137442038f2a33469cc5fddd7e0ad2abf83d7497c18a2b6923/flatbuffers-25.9.23.tar.gz", hash = "sha256:676f9fa62750bb50cf531b42a0a2a118ad8f7f797a511eda12881c016f093b12", size = 22067, upload-time = "2025-09-24T05:25:30.106Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ee/1b/00a78aa2e8fbd63f9af08c9c19e6deb3d5d66b4dda677a0f61654680ee89/flatbuffers-25.9.23-py2.py3-none-any.whl", hash = "sha256:255538574d6cb6d0a79a17ec8bc0d30985913b87513a01cce8bcdb6b4c44d0e2", size = 30869 },
+    { url = "https://files.pythonhosted.org/packages/ee/1b/00a78aa2e8fbd63f9af08c9c19e6deb3d5d66b4dda677a0f61654680ee89/flatbuffers-25.9.23-py2.py3-none-any.whl", hash = "sha256:255538574d6cb6d0a79a17ec8bc0d30985913b87513a01cce8bcdb6b4c44d0e2", size = 30869, upload-time = "2025-09-24T05:25:28.912Z" },
 ]
 
 [[package]]
 name = "frozenlist"
 version = "1.7.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/79/b1/b64018016eeb087db503b038296fd782586432b9c077fc5c7839e9cb6ef6/frozenlist-1.7.0.tar.gz", hash = "sha256:2e310d81923c2437ea8670467121cc3e9b0f76d3043cc1d2331d56c7fb7a3a8f", size = 45078 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/af/36/0da0a49409f6b47cc2d060dc8c9040b897b5902a8a4e37d9bc1deb11f680/frozenlist-1.7.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cc4df77d638aa2ed703b878dd093725b72a824c3c546c076e8fdf276f78ee84a", size = 81304 },
-    { url = "https://files.pythonhosted.org/packages/77/f0/77c11d13d39513b298e267b22eb6cb559c103d56f155aa9a49097221f0b6/frozenlist-1.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:716a9973a2cc963160394f701964fe25012600f3d311f60c790400b00e568b61", size = 47735 },
-    { url = "https://files.pythonhosted.org/packages/37/12/9d07fa18971a44150593de56b2f2947c46604819976784bcf6ea0d5db43b/frozenlist-1.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a0fd1bad056a3600047fb9462cff4c5322cebc59ebf5d0a3725e0ee78955001d", size = 46775 },
-    { url = "https://files.pythonhosted.org/packages/70/34/f73539227e06288fcd1f8a76853e755b2b48bca6747e99e283111c18bcd4/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3789ebc19cb811163e70fe2bd354cea097254ce6e707ae42e56f45e31e96cb8e", size = 224644 },
-    { url = "https://files.pythonhosted.org/packages/fb/68/c1d9c2f4a6e438e14613bad0f2973567586610cc22dcb1e1241da71de9d3/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:af369aa35ee34f132fcfad5be45fbfcde0e3a5f6a1ec0712857f286b7d20cca9", size = 222125 },
-    { url = "https://files.pythonhosted.org/packages/b9/d0/98e8f9a515228d708344d7c6986752be3e3192d1795f748c24bcf154ad99/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac64b6478722eeb7a3313d494f8342ef3478dff539d17002f849101b212ef97c", size = 233455 },
-    { url = "https://files.pythonhosted.org/packages/79/df/8a11bcec5600557f40338407d3e5bea80376ed1c01a6c0910fcfdc4b8993/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f89f65d85774f1797239693cef07ad4c97fdd0639544bad9ac4b869782eb1981", size = 227339 },
-    { url = "https://files.pythonhosted.org/packages/50/82/41cb97d9c9a5ff94438c63cc343eb7980dac4187eb625a51bdfdb7707314/frozenlist-1.7.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1073557c941395fdfcfac13eb2456cb8aad89f9de27bae29fabca8e563b12615", size = 212969 },
-    { url = "https://files.pythonhosted.org/packages/13/47/f9179ee5ee4f55629e4f28c660b3fdf2775c8bfde8f9c53f2de2d93f52a9/frozenlist-1.7.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ed8d2fa095aae4bdc7fdd80351009a48d286635edffee66bf865e37a9125c50", size = 222862 },
-    { url = "https://files.pythonhosted.org/packages/1a/52/df81e41ec6b953902c8b7e3a83bee48b195cb0e5ec2eabae5d8330c78038/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:24c34bea555fe42d9f928ba0a740c553088500377448febecaa82cc3e88aa1fa", size = 222492 },
-    { url = "https://files.pythonhosted.org/packages/84/17/30d6ea87fa95a9408245a948604b82c1a4b8b3e153cea596421a2aef2754/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:69cac419ac6a6baad202c85aaf467b65ac860ac2e7f2ac1686dc40dbb52f6577", size = 238250 },
-    { url = "https://files.pythonhosted.org/packages/8f/00/ecbeb51669e3c3df76cf2ddd66ae3e48345ec213a55e3887d216eb4fbab3/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:960d67d0611f4c87da7e2ae2eacf7ea81a5be967861e0c63cf205215afbfac59", size = 218720 },
-    { url = "https://files.pythonhosted.org/packages/1a/c0/c224ce0e0eb31cc57f67742071bb470ba8246623c1823a7530be0e76164c/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:41be2964bd4b15bf575e5daee5a5ce7ed3115320fb3c2b71fca05582ffa4dc9e", size = 232585 },
-    { url = "https://files.pythonhosted.org/packages/55/3c/34cb694abf532f31f365106deebdeac9e45c19304d83cf7d51ebbb4ca4d1/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:46d84d49e00c9429238a7ce02dc0be8f6d7cd0cd405abd1bebdc991bf27c15bd", size = 234248 },
-    { url = "https://files.pythonhosted.org/packages/98/c0/2052d8b6cecda2e70bd81299e3512fa332abb6dcd2969b9c80dfcdddbf75/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:15900082e886edb37480335d9d518cec978afc69ccbc30bd18610b7c1b22a718", size = 221621 },
-    { url = "https://files.pythonhosted.org/packages/c5/bf/7dcebae315436903b1d98ffb791a09d674c88480c158aa171958a3ac07f0/frozenlist-1.7.0-cp310-cp310-win32.whl", hash = "sha256:400ddd24ab4e55014bba442d917203c73b2846391dd42ca5e38ff52bb18c3c5e", size = 39578 },
-    { url = "https://files.pythonhosted.org/packages/8f/5f/f69818f017fa9a3d24d1ae39763e29b7f60a59e46d5f91b9c6b21622f4cd/frozenlist-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:6eb93efb8101ef39d32d50bce242c84bcbddb4f7e9febfa7b524532a239b4464", size = 43830 },
-    { url = "https://files.pythonhosted.org/packages/34/7e/803dde33760128acd393a27eb002f2020ddb8d99d30a44bfbaab31c5f08a/frozenlist-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa51e147a66b2d74de1e6e2cf5921890de6b0f4820b257465101d7f37b49fb5a", size = 82251 },
-    { url = "https://files.pythonhosted.org/packages/75/a9/9c2c5760b6ba45eae11334db454c189d43d34a4c0b489feb2175e5e64277/frozenlist-1.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9b35db7ce1cd71d36ba24f80f0c9e7cff73a28d7a74e91fe83e23d27c7828750", size = 48183 },
-    { url = "https://files.pythonhosted.org/packages/47/be/4038e2d869f8a2da165f35a6befb9158c259819be22eeaf9c9a8f6a87771/frozenlist-1.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:34a69a85e34ff37791e94542065c8416c1afbf820b68f720452f636d5fb990cd", size = 47107 },
-    { url = "https://files.pythonhosted.org/packages/79/26/85314b8a83187c76a37183ceed886381a5f992975786f883472fcb6dc5f2/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a646531fa8d82c87fe4bb2e596f23173caec9185bfbca5d583b4ccfb95183e2", size = 237333 },
-    { url = "https://files.pythonhosted.org/packages/1f/fd/e5b64f7d2c92a41639ffb2ad44a6a82f347787abc0c7df5f49057cf11770/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:79b2ffbba483f4ed36a0f236ccb85fbb16e670c9238313709638167670ba235f", size = 231724 },
-    { url = "https://files.pythonhosted.org/packages/20/fb/03395c0a43a5976af4bf7534759d214405fbbb4c114683f434dfdd3128ef/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a26f205c9ca5829cbf82bb2a84b5c36f7184c4316617d7ef1b271a56720d6b30", size = 245842 },
-    { url = "https://files.pythonhosted.org/packages/d0/15/c01c8e1dffdac5d9803507d824f27aed2ba76b6ed0026fab4d9866e82f1f/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bcacfad3185a623fa11ea0e0634aac7b691aa925d50a440f39b458e41c561d98", size = 239767 },
-    { url = "https://files.pythonhosted.org/packages/14/99/3f4c6fe882c1f5514b6848aa0a69b20cb5e5d8e8f51a339d48c0e9305ed0/frozenlist-1.7.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72c1b0fe8fe451b34f12dce46445ddf14bd2a5bcad7e324987194dc8e3a74c86", size = 224130 },
-    { url = "https://files.pythonhosted.org/packages/4d/83/220a374bd7b2aeba9d0725130665afe11de347d95c3620b9b82cc2fcab97/frozenlist-1.7.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61d1a5baeaac6c0798ff6edfaeaa00e0e412d49946c53fae8d4b8e8b3566c4ae", size = 235301 },
-    { url = "https://files.pythonhosted.org/packages/03/3c/3e3390d75334a063181625343e8daab61b77e1b8214802cc4e8a1bb678fc/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7edf5c043c062462f09b6820de9854bf28cc6cc5b6714b383149745e287181a8", size = 234606 },
-    { url = "https://files.pythonhosted.org/packages/23/1e/58232c19608b7a549d72d9903005e2d82488f12554a32de2d5fb59b9b1ba/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d50ac7627b3a1bd2dcef6f9da89a772694ec04d9a61b66cf87f7d9446b4a0c31", size = 248372 },
-    { url = "https://files.pythonhosted.org/packages/c0/a4/e4a567e01702a88a74ce8a324691e62a629bf47d4f8607f24bf1c7216e7f/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ce48b2fece5aeb45265bb7a58259f45027db0abff478e3077e12b05b17fb9da7", size = 229860 },
-    { url = "https://files.pythonhosted.org/packages/73/a6/63b3374f7d22268b41a9db73d68a8233afa30ed164c46107b33c4d18ecdd/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:fe2365ae915a1fafd982c146754e1de6ab3478def8a59c86e1f7242d794f97d5", size = 245893 },
-    { url = "https://files.pythonhosted.org/packages/6d/eb/d18b3f6e64799a79673c4ba0b45e4cfbe49c240edfd03a68be20002eaeaa/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:45a6f2fdbd10e074e8814eb98b05292f27bad7d1883afbe009d96abdcf3bc898", size = 246323 },
-    { url = "https://files.pythonhosted.org/packages/5a/f5/720f3812e3d06cd89a1d5db9ff6450088b8f5c449dae8ffb2971a44da506/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:21884e23cffabb157a9dd7e353779077bf5b8f9a58e9b262c6caad2ef5f80a56", size = 233149 },
-    { url = "https://files.pythonhosted.org/packages/69/68/03efbf545e217d5db8446acfd4c447c15b7c8cf4dbd4a58403111df9322d/frozenlist-1.7.0-cp311-cp311-win32.whl", hash = "sha256:284d233a8953d7b24f9159b8a3496fc1ddc00f4db99c324bd5fb5f22d8698ea7", size = 39565 },
-    { url = "https://files.pythonhosted.org/packages/58/17/fe61124c5c333ae87f09bb67186d65038834a47d974fc10a5fadb4cc5ae1/frozenlist-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:387cbfdcde2f2353f19c2f66bbb52406d06ed77519ac7ee21be0232147c2592d", size = 44019 },
-    { url = "https://files.pythonhosted.org/packages/ef/a2/c8131383f1e66adad5f6ecfcce383d584ca94055a34d683bbb24ac5f2f1c/frozenlist-1.7.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3dbf9952c4bb0e90e98aec1bd992b3318685005702656bc6f67c1a32b76787f2", size = 81424 },
-    { url = "https://files.pythonhosted.org/packages/4c/9d/02754159955088cb52567337d1113f945b9e444c4960771ea90eb73de8db/frozenlist-1.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1f5906d3359300b8a9bb194239491122e6cf1444c2efb88865426f170c262cdb", size = 47952 },
-    { url = "https://files.pythonhosted.org/packages/01/7a/0046ef1bd6699b40acd2067ed6d6670b4db2f425c56980fa21c982c2a9db/frozenlist-1.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3dabd5a8f84573c8d10d8859a50ea2dec01eea372031929871368c09fa103478", size = 46688 },
-    { url = "https://files.pythonhosted.org/packages/d6/a2/a910bafe29c86997363fb4c02069df4ff0b5bc39d33c5198b4e9dd42d8f8/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa57daa5917f1738064f302bf2626281a1cb01920c32f711fbc7bc36111058a8", size = 243084 },
-    { url = "https://files.pythonhosted.org/packages/64/3e/5036af9d5031374c64c387469bfcc3af537fc0f5b1187d83a1cf6fab1639/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c193dda2b6d49f4c4398962810fa7d7c78f032bf45572b3e04dd5249dff27e08", size = 233524 },
-    { url = "https://files.pythonhosted.org/packages/06/39/6a17b7c107a2887e781a48ecf20ad20f1c39d94b2a548c83615b5b879f28/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe2b675cf0aaa6d61bf8fbffd3c274b3c9b7b1623beb3809df8a81399a4a9c4", size = 248493 },
-    { url = "https://files.pythonhosted.org/packages/be/00/711d1337c7327d88c44d91dd0f556a1c47fb99afc060ae0ef66b4d24793d/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fc5d5cda37f62b262405cf9652cf0856839c4be8ee41be0afe8858f17f4c94b", size = 244116 },
-    { url = "https://files.pythonhosted.org/packages/24/fe/74e6ec0639c115df13d5850e75722750adabdc7de24e37e05a40527ca539/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0d5ce521d1dd7d620198829b87ea002956e4319002ef0bc8d3e6d045cb4646e", size = 224557 },
-    { url = "https://files.pythonhosted.org/packages/8d/db/48421f62a6f77c553575201e89048e97198046b793f4a089c79a6e3268bd/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:488d0a7d6a0008ca0db273c542098a0fa9e7dfaa7e57f70acef43f32b3f69dca", size = 241820 },
-    { url = "https://files.pythonhosted.org/packages/1d/fa/cb4a76bea23047c8462976ea7b7a2bf53997a0ca171302deae9d6dd12096/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:15a7eaba63983d22c54d255b854e8108e7e5f3e89f647fc854bd77a237e767df", size = 236542 },
-    { url = "https://files.pythonhosted.org/packages/5d/32/476a4b5cfaa0ec94d3f808f193301debff2ea42288a099afe60757ef6282/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1eaa7e9c6d15df825bf255649e05bd8a74b04a4d2baa1ae46d9c2d00b2ca2cb5", size = 249350 },
-    { url = "https://files.pythonhosted.org/packages/8d/ba/9a28042f84a6bf8ea5dbc81cfff8eaef18d78b2a1ad9d51c7bc5b029ad16/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4389e06714cfa9d47ab87f784a7c5be91d3934cd6e9a7b85beef808297cc025", size = 225093 },
-    { url = "https://files.pythonhosted.org/packages/bc/29/3a32959e68f9cf000b04e79ba574527c17e8842e38c91d68214a37455786/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:73bd45e1488c40b63fe5a7df892baf9e2a4d4bb6409a2b3b78ac1c6236178e01", size = 245482 },
-    { url = "https://files.pythonhosted.org/packages/80/e8/edf2f9e00da553f07f5fa165325cfc302dead715cab6ac8336a5f3d0adc2/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99886d98e1643269760e5fe0df31e5ae7050788dd288947f7f007209b8c33f08", size = 249590 },
-    { url = "https://files.pythonhosted.org/packages/1c/80/9a0eb48b944050f94cc51ee1c413eb14a39543cc4f760ed12657a5a3c45a/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:290a172aae5a4c278c6da8a96222e6337744cd9c77313efe33d5670b9f65fc43", size = 237785 },
-    { url = "https://files.pythonhosted.org/packages/f3/74/87601e0fb0369b7a2baf404ea921769c53b7ae00dee7dcfe5162c8c6dbf0/frozenlist-1.7.0-cp312-cp312-win32.whl", hash = "sha256:426c7bc70e07cfebc178bc4c2bf2d861d720c4fff172181eeb4a4c41d4ca2ad3", size = 39487 },
-    { url = "https://files.pythonhosted.org/packages/0b/15/c026e9a9fc17585a9d461f65d8593d281fedf55fbf7eb53f16c6df2392f9/frozenlist-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:563b72efe5da92e02eb68c59cb37205457c977aa7a449ed1b37e6939e5c47c6a", size = 43874 },
-    { url = "https://files.pythonhosted.org/packages/24/90/6b2cebdabdbd50367273c20ff6b57a3dfa89bd0762de02c3a1eb42cb6462/frozenlist-1.7.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee80eeda5e2a4e660651370ebffd1286542b67e268aa1ac8d6dbe973120ef7ee", size = 79791 },
-    { url = "https://files.pythonhosted.org/packages/83/2e/5b70b6a3325363293fe5fc3ae74cdcbc3e996c2a11dde2fd9f1fb0776d19/frozenlist-1.7.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d1a81c85417b914139e3a9b995d4a1c84559afc839a93cf2cb7f15e6e5f6ed2d", size = 47165 },
-    { url = "https://files.pythonhosted.org/packages/f4/25/a0895c99270ca6966110f4ad98e87e5662eab416a17e7fd53c364bf8b954/frozenlist-1.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cbb65198a9132ebc334f237d7b0df163e4de83fb4f2bdfe46c1e654bdb0c5d43", size = 45881 },
-    { url = "https://files.pythonhosted.org/packages/19/7c/71bb0bbe0832793c601fff68cd0cf6143753d0c667f9aec93d3c323f4b55/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dab46c723eeb2c255a64f9dc05b8dd601fde66d6b19cdb82b2e09cc6ff8d8b5d", size = 232409 },
-    { url = "https://files.pythonhosted.org/packages/c0/45/ed2798718910fe6eb3ba574082aaceff4528e6323f9a8570be0f7028d8e9/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6aeac207a759d0dedd2e40745575ae32ab30926ff4fa49b1635def65806fddee", size = 225132 },
-    { url = "https://files.pythonhosted.org/packages/ba/e2/8417ae0f8eacb1d071d4950f32f229aa6bf68ab69aab797b72a07ea68d4f/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bd8c4e58ad14b4fa7802b8be49d47993182fdd4023393899632c88fd8cd994eb", size = 237638 },
-    { url = "https://files.pythonhosted.org/packages/f8/b7/2ace5450ce85f2af05a871b8c8719b341294775a0a6c5585d5e6170f2ce7/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04fb24d104f425da3540ed83cbfc31388a586a7696142004c577fa61c6298c3f", size = 233539 },
-    { url = "https://files.pythonhosted.org/packages/46/b9/6989292c5539553dba63f3c83dc4598186ab2888f67c0dc1d917e6887db6/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a5c505156368e4ea6b53b5ac23c92d7edc864537ff911d2fb24c140bb175e60", size = 215646 },
-    { url = "https://files.pythonhosted.org/packages/72/31/bc8c5c99c7818293458fe745dab4fd5730ff49697ccc82b554eb69f16a24/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bd7eb96a675f18aa5c553eb7ddc24a43c8c18f22e1f9925528128c052cdbe00", size = 232233 },
-    { url = "https://files.pythonhosted.org/packages/59/52/460db4d7ba0811b9ccb85af996019f5d70831f2f5f255f7cc61f86199795/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:05579bf020096fe05a764f1f84cd104a12f78eaab68842d036772dc6d4870b4b", size = 227996 },
-    { url = "https://files.pythonhosted.org/packages/ba/c9/f4b39e904c03927b7ecf891804fd3b4df3db29b9e487c6418e37988d6e9d/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:376b6222d114e97eeec13d46c486facd41d4f43bab626b7c3f6a8b4e81a5192c", size = 242280 },
-    { url = "https://files.pythonhosted.org/packages/b8/33/3f8d6ced42f162d743e3517781566b8481322be321b486d9d262adf70bfb/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0aa7e176ebe115379b5b1c95b4096fb1c17cce0847402e227e712c27bdb5a949", size = 217717 },
-    { url = "https://files.pythonhosted.org/packages/3e/e8/ad683e75da6ccef50d0ab0c2b2324b32f84fc88ceee778ed79b8e2d2fe2e/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3fbba20e662b9c2130dc771e332a99eff5da078b2b2648153a40669a6d0e36ca", size = 236644 },
-    { url = "https://files.pythonhosted.org/packages/b2/14/8d19ccdd3799310722195a72ac94ddc677541fb4bef4091d8e7775752360/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f3f4410a0a601d349dd406b5713fec59b4cee7e71678d5b17edda7f4655a940b", size = 238879 },
-    { url = "https://files.pythonhosted.org/packages/ce/13/c12bf657494c2fd1079a48b2db49fa4196325909249a52d8f09bc9123fd7/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e2cdfaaec6a2f9327bf43c933c0319a7c429058e8537c508964a133dffee412e", size = 232502 },
-    { url = "https://files.pythonhosted.org/packages/d7/8b/e7f9dfde869825489382bc0d512c15e96d3964180c9499efcec72e85db7e/frozenlist-1.7.0-cp313-cp313-win32.whl", hash = "sha256:5fc4df05a6591c7768459caba1b342d9ec23fa16195e744939ba5914596ae3e1", size = 39169 },
-    { url = "https://files.pythonhosted.org/packages/35/89/a487a98d94205d85745080a37860ff5744b9820a2c9acbcdd9440bfddf98/frozenlist-1.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:52109052b9791a3e6b5d1b65f4b909703984b770694d3eb64fad124c835d7cba", size = 43219 },
-    { url = "https://files.pythonhosted.org/packages/56/d5/5c4cf2319a49eddd9dd7145e66c4866bdc6f3dbc67ca3d59685149c11e0d/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a6f86e4193bb0e235ef6ce3dde5cbabed887e0b11f516ce8a0f4d3b33078ec2d", size = 84345 },
-    { url = "https://files.pythonhosted.org/packages/a4/7d/ec2c1e1dc16b85bc9d526009961953df9cec8481b6886debb36ec9107799/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:82d664628865abeb32d90ae497fb93df398a69bb3434463d172b80fc25b0dd7d", size = 48880 },
-    { url = "https://files.pythonhosted.org/packages/69/86/f9596807b03de126e11e7d42ac91e3d0b19a6599c714a1989a4e85eeefc4/frozenlist-1.7.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:912a7e8375a1c9a68325a902f3953191b7b292aa3c3fb0d71a216221deca460b", size = 48498 },
-    { url = "https://files.pythonhosted.org/packages/5e/cb/df6de220f5036001005f2d726b789b2c0b65f2363b104bbc16f5be8084f8/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9537c2777167488d539bc5de2ad262efc44388230e5118868e172dd4a552b146", size = 292296 },
-    { url = "https://files.pythonhosted.org/packages/83/1f/de84c642f17c8f851a2905cee2dae401e5e0daca9b5ef121e120e19aa825/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f34560fb1b4c3e30ba35fa9a13894ba39e5acfc5f60f57d8accde65f46cc5e74", size = 273103 },
-    { url = "https://files.pythonhosted.org/packages/88/3c/c840bfa474ba3fa13c772b93070893c6e9d5c0350885760376cbe3b6c1b3/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:acd03d224b0175f5a850edc104ac19040d35419eddad04e7cf2d5986d98427f1", size = 292869 },
-    { url = "https://files.pythonhosted.org/packages/a6/1c/3efa6e7d5a39a1d5ef0abeb51c48fb657765794a46cf124e5aca2c7a592c/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2038310bc582f3d6a09b3816ab01737d60bf7b1ec70f5356b09e84fb7408ab1", size = 291467 },
-    { url = "https://files.pythonhosted.org/packages/4f/00/d5c5e09d4922c395e2f2f6b79b9a20dab4b67daaf78ab92e7729341f61f6/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8c05e4c8e5f36e5e088caa1bf78a687528f83c043706640a92cb76cd6999384", size = 266028 },
-    { url = "https://files.pythonhosted.org/packages/4e/27/72765be905619dfde25a7f33813ac0341eb6b076abede17a2e3fbfade0cb/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:765bb588c86e47d0b68f23c1bee323d4b703218037765dcf3f25c838c6fecceb", size = 284294 },
-    { url = "https://files.pythonhosted.org/packages/88/67/c94103a23001b17808eb7dd1200c156bb69fb68e63fcf0693dde4cd6228c/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:32dc2e08c67d86d0969714dd484fd60ff08ff81d1a1e40a77dd34a387e6ebc0c", size = 281898 },
-    { url = "https://files.pythonhosted.org/packages/42/34/a3e2c00c00f9e2a9db5653bca3fec306349e71aff14ae45ecc6d0951dd24/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:c0303e597eb5a5321b4de9c68e9845ac8f290d2ab3f3e2c864437d3c5a30cd65", size = 290465 },
-    { url = "https://files.pythonhosted.org/packages/bb/73/f89b7fbce8b0b0c095d82b008afd0590f71ccb3dee6eee41791cf8cd25fd/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:a47f2abb4e29b3a8d0b530f7c3598badc6b134562b1a5caee867f7c62fee51e3", size = 266385 },
-    { url = "https://files.pythonhosted.org/packages/cd/45/e365fdb554159462ca12df54bc59bfa7a9a273ecc21e99e72e597564d1ae/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:3d688126c242a6fabbd92e02633414d40f50bb6002fa4cf995a1d18051525657", size = 288771 },
-    { url = "https://files.pythonhosted.org/packages/00/11/47b6117002a0e904f004d70ec5194fe9144f117c33c851e3d51c765962d0/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4e7e9652b3d367c7bd449a727dc79d5043f48b88d0cbfd4f9f1060cf2b414104", size = 288206 },
-    { url = "https://files.pythonhosted.org/packages/40/37/5f9f3c3fd7f7746082ec67bcdc204db72dad081f4f83a503d33220a92973/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:1a85e345b4c43db8b842cab1feb41be5cc0b10a1830e6295b69d7310f99becaf", size = 282620 },
-    { url = "https://files.pythonhosted.org/packages/0b/31/8fbc5af2d183bff20f21aa743b4088eac4445d2bb1cdece449ae80e4e2d1/frozenlist-1.7.0-cp313-cp313t-win32.whl", hash = "sha256:3a14027124ddb70dfcee5148979998066897e79f89f64b13328595c4bdf77c81", size = 43059 },
-    { url = "https://files.pythonhosted.org/packages/bb/ed/41956f52105b8dbc26e457c5705340c67c8cc2b79f394b79bffc09d0e938/frozenlist-1.7.0-cp313-cp313t-win_amd64.whl", hash = "sha256:3bf8010d71d4507775f658e9823210b7427be36625b387221642725b515dcf3e", size = 47516 },
-    { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106 },
+sdist = { url = "https://files.pythonhosted.org/packages/79/b1/b64018016eeb087db503b038296fd782586432b9c077fc5c7839e9cb6ef6/frozenlist-1.7.0.tar.gz", hash = "sha256:2e310d81923c2437ea8670467121cc3e9b0f76d3043cc1d2331d56c7fb7a3a8f", size = 45078, upload-time = "2025-06-09T23:02:35.538Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/af/36/0da0a49409f6b47cc2d060dc8c9040b897b5902a8a4e37d9bc1deb11f680/frozenlist-1.7.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cc4df77d638aa2ed703b878dd093725b72a824c3c546c076e8fdf276f78ee84a", size = 81304, upload-time = "2025-06-09T22:59:46.226Z" },
+    { url = "https://files.pythonhosted.org/packages/77/f0/77c11d13d39513b298e267b22eb6cb559c103d56f155aa9a49097221f0b6/frozenlist-1.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:716a9973a2cc963160394f701964fe25012600f3d311f60c790400b00e568b61", size = 47735, upload-time = "2025-06-09T22:59:48.133Z" },
+    { url = "https://files.pythonhosted.org/packages/37/12/9d07fa18971a44150593de56b2f2947c46604819976784bcf6ea0d5db43b/frozenlist-1.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a0fd1bad056a3600047fb9462cff4c5322cebc59ebf5d0a3725e0ee78955001d", size = 46775, upload-time = "2025-06-09T22:59:49.564Z" },
+    { url = "https://files.pythonhosted.org/packages/70/34/f73539227e06288fcd1f8a76853e755b2b48bca6747e99e283111c18bcd4/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3789ebc19cb811163e70fe2bd354cea097254ce6e707ae42e56f45e31e96cb8e", size = 224644, upload-time = "2025-06-09T22:59:51.35Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/68/c1d9c2f4a6e438e14613bad0f2973567586610cc22dcb1e1241da71de9d3/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:af369aa35ee34f132fcfad5be45fbfcde0e3a5f6a1ec0712857f286b7d20cca9", size = 222125, upload-time = "2025-06-09T22:59:52.884Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/d0/98e8f9a515228d708344d7c6986752be3e3192d1795f748c24bcf154ad99/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac64b6478722eeb7a3313d494f8342ef3478dff539d17002f849101b212ef97c", size = 233455, upload-time = "2025-06-09T22:59:54.74Z" },
+    { url = "https://files.pythonhosted.org/packages/79/df/8a11bcec5600557f40338407d3e5bea80376ed1c01a6c0910fcfdc4b8993/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f89f65d85774f1797239693cef07ad4c97fdd0639544bad9ac4b869782eb1981", size = 227339, upload-time = "2025-06-09T22:59:56.187Z" },
+    { url = "https://files.pythonhosted.org/packages/50/82/41cb97d9c9a5ff94438c63cc343eb7980dac4187eb625a51bdfdb7707314/frozenlist-1.7.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1073557c941395fdfcfac13eb2456cb8aad89f9de27bae29fabca8e563b12615", size = 212969, upload-time = "2025-06-09T22:59:57.604Z" },
+    { url = "https://files.pythonhosted.org/packages/13/47/f9179ee5ee4f55629e4f28c660b3fdf2775c8bfde8f9c53f2de2d93f52a9/frozenlist-1.7.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ed8d2fa095aae4bdc7fdd80351009a48d286635edffee66bf865e37a9125c50", size = 222862, upload-time = "2025-06-09T22:59:59.498Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/52/df81e41ec6b953902c8b7e3a83bee48b195cb0e5ec2eabae5d8330c78038/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:24c34bea555fe42d9f928ba0a740c553088500377448febecaa82cc3e88aa1fa", size = 222492, upload-time = "2025-06-09T23:00:01.026Z" },
+    { url = "https://files.pythonhosted.org/packages/84/17/30d6ea87fa95a9408245a948604b82c1a4b8b3e153cea596421a2aef2754/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:69cac419ac6a6baad202c85aaf467b65ac860ac2e7f2ac1686dc40dbb52f6577", size = 238250, upload-time = "2025-06-09T23:00:03.401Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/00/ecbeb51669e3c3df76cf2ddd66ae3e48345ec213a55e3887d216eb4fbab3/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:960d67d0611f4c87da7e2ae2eacf7ea81a5be967861e0c63cf205215afbfac59", size = 218720, upload-time = "2025-06-09T23:00:05.282Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/c0/c224ce0e0eb31cc57f67742071bb470ba8246623c1823a7530be0e76164c/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:41be2964bd4b15bf575e5daee5a5ce7ed3115320fb3c2b71fca05582ffa4dc9e", size = 232585, upload-time = "2025-06-09T23:00:07.962Z" },
+    { url = "https://files.pythonhosted.org/packages/55/3c/34cb694abf532f31f365106deebdeac9e45c19304d83cf7d51ebbb4ca4d1/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:46d84d49e00c9429238a7ce02dc0be8f6d7cd0cd405abd1bebdc991bf27c15bd", size = 234248, upload-time = "2025-06-09T23:00:09.428Z" },
+    { url = "https://files.pythonhosted.org/packages/98/c0/2052d8b6cecda2e70bd81299e3512fa332abb6dcd2969b9c80dfcdddbf75/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:15900082e886edb37480335d9d518cec978afc69ccbc30bd18610b7c1b22a718", size = 221621, upload-time = "2025-06-09T23:00:11.32Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/bf/7dcebae315436903b1d98ffb791a09d674c88480c158aa171958a3ac07f0/frozenlist-1.7.0-cp310-cp310-win32.whl", hash = "sha256:400ddd24ab4e55014bba442d917203c73b2846391dd42ca5e38ff52bb18c3c5e", size = 39578, upload-time = "2025-06-09T23:00:13.526Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/5f/f69818f017fa9a3d24d1ae39763e29b7f60a59e46d5f91b9c6b21622f4cd/frozenlist-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:6eb93efb8101ef39d32d50bce242c84bcbddb4f7e9febfa7b524532a239b4464", size = 43830, upload-time = "2025-06-09T23:00:14.98Z" },
+    { url = "https://files.pythonhosted.org/packages/34/7e/803dde33760128acd393a27eb002f2020ddb8d99d30a44bfbaab31c5f08a/frozenlist-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa51e147a66b2d74de1e6e2cf5921890de6b0f4820b257465101d7f37b49fb5a", size = 82251, upload-time = "2025-06-09T23:00:16.279Z" },
+    { url = "https://files.pythonhosted.org/packages/75/a9/9c2c5760b6ba45eae11334db454c189d43d34a4c0b489feb2175e5e64277/frozenlist-1.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9b35db7ce1cd71d36ba24f80f0c9e7cff73a28d7a74e91fe83e23d27c7828750", size = 48183, upload-time = "2025-06-09T23:00:17.698Z" },
+    { url = "https://files.pythonhosted.org/packages/47/be/4038e2d869f8a2da165f35a6befb9158c259819be22eeaf9c9a8f6a87771/frozenlist-1.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:34a69a85e34ff37791e94542065c8416c1afbf820b68f720452f636d5fb990cd", size = 47107, upload-time = "2025-06-09T23:00:18.952Z" },
+    { url = "https://files.pythonhosted.org/packages/79/26/85314b8a83187c76a37183ceed886381a5f992975786f883472fcb6dc5f2/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a646531fa8d82c87fe4bb2e596f23173caec9185bfbca5d583b4ccfb95183e2", size = 237333, upload-time = "2025-06-09T23:00:20.275Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/fd/e5b64f7d2c92a41639ffb2ad44a6a82f347787abc0c7df5f49057cf11770/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:79b2ffbba483f4ed36a0f236ccb85fbb16e670c9238313709638167670ba235f", size = 231724, upload-time = "2025-06-09T23:00:21.705Z" },
+    { url = "https://files.pythonhosted.org/packages/20/fb/03395c0a43a5976af4bf7534759d214405fbbb4c114683f434dfdd3128ef/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a26f205c9ca5829cbf82bb2a84b5c36f7184c4316617d7ef1b271a56720d6b30", size = 245842, upload-time = "2025-06-09T23:00:23.148Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/15/c01c8e1dffdac5d9803507d824f27aed2ba76b6ed0026fab4d9866e82f1f/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bcacfad3185a623fa11ea0e0634aac7b691aa925d50a440f39b458e41c561d98", size = 239767, upload-time = "2025-06-09T23:00:25.103Z" },
+    { url = "https://files.pythonhosted.org/packages/14/99/3f4c6fe882c1f5514b6848aa0a69b20cb5e5d8e8f51a339d48c0e9305ed0/frozenlist-1.7.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72c1b0fe8fe451b34f12dce46445ddf14bd2a5bcad7e324987194dc8e3a74c86", size = 224130, upload-time = "2025-06-09T23:00:27.061Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/83/220a374bd7b2aeba9d0725130665afe11de347d95c3620b9b82cc2fcab97/frozenlist-1.7.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61d1a5baeaac6c0798ff6edfaeaa00e0e412d49946c53fae8d4b8e8b3566c4ae", size = 235301, upload-time = "2025-06-09T23:00:29.02Z" },
+    { url = "https://files.pythonhosted.org/packages/03/3c/3e3390d75334a063181625343e8daab61b77e1b8214802cc4e8a1bb678fc/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7edf5c043c062462f09b6820de9854bf28cc6cc5b6714b383149745e287181a8", size = 234606, upload-time = "2025-06-09T23:00:30.514Z" },
+    { url = "https://files.pythonhosted.org/packages/23/1e/58232c19608b7a549d72d9903005e2d82488f12554a32de2d5fb59b9b1ba/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d50ac7627b3a1bd2dcef6f9da89a772694ec04d9a61b66cf87f7d9446b4a0c31", size = 248372, upload-time = "2025-06-09T23:00:31.966Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/a4/e4a567e01702a88a74ce8a324691e62a629bf47d4f8607f24bf1c7216e7f/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ce48b2fece5aeb45265bb7a58259f45027db0abff478e3077e12b05b17fb9da7", size = 229860, upload-time = "2025-06-09T23:00:33.375Z" },
+    { url = "https://files.pythonhosted.org/packages/73/a6/63b3374f7d22268b41a9db73d68a8233afa30ed164c46107b33c4d18ecdd/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:fe2365ae915a1fafd982c146754e1de6ab3478def8a59c86e1f7242d794f97d5", size = 245893, upload-time = "2025-06-09T23:00:35.002Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/eb/d18b3f6e64799a79673c4ba0b45e4cfbe49c240edfd03a68be20002eaeaa/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:45a6f2fdbd10e074e8814eb98b05292f27bad7d1883afbe009d96abdcf3bc898", size = 246323, upload-time = "2025-06-09T23:00:36.468Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/f5/720f3812e3d06cd89a1d5db9ff6450088b8f5c449dae8ffb2971a44da506/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:21884e23cffabb157a9dd7e353779077bf5b8f9a58e9b262c6caad2ef5f80a56", size = 233149, upload-time = "2025-06-09T23:00:37.963Z" },
+    { url = "https://files.pythonhosted.org/packages/69/68/03efbf545e217d5db8446acfd4c447c15b7c8cf4dbd4a58403111df9322d/frozenlist-1.7.0-cp311-cp311-win32.whl", hash = "sha256:284d233a8953d7b24f9159b8a3496fc1ddc00f4db99c324bd5fb5f22d8698ea7", size = 39565, upload-time = "2025-06-09T23:00:39.753Z" },
+    { url = "https://files.pythonhosted.org/packages/58/17/fe61124c5c333ae87f09bb67186d65038834a47d974fc10a5fadb4cc5ae1/frozenlist-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:387cbfdcde2f2353f19c2f66bbb52406d06ed77519ac7ee21be0232147c2592d", size = 44019, upload-time = "2025-06-09T23:00:40.988Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/a2/c8131383f1e66adad5f6ecfcce383d584ca94055a34d683bbb24ac5f2f1c/frozenlist-1.7.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3dbf9952c4bb0e90e98aec1bd992b3318685005702656bc6f67c1a32b76787f2", size = 81424, upload-time = "2025-06-09T23:00:42.24Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/9d/02754159955088cb52567337d1113f945b9e444c4960771ea90eb73de8db/frozenlist-1.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1f5906d3359300b8a9bb194239491122e6cf1444c2efb88865426f170c262cdb", size = 47952, upload-time = "2025-06-09T23:00:43.481Z" },
+    { url = "https://files.pythonhosted.org/packages/01/7a/0046ef1bd6699b40acd2067ed6d6670b4db2f425c56980fa21c982c2a9db/frozenlist-1.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3dabd5a8f84573c8d10d8859a50ea2dec01eea372031929871368c09fa103478", size = 46688, upload-time = "2025-06-09T23:00:44.793Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/a2/a910bafe29c86997363fb4c02069df4ff0b5bc39d33c5198b4e9dd42d8f8/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa57daa5917f1738064f302bf2626281a1cb01920c32f711fbc7bc36111058a8", size = 243084, upload-time = "2025-06-09T23:00:46.125Z" },
+    { url = "https://files.pythonhosted.org/packages/64/3e/5036af9d5031374c64c387469bfcc3af537fc0f5b1187d83a1cf6fab1639/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c193dda2b6d49f4c4398962810fa7d7c78f032bf45572b3e04dd5249dff27e08", size = 233524, upload-time = "2025-06-09T23:00:47.73Z" },
+    { url = "https://files.pythonhosted.org/packages/06/39/6a17b7c107a2887e781a48ecf20ad20f1c39d94b2a548c83615b5b879f28/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe2b675cf0aaa6d61bf8fbffd3c274b3c9b7b1623beb3809df8a81399a4a9c4", size = 248493, upload-time = "2025-06-09T23:00:49.742Z" },
+    { url = "https://files.pythonhosted.org/packages/be/00/711d1337c7327d88c44d91dd0f556a1c47fb99afc060ae0ef66b4d24793d/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fc5d5cda37f62b262405cf9652cf0856839c4be8ee41be0afe8858f17f4c94b", size = 244116, upload-time = "2025-06-09T23:00:51.352Z" },
+    { url = "https://files.pythonhosted.org/packages/24/fe/74e6ec0639c115df13d5850e75722750adabdc7de24e37e05a40527ca539/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0d5ce521d1dd7d620198829b87ea002956e4319002ef0bc8d3e6d045cb4646e", size = 224557, upload-time = "2025-06-09T23:00:52.855Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/db/48421f62a6f77c553575201e89048e97198046b793f4a089c79a6e3268bd/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:488d0a7d6a0008ca0db273c542098a0fa9e7dfaa7e57f70acef43f32b3f69dca", size = 241820, upload-time = "2025-06-09T23:00:54.43Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/fa/cb4a76bea23047c8462976ea7b7a2bf53997a0ca171302deae9d6dd12096/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:15a7eaba63983d22c54d255b854e8108e7e5f3e89f647fc854bd77a237e767df", size = 236542, upload-time = "2025-06-09T23:00:56.409Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/32/476a4b5cfaa0ec94d3f808f193301debff2ea42288a099afe60757ef6282/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1eaa7e9c6d15df825bf255649e05bd8a74b04a4d2baa1ae46d9c2d00b2ca2cb5", size = 249350, upload-time = "2025-06-09T23:00:58.468Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/ba/9a28042f84a6bf8ea5dbc81cfff8eaef18d78b2a1ad9d51c7bc5b029ad16/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4389e06714cfa9d47ab87f784a7c5be91d3934cd6e9a7b85beef808297cc025", size = 225093, upload-time = "2025-06-09T23:01:00.015Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/29/3a32959e68f9cf000b04e79ba574527c17e8842e38c91d68214a37455786/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:73bd45e1488c40b63fe5a7df892baf9e2a4d4bb6409a2b3b78ac1c6236178e01", size = 245482, upload-time = "2025-06-09T23:01:01.474Z" },
+    { url = "https://files.pythonhosted.org/packages/80/e8/edf2f9e00da553f07f5fa165325cfc302dead715cab6ac8336a5f3d0adc2/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99886d98e1643269760e5fe0df31e5ae7050788dd288947f7f007209b8c33f08", size = 249590, upload-time = "2025-06-09T23:01:02.961Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/80/9a0eb48b944050f94cc51ee1c413eb14a39543cc4f760ed12657a5a3c45a/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:290a172aae5a4c278c6da8a96222e6337744cd9c77313efe33d5670b9f65fc43", size = 237785, upload-time = "2025-06-09T23:01:05.095Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/74/87601e0fb0369b7a2baf404ea921769c53b7ae00dee7dcfe5162c8c6dbf0/frozenlist-1.7.0-cp312-cp312-win32.whl", hash = "sha256:426c7bc70e07cfebc178bc4c2bf2d861d720c4fff172181eeb4a4c41d4ca2ad3", size = 39487, upload-time = "2025-06-09T23:01:06.54Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/15/c026e9a9fc17585a9d461f65d8593d281fedf55fbf7eb53f16c6df2392f9/frozenlist-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:563b72efe5da92e02eb68c59cb37205457c977aa7a449ed1b37e6939e5c47c6a", size = 43874, upload-time = "2025-06-09T23:01:07.752Z" },
+    { url = "https://files.pythonhosted.org/packages/24/90/6b2cebdabdbd50367273c20ff6b57a3dfa89bd0762de02c3a1eb42cb6462/frozenlist-1.7.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee80eeda5e2a4e660651370ebffd1286542b67e268aa1ac8d6dbe973120ef7ee", size = 79791, upload-time = "2025-06-09T23:01:09.368Z" },
+    { url = "https://files.pythonhosted.org/packages/83/2e/5b70b6a3325363293fe5fc3ae74cdcbc3e996c2a11dde2fd9f1fb0776d19/frozenlist-1.7.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d1a81c85417b914139e3a9b995d4a1c84559afc839a93cf2cb7f15e6e5f6ed2d", size = 47165, upload-time = "2025-06-09T23:01:10.653Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/25/a0895c99270ca6966110f4ad98e87e5662eab416a17e7fd53c364bf8b954/frozenlist-1.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cbb65198a9132ebc334f237d7b0df163e4de83fb4f2bdfe46c1e654bdb0c5d43", size = 45881, upload-time = "2025-06-09T23:01:12.296Z" },
+    { url = "https://files.pythonhosted.org/packages/19/7c/71bb0bbe0832793c601fff68cd0cf6143753d0c667f9aec93d3c323f4b55/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dab46c723eeb2c255a64f9dc05b8dd601fde66d6b19cdb82b2e09cc6ff8d8b5d", size = 232409, upload-time = "2025-06-09T23:01:13.641Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/45/ed2798718910fe6eb3ba574082aaceff4528e6323f9a8570be0f7028d8e9/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6aeac207a759d0dedd2e40745575ae32ab30926ff4fa49b1635def65806fddee", size = 225132, upload-time = "2025-06-09T23:01:15.264Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/e2/8417ae0f8eacb1d071d4950f32f229aa6bf68ab69aab797b72a07ea68d4f/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bd8c4e58ad14b4fa7802b8be49d47993182fdd4023393899632c88fd8cd994eb", size = 237638, upload-time = "2025-06-09T23:01:16.752Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/b7/2ace5450ce85f2af05a871b8c8719b341294775a0a6c5585d5e6170f2ce7/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04fb24d104f425da3540ed83cbfc31388a586a7696142004c577fa61c6298c3f", size = 233539, upload-time = "2025-06-09T23:01:18.202Z" },
+    { url = "https://files.pythonhosted.org/packages/46/b9/6989292c5539553dba63f3c83dc4598186ab2888f67c0dc1d917e6887db6/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a5c505156368e4ea6b53b5ac23c92d7edc864537ff911d2fb24c140bb175e60", size = 215646, upload-time = "2025-06-09T23:01:19.649Z" },
+    { url = "https://files.pythonhosted.org/packages/72/31/bc8c5c99c7818293458fe745dab4fd5730ff49697ccc82b554eb69f16a24/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bd7eb96a675f18aa5c553eb7ddc24a43c8c18f22e1f9925528128c052cdbe00", size = 232233, upload-time = "2025-06-09T23:01:21.175Z" },
+    { url = "https://files.pythonhosted.org/packages/59/52/460db4d7ba0811b9ccb85af996019f5d70831f2f5f255f7cc61f86199795/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:05579bf020096fe05a764f1f84cd104a12f78eaab68842d036772dc6d4870b4b", size = 227996, upload-time = "2025-06-09T23:01:23.098Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/c9/f4b39e904c03927b7ecf891804fd3b4df3db29b9e487c6418e37988d6e9d/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:376b6222d114e97eeec13d46c486facd41d4f43bab626b7c3f6a8b4e81a5192c", size = 242280, upload-time = "2025-06-09T23:01:24.808Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/33/3f8d6ced42f162d743e3517781566b8481322be321b486d9d262adf70bfb/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0aa7e176ebe115379b5b1c95b4096fb1c17cce0847402e227e712c27bdb5a949", size = 217717, upload-time = "2025-06-09T23:01:26.28Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/e8/ad683e75da6ccef50d0ab0c2b2324b32f84fc88ceee778ed79b8e2d2fe2e/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3fbba20e662b9c2130dc771e332a99eff5da078b2b2648153a40669a6d0e36ca", size = 236644, upload-time = "2025-06-09T23:01:27.887Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/14/8d19ccdd3799310722195a72ac94ddc677541fb4bef4091d8e7775752360/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f3f4410a0a601d349dd406b5713fec59b4cee7e71678d5b17edda7f4655a940b", size = 238879, upload-time = "2025-06-09T23:01:29.524Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/13/c12bf657494c2fd1079a48b2db49fa4196325909249a52d8f09bc9123fd7/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e2cdfaaec6a2f9327bf43c933c0319a7c429058e8537c508964a133dffee412e", size = 232502, upload-time = "2025-06-09T23:01:31.287Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/8b/e7f9dfde869825489382bc0d512c15e96d3964180c9499efcec72e85db7e/frozenlist-1.7.0-cp313-cp313-win32.whl", hash = "sha256:5fc4df05a6591c7768459caba1b342d9ec23fa16195e744939ba5914596ae3e1", size = 39169, upload-time = "2025-06-09T23:01:35.503Z" },
+    { url = "https://files.pythonhosted.org/packages/35/89/a487a98d94205d85745080a37860ff5744b9820a2c9acbcdd9440bfddf98/frozenlist-1.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:52109052b9791a3e6b5d1b65f4b909703984b770694d3eb64fad124c835d7cba", size = 43219, upload-time = "2025-06-09T23:01:36.784Z" },
+    { url = "https://files.pythonhosted.org/packages/56/d5/5c4cf2319a49eddd9dd7145e66c4866bdc6f3dbc67ca3d59685149c11e0d/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a6f86e4193bb0e235ef6ce3dde5cbabed887e0b11f516ce8a0f4d3b33078ec2d", size = 84345, upload-time = "2025-06-09T23:01:38.295Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/7d/ec2c1e1dc16b85bc9d526009961953df9cec8481b6886debb36ec9107799/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:82d664628865abeb32d90ae497fb93df398a69bb3434463d172b80fc25b0dd7d", size = 48880, upload-time = "2025-06-09T23:01:39.887Z" },
+    { url = "https://files.pythonhosted.org/packages/69/86/f9596807b03de126e11e7d42ac91e3d0b19a6599c714a1989a4e85eeefc4/frozenlist-1.7.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:912a7e8375a1c9a68325a902f3953191b7b292aa3c3fb0d71a216221deca460b", size = 48498, upload-time = "2025-06-09T23:01:41.318Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/cb/df6de220f5036001005f2d726b789b2c0b65f2363b104bbc16f5be8084f8/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9537c2777167488d539bc5de2ad262efc44388230e5118868e172dd4a552b146", size = 292296, upload-time = "2025-06-09T23:01:42.685Z" },
+    { url = "https://files.pythonhosted.org/packages/83/1f/de84c642f17c8f851a2905cee2dae401e5e0daca9b5ef121e120e19aa825/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f34560fb1b4c3e30ba35fa9a13894ba39e5acfc5f60f57d8accde65f46cc5e74", size = 273103, upload-time = "2025-06-09T23:01:44.166Z" },
+    { url = "https://files.pythonhosted.org/packages/88/3c/c840bfa474ba3fa13c772b93070893c6e9d5c0350885760376cbe3b6c1b3/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:acd03d224b0175f5a850edc104ac19040d35419eddad04e7cf2d5986d98427f1", size = 292869, upload-time = "2025-06-09T23:01:45.681Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/1c/3efa6e7d5a39a1d5ef0abeb51c48fb657765794a46cf124e5aca2c7a592c/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2038310bc582f3d6a09b3816ab01737d60bf7b1ec70f5356b09e84fb7408ab1", size = 291467, upload-time = "2025-06-09T23:01:47.234Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/00/d5c5e09d4922c395e2f2f6b79b9a20dab4b67daaf78ab92e7729341f61f6/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8c05e4c8e5f36e5e088caa1bf78a687528f83c043706640a92cb76cd6999384", size = 266028, upload-time = "2025-06-09T23:01:48.819Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/27/72765be905619dfde25a7f33813ac0341eb6b076abede17a2e3fbfade0cb/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:765bb588c86e47d0b68f23c1bee323d4b703218037765dcf3f25c838c6fecceb", size = 284294, upload-time = "2025-06-09T23:01:50.394Z" },
+    { url = "https://files.pythonhosted.org/packages/88/67/c94103a23001b17808eb7dd1200c156bb69fb68e63fcf0693dde4cd6228c/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:32dc2e08c67d86d0969714dd484fd60ff08ff81d1a1e40a77dd34a387e6ebc0c", size = 281898, upload-time = "2025-06-09T23:01:52.234Z" },
+    { url = "https://files.pythonhosted.org/packages/42/34/a3e2c00c00f9e2a9db5653bca3fec306349e71aff14ae45ecc6d0951dd24/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:c0303e597eb5a5321b4de9c68e9845ac8f290d2ab3f3e2c864437d3c5a30cd65", size = 290465, upload-time = "2025-06-09T23:01:53.788Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/73/f89b7fbce8b0b0c095d82b008afd0590f71ccb3dee6eee41791cf8cd25fd/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:a47f2abb4e29b3a8d0b530f7c3598badc6b134562b1a5caee867f7c62fee51e3", size = 266385, upload-time = "2025-06-09T23:01:55.769Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/45/e365fdb554159462ca12df54bc59bfa7a9a273ecc21e99e72e597564d1ae/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:3d688126c242a6fabbd92e02633414d40f50bb6002fa4cf995a1d18051525657", size = 288771, upload-time = "2025-06-09T23:01:57.4Z" },
+    { url = "https://files.pythonhosted.org/packages/00/11/47b6117002a0e904f004d70ec5194fe9144f117c33c851e3d51c765962d0/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4e7e9652b3d367c7bd449a727dc79d5043f48b88d0cbfd4f9f1060cf2b414104", size = 288206, upload-time = "2025-06-09T23:01:58.936Z" },
+    { url = "https://files.pythonhosted.org/packages/40/37/5f9f3c3fd7f7746082ec67bcdc204db72dad081f4f83a503d33220a92973/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:1a85e345b4c43db8b842cab1feb41be5cc0b10a1830e6295b69d7310f99becaf", size = 282620, upload-time = "2025-06-09T23:02:00.493Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/31/8fbc5af2d183bff20f21aa743b4088eac4445d2bb1cdece449ae80e4e2d1/frozenlist-1.7.0-cp313-cp313t-win32.whl", hash = "sha256:3a14027124ddb70dfcee5148979998066897e79f89f64b13328595c4bdf77c81", size = 43059, upload-time = "2025-06-09T23:02:02.072Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/ed/41956f52105b8dbc26e457c5705340c67c8cc2b79f394b79bffc09d0e938/frozenlist-1.7.0-cp313-cp313t-win_amd64.whl", hash = "sha256:3bf8010d71d4507775f658e9823210b7427be36625b387221642725b515dcf3e", size = 47516, upload-time = "2025-06-09T23:02:03.779Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/b1/ee59496f51cd244039330015d60f13ce5a54a0f2bd8d79e4a4a375ab7469/frozenlist-1.7.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cea3dbd15aea1341ea2de490574a4a37ca080b2ae24e4b4f4b51b9057b4c3630", size = 82434, upload-time = "2025-06-09T23:02:05.195Z" },
+    { url = "https://files.pythonhosted.org/packages/75/e1/d518391ce36a6279b3fa5bc14327dde80bcb646bb50d059c6ca0756b8d05/frozenlist-1.7.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7d536ee086b23fecc36c2073c371572374ff50ef4db515e4e503925361c24f71", size = 48232, upload-time = "2025-06-09T23:02:07.728Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/8d/a0d04f28b6e821a9685c22e67b5fb798a5a7b68752f104bfbc2dccf080c4/frozenlist-1.7.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dfcebf56f703cb2e346315431699f00db126d158455e513bd14089d992101e44", size = 47186, upload-time = "2025-06-09T23:02:09.243Z" },
+    { url = "https://files.pythonhosted.org/packages/93/3a/a5334c0535c8b7c78eeabda1579179e44fe3d644e07118e59a2276dedaf1/frozenlist-1.7.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:974c5336e61d6e7eb1ea5b929cb645e882aadab0095c5a6974a111e6479f8878", size = 226617, upload-time = "2025-06-09T23:02:10.949Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/67/8258d971f519dc3f278c55069a775096cda6610a267b53f6248152b72b2f/frozenlist-1.7.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c70db4a0ab5ab20878432c40563573229a7ed9241506181bba12f6b7d0dc41cb", size = 224179, upload-time = "2025-06-09T23:02:12.603Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/89/8225905bf889b97c6d935dd3aeb45668461e59d415cb019619383a8a7c3b/frozenlist-1.7.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1137b78384eebaf70560a36b7b229f752fb64d463d38d1304939984d5cb887b6", size = 235783, upload-time = "2025-06-09T23:02:14.678Z" },
+    { url = "https://files.pythonhosted.org/packages/54/6e/ef52375aa93d4bc510d061df06205fa6dcfd94cd631dd22956b09128f0d4/frozenlist-1.7.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e793a9f01b3e8b5c0bc646fb59140ce0efcc580d22a3468d70766091beb81b35", size = 229210, upload-time = "2025-06-09T23:02:16.313Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/55/62c87d1a6547bfbcd645df10432c129100c5bd0fd92a384de6e3378b07c1/frozenlist-1.7.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74739ba8e4e38221d2c5c03d90a7e542cb8ad681915f4ca8f68d04f810ee0a87", size = 215994, upload-time = "2025-06-09T23:02:17.9Z" },
+    { url = "https://files.pythonhosted.org/packages/45/d2/263fea1f658b8ad648c7d94d18a87bca7e8c67bd6a1bbf5445b1bd5b158c/frozenlist-1.7.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e63344c4e929b1a01e29bc184bbb5fd82954869033765bfe8d65d09e336a677", size = 225122, upload-time = "2025-06-09T23:02:19.479Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/22/7145e35d12fb368d92124f679bea87309495e2e9ddf14c6533990cb69218/frozenlist-1.7.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2ea2a7369eb76de2217a842f22087913cdf75f63cf1307b9024ab82dfb525938", size = 224019, upload-time = "2025-06-09T23:02:20.969Z" },
+    { url = "https://files.pythonhosted.org/packages/44/1e/7dae8c54301beb87bcafc6144b9a103bfd2c8f38078c7902984c9a0c4e5b/frozenlist-1.7.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:836b42f472a0e006e02499cef9352ce8097f33df43baaba3e0a28a964c26c7d2", size = 239925, upload-time = "2025-06-09T23:02:22.466Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/1e/99c93e54aa382e949a98976a73b9b20c3aae6d9d893f31bbe4991f64e3a8/frozenlist-1.7.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e22b9a99741294b2571667c07d9f8cceec07cb92aae5ccda39ea1b6052ed4319", size = 220881, upload-time = "2025-06-09T23:02:24.521Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/9c/ca5105fa7fb5abdfa8837581be790447ae051da75d32f25c8f81082ffc45/frozenlist-1.7.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:9a19e85cc503d958abe5218953df722748d87172f71b73cf3c9257a91b999890", size = 234046, upload-time = "2025-06-09T23:02:26.206Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/4d/e99014756093b4ddbb67fb8f0df11fe7a415760d69ace98e2ac6d5d43402/frozenlist-1.7.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f22dac33bb3ee8fe3e013aa7b91dc12f60d61d05b7fe32191ffa84c3aafe77bd", size = 235756, upload-time = "2025-06-09T23:02:27.79Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/72/a19a40bcdaa28a51add2aaa3a1a294ec357f36f27bd836a012e070c5e8a5/frozenlist-1.7.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9ccec739a99e4ccf664ea0775149f2749b8a6418eb5b8384b4dc0a7d15d304cb", size = 222894, upload-time = "2025-06-09T23:02:29.848Z" },
+    { url = "https://files.pythonhosted.org/packages/08/49/0042469993e023a758af81db68c76907cd29e847d772334d4d201cbe9a42/frozenlist-1.7.0-cp39-cp39-win32.whl", hash = "sha256:b3950f11058310008a87757f3eee16a8e1ca97979833239439586857bc25482e", size = 39848, upload-time = "2025-06-09T23:02:31.413Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/45/827d86ee475c877f5f766fbc23fb6acb6fada9e52f1c9720e2ba3eae32da/frozenlist-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:43a82fce6769c70f2f5a06248b614a7d268080a9d20f7457ef10ecee5af82b63", size = 44102, upload-time = "2025-06-09T23:02:32.808Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" },
 ]
 
 [[package]]
 name = "fsspec"
 version = "2025.9.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/de/e0/bab50af11c2d75c9c4a2a26a5254573c0bd97cea152254401510950486fa/fsspec-2025.9.0.tar.gz", hash = "sha256:19fd429483d25d28b65ec68f9f4adc16c17ea2c7c7bf54ec61360d478fb19c19", size = 304847 }
+sdist = { url = "https://files.pythonhosted.org/packages/de/e0/bab50af11c2d75c9c4a2a26a5254573c0bd97cea152254401510950486fa/fsspec-2025.9.0.tar.gz", hash = "sha256:19fd429483d25d28b65ec68f9f4adc16c17ea2c7c7bf54ec61360d478fb19c19", size = 304847, upload-time = "2025-09-02T19:10:49.215Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/47/71/70db47e4f6ce3e5c37a607355f80da8860a33226be640226ac52cb05ef2e/fsspec-2025.9.0-py3-none-any.whl", hash = "sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7", size = 199289 },
+    { url = "https://files.pythonhosted.org/packages/47/71/70db47e4f6ce3e5c37a607355f80da8860a33226be640226ac52cb05ef2e/fsspec-2025.9.0-py3-none-any.whl", hash = "sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7", size = 199289, upload-time = "2025-09-02T19:10:47.708Z" },
 ]
 
 [package.optional-dependencies]
 http = [
-    { name = "aiohttp" },
+    { name = "aiohttp", marker = "python_full_version >= '3.10'" },
+]
+
+[[package]]
+name = "gast"
+version = "0.4.0"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/83/4a/07c7e59cef23fb147454663c3271c21da68ba2ab141427c20548ae5a8a4d/gast-0.4.0.tar.gz", hash = "sha256:40feb7b8b8434785585ab224d1568b857edb18297e5a3047f1ba012bc83b42c1", size = 13804, upload-time = "2020-08-07T21:45:23.526Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b6/48/583c032b79ae5b3daa02225a675aeb673e58d2cb698e78510feceb11958c/gast-0.4.0-py3-none-any.whl", hash = "sha256:b7adcdd5adbebf1adf17378da5ba3f543684dbec47b1cda1f3997e573cd542c4", size = 9824, upload-time = "2020-08-07T21:45:21.32Z" },
 ]
 
 [[package]]
 name = "gast"
 version = "0.6.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/3c/14/c566f5ca00c115db7725263408ff952b8ae6d6a4e792ef9c84e77d9af7a1/gast-0.6.0.tar.gz", hash = "sha256:88fc5300d32c7ac6ca7b515310862f71e6fdf2c029bbec7c66c0f5dd47b6b1fb", size = 27708 }
+resolution-markers = [
+    "python_full_version >= '3.14'",
+    "python_full_version == '3.13.*'",
+    "python_full_version == '3.12.*'",
+    "python_full_version == '3.11.*'",
+    "python_full_version == '3.10.*'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3c/14/c566f5ca00c115db7725263408ff952b8ae6d6a4e792ef9c84e77d9af7a1/gast-0.6.0.tar.gz", hash = "sha256:88fc5300d32c7ac6ca7b515310862f71e6fdf2c029bbec7c66c0f5dd47b6b1fb", size = 27708, upload-time = "2024-06-27T20:31:49.527Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a3/61/8001b38461d751cd1a0c3a6ae84346796a5758123f3ed97a1b121dfbf4f3/gast-0.6.0-py3-none-any.whl", hash = "sha256:52b182313f7330389f72b069ba00f174cfe2a06411099547288839c6cbafbd54", size = 21173 },
+    { url = "https://files.pythonhosted.org/packages/a3/61/8001b38461d751cd1a0c3a6ae84346796a5758123f3ed97a1b121dfbf4f3/gast-0.6.0-py3-none-any.whl", hash = "sha256:52b182313f7330389f72b069ba00f174cfe2a06411099547288839c6cbafbd54", size = 21173, upload-time = "2024-07-09T13:15:15.615Z" },
 ]
 
 [[package]]
@@ -575,55 +730,65 @@ version = "0.6.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "arro3-core" },
-    { name = "pyproj", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "pyproj", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "pyproj", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "pyproj", version = "3.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/77/2d/3e994dd76223fac0eb597a6f55647cca51bd5a4f446d09b668697f901724/geoarrow_rust_core-0.6.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:84d972cc3dd45a797fd99588d7ee68f257e4083ebdcecad9ec773260067f71a6", size = 3570129 },
-    { url = "https://files.pythonhosted.org/packages/5f/2a/e19df203b4ffb225f39627e1bd1b89ce7b2220e39f1d6972692174820c57/geoarrow_rust_core-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bc0f382d4ed41e85d2d89fc2c7c8c3d046681c9a5e19350ce79e0e930cf69821", size = 3333881 },
-    { url = "https://files.pythonhosted.org/packages/52/98/b749a2165dfc5d9c54a1c19eb3e6a75b6d005ecde42289b25c1c355346b7/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80e719edcaf6698ed2b1aa9525bd97cf79e23a500a39b1e83566cd9a16a294d3", size = 3806366 },
-    { url = "https://files.pythonhosted.org/packages/84/93/7c0e42ba7d46208fb0f851e06c05de071962170f3a3b2a2260d8a3f66e7a/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d0f3546a15503329880063aca31266b301b0b781f618f832585bcd1c9efcc876", size = 3981800 },
-    { url = "https://files.pythonhosted.org/packages/de/43/9c5736569dead60b33e46b7c485e24804d950693df70dee306e153547789/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6937f3cabebf673f8b726d60d8ca160b46401de8b08c8e257be22772c12c2001", size = 5068955 },
-    { url = "https://files.pythonhosted.org/packages/71/5e/f26f9bea2af96b0d070e980dcc2196d369a678e06141ed260de5ca72bcc2/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f29ba92053e8ad4bd60d72188518f033ca4abc1f34eecebeb41ee7b790612e00", size = 4104946 },
-    { url = "https://files.pythonhosted.org/packages/fa/08/473796b3e0c03b35292220de88c8efa3e74d6174e807b26a371f2523a4b0/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14a5d05a312fbb76821566b1d144c64d0923fcbd790b2c7376ee11f62472b2fe", size = 3917533 },
-    { url = "https://files.pythonhosted.org/packages/b9/7a/7b62b839c3a9878a7d91b8395e0b7b04483e4bec687e073df0fbd4056583/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:88fe8fd33b16a06e9b3b7638b51d24047f1d01af12cc2e3e2653140877bddef6", size = 4318837 },
-    { url = "https://files.pythonhosted.org/packages/ea/86/309c55a9c63f316e3a04949ade8847b8e5acbdd21645696911175f0e1814/geoarrow_rust_core-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:dbecc2487cc95526ac77797cd70c199e196811b0a9e877c1b61fcaca508575fa", size = 3320081 },
-    { url = "https://files.pythonhosted.org/packages/1a/ed/514cff089185d71242a62e774e2c59dda147baab65929851b66d72198d5d/geoarrow_rust_core-0.6.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:e26ca240d7a6a0fa1b4f56a9ebe07b2e14fc7c1c9507aa862bd31ef14e0521f0", size = 3572326 },
-    { url = "https://files.pythonhosted.org/packages/77/21/22f8233235bd020db22b4f2bf888f9aeed08813eda7b8b421a6963bdc7e4/geoarrow_rust_core-0.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:46876e3528685673e08b4cbc696dca7f22fb073a83318708b0eaf640107b923b", size = 3335166 },
-    { url = "https://files.pythonhosted.org/packages/bb/eb/0c2e40a6a1bd450347a8a9fc7648ca840710bc177ff6eed3fc5da6ef981a/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5502bd12ede712d9b4725753df4db231a0aa6d3e131079bc4b6452c436e37b7", size = 3800540 },
-    { url = "https://files.pythonhosted.org/packages/4c/42/22d3b8441bb7041a6fcdb4cf0a1108e150513a52f8a407715188412bc71f/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f04dd7dd03449dba6d15f7d35c6c708637ac05f125638f56206e876756cd4c5", size = 3984840 },
-    { url = "https://files.pythonhosted.org/packages/12/44/477b6b2389398dc983026a4ab7dbb7ec121284ad5fb864a1b7a4658c3881/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d2afce33d0c3fa87d5d4d24d6617732e4297da3372b1746569b759f9b62aede1", size = 5067358 },
-    { url = "https://files.pythonhosted.org/packages/62/50/6995e9d11462635972b2fc09c8e1e510928563ca4fb0fd2c9145cf6ef771/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e63cdb661652a9836dc86cb5995ad269817d88b80f4cce6ed236a7f80f0aba", size = 4105773 },
-    { url = "https://files.pythonhosted.org/packages/a3/21/b369208495f213db0a0e7d563358307a706cc6af0cb9c897dacf28ae06a1/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:adbaf97cb770aef69df8a16437c9faa67adb2b04856faf45bcb61d5b986101dc", size = 3914659 },
-    { url = "https://files.pythonhosted.org/packages/1d/49/fccb14c6ee9bb715451e4d5bbe3d571eb59a8a1abe21b2abe0d9d48a7fac/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:202f35b301caa5154d95fd74424a1ef6449306e4f6fbfb5140270e48e94188a5", size = 4315153 },
-    { url = "https://files.pythonhosted.org/packages/c0/1c/88b16510e24a4a3332284669085673701b9fe4d6a511b4466c90655a9daf/geoarrow_rust_core-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:491405dfcc821a2c599e381cc9923e04a758deb1cc84fdb5794b519446c2f8a8", size = 3320510 },
-    { url = "https://files.pythonhosted.org/packages/cb/5f/1dbdbc1dde2140937cff20188cb25034b6f39e1734c14ca6510cf464bf77/geoarrow_rust_core-0.6.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a8145a562e94419402dd0882bb62429853804c53d47dbea944f2a24abc57abd2", size = 3568115 },
-    { url = "https://files.pythonhosted.org/packages/fd/e1/b62676f89ef3b866676967989ee8dbbd3d16c77f69aa4287825703268c42/geoarrow_rust_core-0.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:51040a5afcfa0cd3ab372d981375c7fe8eb652d155e3964d52ed51d14faa04e8", size = 3325336 },
-    { url = "https://files.pythonhosted.org/packages/1f/89/94e20f255712ff0eaccf9bfeac4bf51953ebcef0599cfc92f67037f8ab1a/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fbf8506848b0254b3c89b27c045be38bbef6372b21714cad45d76b0c8cb92ce", size = 3808535 },
-    { url = "https://files.pythonhosted.org/packages/e7/e4/37c7e2c9e251148be17292d0656d7d1ab35019678f6bd11090a41c270d18/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c1a0d9c14bf2f36676016c753517d9470381969c2a67859716cceae33735f3ee", size = 3978997 },
-    { url = "https://files.pythonhosted.org/packages/71/27/c4ba353d9b77889136bdfd1c0cd1a04d6eade9da6e0748b06719c458afb5/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6df97301782ecbaf5f2f0252011a9ff309471cde25435bdf1e17b29c263ebc16", size = 5066492 },
-    { url = "https://files.pythonhosted.org/packages/a6/81/34107fc9aacc489e41afed420202645675b41d85b46dc70d5ba222312791/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1948cfdd0e1c7d03a0c2067821dd536ab34d1e726515202e51fbd6b0d9f775f", size = 4106130 },
-    { url = "https://files.pythonhosted.org/packages/92/5f/2e348b884738fb213fb3b4745955baeeaf047aecb37639e39a4dd8f12d99/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95b1611b66c386cc6c74e990df4f114bcf24956a35e18e51bf6331c079a36688", size = 3913166 },
-    { url = "https://files.pythonhosted.org/packages/bf/81/fdda8bb5f84df82bc9e000435a88be46d46dda41eb5149f624ed96b7031c/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1751357a1aaa26aeb5feb6f66873b6a2d369655039f7278dedcb692b512111cc", size = 4313573 },
-    { url = "https://files.pythonhosted.org/packages/a0/14/ca0bc7d3b158094e769ba2bbc43d203330e7e457ed67b50af97d3eac45df/geoarrow_rust_core-0.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:16fe159043a444579948864808ebec8c49ec167ec0df3cb772dfb88de268bc91", size = 3318746 },
-    { url = "https://files.pythonhosted.org/packages/85/b8/94e4f8fb32ef705cf65031a24c58cdc441042a68a794b74757a6561cbc60/geoarrow_rust_core-0.6.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6c1b692f76b613757438bf23cfe3be4a8715f0268afd8ad3ca0063c257a3be4b", size = 3568328 },
-    { url = "https://files.pythonhosted.org/packages/7c/45/a96e64f9febc3436766c5055508c4e823cce56577529d7b76c4e4f584bc4/geoarrow_rust_core-0.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1a2b4f9a8cfe852a0ba9a667258307db9e354b470b7e0a03edffd0b7daf9b6f5", size = 3325879 },
-    { url = "https://files.pythonhosted.org/packages/58/c0/c719ce3fb4e982e28c71f65a80cf697d07d733336e6b74d7d1b8a7daf9d0/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8248330f5c3e7ec5852d0a23c23b31a08395300ef9544109e2991317beddfee3", size = 3809144 },
-    { url = "https://files.pythonhosted.org/packages/e2/8e/2ab3563b2ffd13f2dd69c050a901de0a4bb325879531a66f56d30bc7337e/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:775e9fe45c06d02be59b1497c60aa4f7a7c1d460387bf5f63142faf39b8ad4ff", size = 3978886 },
-    { url = "https://files.pythonhosted.org/packages/db/0a/31625caa0a32e8e9e7aaf2514a840dda0dadf8e2452710ebc10e5f469494/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94de8fb01da3f22332eab28b03570c43cc36492ce482c254fe87e851ae21285b", size = 5065429 },
-    { url = "https://files.pythonhosted.org/packages/11/8d/ee247bd4ccf3b0791b8669357d440e3960d4dbd5cca940a2e226e8910c31/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c70a63d1d36687a53dc6c2933446b1435c187e4c616cd84844d89b6ba13bc4f6", size = 4105436 },
-    { url = "https://files.pythonhosted.org/packages/a9/fb/c1e92716ee5aa00d48b650f0cb43220a1bf4088c8d572dfc21d400b16723/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e505312f2761393fe5158242f3f2d77e9daa5cca63badd8d66e6d1d69fc17bf", size = 3913672 },
-    { url = "https://files.pythonhosted.org/packages/f8/6f/ef47f6070c5d5cf0d061d5f5ba95aed7e895e4720a784b84c911c0209fc0/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a732e58549108df8267ab72fa6cc7c54e5a9e30b818d8d869e301a9de9d3029e", size = 4313496 },
-    { url = "https://files.pythonhosted.org/packages/3c/ac/2696b979623ea02129e342f8820c89d03fa5a253a913ad00b588d6dd2948/geoarrow_rust_core-0.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:9e1d6492b1388b9d5ae898728838ada78dbf2340d2e9dd25ad3df6ccdd058813", size = 3318780 },
-    { url = "https://files.pythonhosted.org/packages/4e/42/0cb3af24b01d3897a9eee6af5cc0676bf6b80364e0d4638e45a5fc873d35/geoarrow_rust_core-0.6.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3748cc8e8cb2bcedaede27cefed6749d4eea93e358b49a2f0b061d8974dd1b91", size = 3560313 },
-    { url = "https://files.pythonhosted.org/packages/51/bc/33f8c918e46188707ab358752b993bee9184fa62e580998c1ec4c37885c1/geoarrow_rust_core-0.6.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1b0e232fe4e239ca435d0bab638934eee87d758024c1727ee24a2b8bc4d8bc7b", size = 3321855 },
-    { url = "https://files.pythonhosted.org/packages/f4/d7/aeb2a3922670ad57f62cb591bd0309a8300ceeec6efc7f925a563c9da672/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:843444ada2c7f7670fd9df3bdebd93e5247b376d1dd20c4fb3828632847ab78e", size = 3799057 },
-    { url = "https://files.pythonhosted.org/packages/76/08/606e55fc2a0e85b02e0fde7dec2014eb8f1463e8a823496d72a3095de73d/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:880641183a09ebfbca3a6357071f137d1a4b0f1ba606fb9127a01cf58faaef56", size = 3968892 },
-    { url = "https://files.pythonhosted.org/packages/10/1f/e75fd5b59e9e582190c11ec73c91728d96e90608a22e0aed7365439d9534/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6bb69024257d2fd20da691d1e15bcced874d278884218b64690256982fa30cb1", size = 5049247 },
-    { url = "https://files.pythonhosted.org/packages/7e/95/2257b9b148c8c6557387e67828a5096ebc519b997a158ffb67a0987589e5/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:85464a1bab81068789de5fb19684e43709d2ba6d64d5655aace7c50b35893d6d", size = 4099850 },
-    { url = "https://files.pythonhosted.org/packages/b9/07/8c8aaf8755ee7c137f0898823bd005ffb16edaa6accc0cc1a9a747d56ddc/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7eb773a101f1d9716d750bb326991885a7c4576e85d9a016a567a3b07380bf07", size = 3908308 },
-    { url = "https://files.pythonhosted.org/packages/dc/7e/b8f1933be03d9a3a6416edf29fc23d520e45f00fbde6bd8f0614ad6f8a69/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:920e6fed857acd2145a8fca7c6fad17094873f586ac5efed7049ce43a7af4ff6", size = 4307178 },
-    { url = "https://files.pythonhosted.org/packages/df/95/a8ba3d7e51ec02ec954d0247c6021b36de5935a9a3845c1cf6c1348cd6e3/geoarrow_rust_core-0.6.1-cp314-cp314-win_amd64.whl", hash = "sha256:9887119cc31a763c34ed8676d06434b47971517e86f8e35c640b494d05e7d5ac", size = 3316511 },
+    { url = "https://files.pythonhosted.org/packages/77/2d/3e994dd76223fac0eb597a6f55647cca51bd5a4f446d09b668697f901724/geoarrow_rust_core-0.6.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:84d972cc3dd45a797fd99588d7ee68f257e4083ebdcecad9ec773260067f71a6", size = 3570129, upload-time = "2025-12-03T18:51:07.148Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/2a/e19df203b4ffb225f39627e1bd1b89ce7b2220e39f1d6972692174820c57/geoarrow_rust_core-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bc0f382d4ed41e85d2d89fc2c7c8c3d046681c9a5e19350ce79e0e930cf69821", size = 3333881, upload-time = "2025-11-21T01:49:28.959Z" },
+    { url = "https://files.pythonhosted.org/packages/52/98/b749a2165dfc5d9c54a1c19eb3e6a75b6d005ecde42289b25c1c355346b7/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80e719edcaf6698ed2b1aa9525bd97cf79e23a500a39b1e83566cd9a16a294d3", size = 3806366, upload-time = "2025-11-21T01:48:03.525Z" },
+    { url = "https://files.pythonhosted.org/packages/84/93/7c0e42ba7d46208fb0f851e06c05de071962170f3a3b2a2260d8a3f66e7a/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d0f3546a15503329880063aca31266b301b0b781f618f832585bcd1c9efcc876", size = 3981800, upload-time = "2025-11-21T01:48:17.789Z" },
+    { url = "https://files.pythonhosted.org/packages/de/43/9c5736569dead60b33e46b7c485e24804d950693df70dee306e153547789/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6937f3cabebf673f8b726d60d8ca160b46401de8b08c8e257be22772c12c2001", size = 5068955, upload-time = "2025-11-21T01:48:32.569Z" },
+    { url = "https://files.pythonhosted.org/packages/71/5e/f26f9bea2af96b0d070e980dcc2196d369a678e06141ed260de5ca72bcc2/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f29ba92053e8ad4bd60d72188518f033ca4abc1f34eecebeb41ee7b790612e00", size = 4104946, upload-time = "2025-11-21T01:48:45.801Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/08/473796b3e0c03b35292220de88c8efa3e74d6174e807b26a371f2523a4b0/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14a5d05a312fbb76821566b1d144c64d0923fcbd790b2c7376ee11f62472b2fe", size = 3917533, upload-time = "2025-11-21T01:49:14.631Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/7a/7b62b839c3a9878a7d91b8395e0b7b04483e4bec687e073df0fbd4056583/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:88fe8fd33b16a06e9b3b7638b51d24047f1d01af12cc2e3e2653140877bddef6", size = 4318837, upload-time = "2025-11-21T01:48:58.953Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/86/309c55a9c63f316e3a04949ade8847b8e5acbdd21645696911175f0e1814/geoarrow_rust_core-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:dbecc2487cc95526ac77797cd70c199e196811b0a9e877c1b61fcaca508575fa", size = 3320081, upload-time = "2025-11-21T01:49:58.861Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/ed/514cff089185d71242a62e774e2c59dda147baab65929851b66d72198d5d/geoarrow_rust_core-0.6.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:e26ca240d7a6a0fa1b4f56a9ebe07b2e14fc7c1c9507aa862bd31ef14e0521f0", size = 3572326, upload-time = "2025-12-03T18:51:08.477Z" },
+    { url = "https://files.pythonhosted.org/packages/77/21/22f8233235bd020db22b4f2bf888f9aeed08813eda7b8b421a6963bdc7e4/geoarrow_rust_core-0.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:46876e3528685673e08b4cbc696dca7f22fb073a83318708b0eaf640107b923b", size = 3335166, upload-time = "2025-11-21T01:49:30.632Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/eb/0c2e40a6a1bd450347a8a9fc7648ca840710bc177ff6eed3fc5da6ef981a/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5502bd12ede712d9b4725753df4db231a0aa6d3e131079bc4b6452c436e37b7", size = 3800540, upload-time = "2025-11-21T01:48:05.583Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/42/22d3b8441bb7041a6fcdb4cf0a1108e150513a52f8a407715188412bc71f/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f04dd7dd03449dba6d15f7d35c6c708637ac05f125638f56206e876756cd4c5", size = 3984840, upload-time = "2025-11-21T01:48:19.719Z" },
+    { url = "https://files.pythonhosted.org/packages/12/44/477b6b2389398dc983026a4ab7dbb7ec121284ad5fb864a1b7a4658c3881/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d2afce33d0c3fa87d5d4d24d6617732e4297da3372b1746569b759f9b62aede1", size = 5067358, upload-time = "2025-11-21T01:48:34.373Z" },
+    { url = "https://files.pythonhosted.org/packages/62/50/6995e9d11462635972b2fc09c8e1e510928563ca4fb0fd2c9145cf6ef771/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e63cdb661652a9836dc86cb5995ad269817d88b80f4cce6ed236a7f80f0aba", size = 4105773, upload-time = "2025-11-21T01:48:47.461Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/21/b369208495f213db0a0e7d563358307a706cc6af0cb9c897dacf28ae06a1/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:adbaf97cb770aef69df8a16437c9faa67adb2b04856faf45bcb61d5b986101dc", size = 3914659, upload-time = "2025-11-21T01:49:16.35Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/49/fccb14c6ee9bb715451e4d5bbe3d571eb59a8a1abe21b2abe0d9d48a7fac/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:202f35b301caa5154d95fd74424a1ef6449306e4f6fbfb5140270e48e94188a5", size = 4315153, upload-time = "2025-11-21T01:49:01.075Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/1c/88b16510e24a4a3332284669085673701b9fe4d6a511b4466c90655a9daf/geoarrow_rust_core-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:491405dfcc821a2c599e381cc9923e04a758deb1cc84fdb5794b519446c2f8a8", size = 3320510, upload-time = "2025-11-21T01:50:15.545Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/5f/1dbdbc1dde2140937cff20188cb25034b6f39e1734c14ca6510cf464bf77/geoarrow_rust_core-0.6.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a8145a562e94419402dd0882bb62429853804c53d47dbea944f2a24abc57abd2", size = 3568115, upload-time = "2025-12-03T18:51:09.743Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/e1/b62676f89ef3b866676967989ee8dbbd3d16c77f69aa4287825703268c42/geoarrow_rust_core-0.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:51040a5afcfa0cd3ab372d981375c7fe8eb652d155e3964d52ed51d14faa04e8", size = 3325336, upload-time = "2025-11-21T01:49:32.67Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/89/94e20f255712ff0eaccf9bfeac4bf51953ebcef0599cfc92f67037f8ab1a/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fbf8506848b0254b3c89b27c045be38bbef6372b21714cad45d76b0c8cb92ce", size = 3808535, upload-time = "2025-11-21T01:48:07.618Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/e4/37c7e2c9e251148be17292d0656d7d1ab35019678f6bd11090a41c270d18/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c1a0d9c14bf2f36676016c753517d9470381969c2a67859716cceae33735f3ee", size = 3978997, upload-time = "2025-11-21T01:48:21.551Z" },
+    { url = "https://files.pythonhosted.org/packages/71/27/c4ba353d9b77889136bdfd1c0cd1a04d6eade9da6e0748b06719c458afb5/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6df97301782ecbaf5f2f0252011a9ff309471cde25435bdf1e17b29c263ebc16", size = 5066492, upload-time = "2025-11-21T01:48:36.142Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/81/34107fc9aacc489e41afed420202645675b41d85b46dc70d5ba222312791/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1948cfdd0e1c7d03a0c2067821dd536ab34d1e726515202e51fbd6b0d9f775f", size = 4106130, upload-time = "2025-11-21T01:48:49.144Z" },
+    { url = "https://files.pythonhosted.org/packages/92/5f/2e348b884738fb213fb3b4745955baeeaf047aecb37639e39a4dd8f12d99/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95b1611b66c386cc6c74e990df4f114bcf24956a35e18e51bf6331c079a36688", size = 3913166, upload-time = "2025-11-21T01:49:18.228Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/81/fdda8bb5f84df82bc9e000435a88be46d46dda41eb5149f624ed96b7031c/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1751357a1aaa26aeb5feb6f66873b6a2d369655039f7278dedcb692b512111cc", size = 4313573, upload-time = "2025-11-21T01:49:03.184Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/14/ca0bc7d3b158094e769ba2bbc43d203330e7e457ed67b50af97d3eac45df/geoarrow_rust_core-0.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:16fe159043a444579948864808ebec8c49ec167ec0df3cb772dfb88de268bc91", size = 3318746, upload-time = "2025-11-21T01:50:17.319Z" },
+    { url = "https://files.pythonhosted.org/packages/85/b8/94e4f8fb32ef705cf65031a24c58cdc441042a68a794b74757a6561cbc60/geoarrow_rust_core-0.6.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6c1b692f76b613757438bf23cfe3be4a8715f0268afd8ad3ca0063c257a3be4b", size = 3568328, upload-time = "2025-12-03T18:51:11.291Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/45/a96e64f9febc3436766c5055508c4e823cce56577529d7b76c4e4f584bc4/geoarrow_rust_core-0.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1a2b4f9a8cfe852a0ba9a667258307db9e354b470b7e0a03edffd0b7daf9b6f5", size = 3325879, upload-time = "2025-11-21T01:49:34.941Z" },
+    { url = "https://files.pythonhosted.org/packages/58/c0/c719ce3fb4e982e28c71f65a80cf697d07d733336e6b74d7d1b8a7daf9d0/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8248330f5c3e7ec5852d0a23c23b31a08395300ef9544109e2991317beddfee3", size = 3809144, upload-time = "2025-11-21T01:48:09.562Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/8e/2ab3563b2ffd13f2dd69c050a901de0a4bb325879531a66f56d30bc7337e/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:775e9fe45c06d02be59b1497c60aa4f7a7c1d460387bf5f63142faf39b8ad4ff", size = 3978886, upload-time = "2025-11-21T01:48:23.335Z" },
+    { url = "https://files.pythonhosted.org/packages/db/0a/31625caa0a32e8e9e7aaf2514a840dda0dadf8e2452710ebc10e5f469494/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94de8fb01da3f22332eab28b03570c43cc36492ce482c254fe87e851ae21285b", size = 5065429, upload-time = "2025-11-21T01:48:37.896Z" },
+    { url = "https://files.pythonhosted.org/packages/11/8d/ee247bd4ccf3b0791b8669357d440e3960d4dbd5cca940a2e226e8910c31/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c70a63d1d36687a53dc6c2933446b1435c187e4c616cd84844d89b6ba13bc4f6", size = 4105436, upload-time = "2025-11-21T01:48:50.874Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/fb/c1e92716ee5aa00d48b650f0cb43220a1bf4088c8d572dfc21d400b16723/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e505312f2761393fe5158242f3f2d77e9daa5cca63badd8d66e6d1d69fc17bf", size = 3913672, upload-time = "2025-11-21T01:49:19.873Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/6f/ef47f6070c5d5cf0d061d5f5ba95aed7e895e4720a784b84c911c0209fc0/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a732e58549108df8267ab72fa6cc7c54e5a9e30b818d8d869e301a9de9d3029e", size = 4313496, upload-time = "2025-11-21T01:49:04.953Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/ac/2696b979623ea02129e342f8820c89d03fa5a253a913ad00b588d6dd2948/geoarrow_rust_core-0.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:9e1d6492b1388b9d5ae898728838ada78dbf2340d2e9dd25ad3df6ccdd058813", size = 3318780, upload-time = "2025-11-21T01:50:18.928Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/42/0cb3af24b01d3897a9eee6af5cc0676bf6b80364e0d4638e45a5fc873d35/geoarrow_rust_core-0.6.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3748cc8e8cb2bcedaede27cefed6749d4eea93e358b49a2f0b061d8974dd1b91", size = 3560313, upload-time = "2025-12-03T18:51:12.897Z" },
+    { url = "https://files.pythonhosted.org/packages/51/bc/33f8c918e46188707ab358752b993bee9184fa62e580998c1ec4c37885c1/geoarrow_rust_core-0.6.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1b0e232fe4e239ca435d0bab638934eee87d758024c1727ee24a2b8bc4d8bc7b", size = 3321855, upload-time = "2025-12-03T18:51:00.056Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/d7/aeb2a3922670ad57f62cb591bd0309a8300ceeec6efc7f925a563c9da672/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:843444ada2c7f7670fd9df3bdebd93e5247b376d1dd20c4fb3828632847ab78e", size = 3799057, upload-time = "2025-12-03T18:50:28.982Z" },
+    { url = "https://files.pythonhosted.org/packages/76/08/606e55fc2a0e85b02e0fde7dec2014eb8f1463e8a823496d72a3095de73d/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:880641183a09ebfbca3a6357071f137d1a4b0f1ba606fb9127a01cf58faaef56", size = 3968892, upload-time = "2025-12-03T18:50:34.661Z" },
+    { url = "https://files.pythonhosted.org/packages/10/1f/e75fd5b59e9e582190c11ec73c91728d96e90608a22e0aed7365439d9534/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6bb69024257d2fd20da691d1e15bcced874d278884218b64690256982fa30cb1", size = 5049247, upload-time = "2025-12-03T18:50:40.542Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/95/2257b9b148c8c6557387e67828a5096ebc519b997a158ffb67a0987589e5/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:85464a1bab81068789de5fb19684e43709d2ba6d64d5655aace7c50b35893d6d", size = 4099850, upload-time = "2025-12-03T18:50:45.341Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/07/8c8aaf8755ee7c137f0898823bd005ffb16edaa6accc0cc1a9a747d56ddc/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7eb773a101f1d9716d750bb326991885a7c4576e85d9a016a567a3b07380bf07", size = 3908308, upload-time = "2025-12-03T18:50:55.587Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/7e/b8f1933be03d9a3a6416edf29fc23d520e45f00fbde6bd8f0614ad6f8a69/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:920e6fed857acd2145a8fca7c6fad17094873f586ac5efed7049ce43a7af4ff6", size = 4307178, upload-time = "2025-12-03T18:50:50.429Z" },
+    { url = "https://files.pythonhosted.org/packages/df/95/a8ba3d7e51ec02ec954d0247c6021b36de5935a9a3845c1cf6c1348cd6e3/geoarrow_rust_core-0.6.1-cp314-cp314-win_amd64.whl", hash = "sha256:9887119cc31a763c34ed8676d06434b47971517e86f8e35c640b494d05e7d5ac", size = 3316511, upload-time = "2025-12-03T18:51:18.831Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/6d/4b2f51d0e4ac683217852d79c3acef719ca116f418d9ce8f4dcc6d717716/geoarrow_rust_core-0.6.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:420a720217b5a7ec6f7977cfe7e7a729c73381ed5e63112fdef33bd805b9cf8a", size = 3572216, upload-time = "2025-12-03T18:51:14.544Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/55/85a2948b10ad9ea347597f90355d8992745f00fedae54916205c8c9b80fb/geoarrow_rust_core-0.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0fb9c8c6bba4e712edf475ce3c78bf13f7b10f750256f57deb29c3222eaef033", size = 3335928, upload-time = "2025-11-21T01:49:51.601Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/98/fdd6c34ff8acd878c31e9f5fe4792f49d437e0465e0b60c24d6cdc287ed7/geoarrow_rust_core-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9547ead76eac906b7a583ee65fa137e6b8ed34c0f128c1745a290c451726f27", size = 3808249, upload-time = "2025-11-21T01:48:11.192Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/a1/fd6741b5c1d7d48b5f6ab58a994a91c86e29d19ee7bca2636590b8ac9a54/geoarrow_rust_core-0.6.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eaa8e8f40ca8fcd367735cb4226c5aa5171a713d75bc2caab9a03bd9f59d7bf2", size = 3984081, upload-time = "2025-11-21T01:48:25.595Z" },
+    { url = "https://files.pythonhosted.org/packages/91/1e/2b5a9b65bf19a79d212ea0fe60fa5632ec4c89bb64ee446272b47e5cd6ac/geoarrow_rust_core-0.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:08992719a2accbf993837a6aad615e3f2bf1954d2d9152e507dd79621c87e9d3", size = 5071749, upload-time = "2025-11-21T01:48:39.673Z" },
+    { url = "https://files.pythonhosted.org/packages/08/7a/6b37f5e52300b60854b74f4cdc9fbe613c692a15c3ae42f1952f3849bc86/geoarrow_rust_core-0.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:071c0e72c4c2047326ebec8d76ce2debcdd59e187207433c3a29ac2da861ca92", size = 4107621, upload-time = "2025-11-21T01:48:52.632Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/3e/f849642ef4e1f54bcc651903f19a219c3d2be68d27f4ceb282a07ebba7cd/geoarrow_rust_core-0.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c49d5a9e7b73c30dd1790a3e0faf30b7a4ee393c127c5a799d543653d1d80f0c", size = 3919352, upload-time = "2025-11-21T01:49:21.495Z" },
+    { url = "https://files.pythonhosted.org/packages/84/c8/57318cb04d061788d5ba523984915c98523e9eb9b7ba4937ff3438e045ef/geoarrow_rust_core-0.6.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:909152922ee42197b8ae846a8b6c5383c6f3ab39fe627ec8539765e3a634de68", size = 4320006, upload-time = "2025-11-21T01:49:06.588Z" },
+    { url = "https://files.pythonhosted.org/packages/13/9f/be16e191fdedbac4d9c01096327917a948625619423c666ec3db2191b4ab/geoarrow_rust_core-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:796c84184fe5e65e30df9f9f45aa8c1680f07689ea71ed1960faa7324fb67e52", size = 3321071, upload-time = "2025-11-21T01:50:20.844Z" },
 ]
 
 [[package]]
@@ -632,55 +797,64 @@ version = "0.6.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "arro3-core" },
-    { name = "pyproj", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "pyproj", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "pyproj", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "pyproj", version = "3.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/bf/30/34858dfea53d05ccc4222cd1a40e4a8cd67a0db26dc4571c23b17184de04/geoarrow_rust_io-0.6.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3d9da006559ef26bb51f76a292182ded022516792dd44e925fb96d164f29b710", size = 9779187 },
-    { url = "https://files.pythonhosted.org/packages/66/57/989ff25af2edb552047f725a4538fd2e3581e06c5a01f1928a93722b7e38/geoarrow_rust_io-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c1485af3a34d8d04077c14b259c0d2c28bd34d73d0a09e1e57b6784fd851618a", size = 9315328 },
-    { url = "https://files.pythonhosted.org/packages/0b/bc/726bc19080b16b485ba7d657b8fd8f7a90b54c2a4669fd5a68fa3562cca6/geoarrow_rust_io-0.6.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:460441ce78ebe348ace2508618c0bf04b8ffb78d6b64d7f64223c439b19677ab", size = 10307443 },
-    { url = "https://files.pythonhosted.org/packages/4b/5b/488cd94412bf10d250fe0073cc77891507f4dbbd02a2ca166ad178e3cded/geoarrow_rust_io-0.6.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9f3c86ddc6ef66de5f4a2107202f326defd50c91a11e61cfc3051ce53325eaac", size = 11287758 },
-    { url = "https://files.pythonhosted.org/packages/05/97/4f1a8809a4b5f51cc69537c0b0990d1fa32a10eef76255093383c1999422/geoarrow_rust_io-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e097a8990c85b8d449286ad495acc0bd1fd7eeabfe168787bbd5d8100a9fa5e8", size = 13300003 },
-    { url = "https://files.pythonhosted.org/packages/e0/35/bc4e80c3553a9fd8c2227bf850a9c2a6b9756623196b17f29a2f394c4304/geoarrow_rust_io-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bcb9788461a2e41a1b5f9041cf797b6395d010f52a5c35012c0be9f0a02ddc8", size = 10485708 },
-    { url = "https://files.pythonhosted.org/packages/af/d8/86b3e8e34b9a999d7c44945a49bb09ea58f6c0d7c5600102e63d1b9a4d2d/geoarrow_rust_io-0.6.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:d9836cd0469d0fabcd5b64ea85fed6ce0c4c2f508e16ee8eab7c3aad82fb351d", size = 10393066 },
-    { url = "https://files.pythonhosted.org/packages/43/54/f24a08a1a9a2eafc798125c9c5897041471032d566de3d3de80244987096/geoarrow_rust_io-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:6030616355e023e18212f2593c0a0f84f31a47fa08799d343081ecce9b1011a5", size = 8987500 },
-    { url = "https://files.pythonhosted.org/packages/05/2d/54a854ded5d1a233a0a13974b0abdcbd8d9bdf48ea1788b321d88500bc0c/geoarrow_rust_io-0.6.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d269f3b20176d8a54c86db2352a35a0a2b8275306d2e72cfa234691bf4d566d8", size = 9780467 },
-    { url = "https://files.pythonhosted.org/packages/e0/15/d816532f335c747c724d7fdd912de1553aaf6c81b642c176d6fdc105f2ca/geoarrow_rust_io-0.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c8066aedd3ae559c4f3d4958adf05ba11f7341ec4f50858bec3360f478263978", size = 9315289 },
-    { url = "https://files.pythonhosted.org/packages/e5/01/a28c42424ec6932f74ec1a1372f4d34ab2f5d557ff7d0b0b1a2a67281e10/geoarrow_rust_io-0.6.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:30658102f96006a8caa9b8ec7ad75e9eb50002b4c51017bd639d56473ec1b807", size = 10304195 },
-    { url = "https://files.pythonhosted.org/packages/87/5e/c689e7095832a2304d91074579bf5c9cef5c6554c9dd15f2c32a346e9977/geoarrow_rust_io-0.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee76e806ddd1273acf0bec46bca045bcd70b1ab679c90c205d1ade9f70f966c4", size = 11281109 },
-    { url = "https://files.pythonhosted.org/packages/e6/9e/3f81c54336ec59c96734889e107a2d11a21dbbebefbab445b133a04b804e/geoarrow_rust_io-0.6.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87deb98f87ec83bfbd2a639a9736ba3079f7e831951c6e67fafc10ca2f95b463", size = 13298204 },
-    { url = "https://files.pythonhosted.org/packages/74/f2/0f3c261a85c8fb999866fdf47c6054b0238826e07209a90205abf953794e/geoarrow_rust_io-0.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:512ace6604c9244eaa91110016e318f6a76ef483e2038ceb3d62006cf7940432", size = 10484580 },
-    { url = "https://files.pythonhosted.org/packages/58/df/37570d23d463b1d2be8d1b8db4d60e17f976340ced55c051d24d81dc573f/geoarrow_rust_io-0.6.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:1458e1f6b96bef79b966c4b360cae8a78d0aaf4e7e05d029fde227e1cbc4bd34", size = 10393813 },
-    { url = "https://files.pythonhosted.org/packages/de/57/0fb5b7414c1f8bb356fb536fbb080e564bed25f1cbe38dd3b19bf67ab5a0/geoarrow_rust_io-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:fdf00469b710b1d59c6c0e14f5ca9c4c2753e14c3de3148f4c9a84415f16723a", size = 8987671 },
-    { url = "https://files.pythonhosted.org/packages/6d/ee/7c841b38c9eaedbb830e1eab077c2d2f86e69f7bef3327167dd142a0d950/geoarrow_rust_io-0.6.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ee3386abfe1302b761a8436b27d45e040fc9f429820c9421793cf2575c90d3f5", size = 9761654 },
-    { url = "https://files.pythonhosted.org/packages/3c/d2/aa36ab40563d95562f75e707fca2ae8e92ed5adbe77517d7b8e12ebfda44/geoarrow_rust_io-0.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bc1449699d41db7c88e85eb3d8248773ad06613094c31ffdcc1e2d08aea8cc58", size = 9299644 },
-    { url = "https://files.pythonhosted.org/packages/3e/ef/dd9fa56248048dd5d971a54272496731d464ecd19833b9336ec0c1bd6dc9/geoarrow_rust_io-0.6.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:182c57b481fefe2a5cd0981a9233aa501445ed8353d907189574996b571ebc8e", size = 10307787 },
-    { url = "https://files.pythonhosted.org/packages/38/d6/211c7d5534a346a91033b29cd24e279956f48ec5497bb0710811121f9be0/geoarrow_rust_io-0.6.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:766e319cd2d12dcb8c00fd8e37c5577667fe916ca79ad3378c8f0f30318fb886", size = 11291384 },
-    { url = "https://files.pythonhosted.org/packages/91/d0/6aefa98a808910645d96d366bde1c72bd0ccda707ec1f0a46cdbfb8c83fd/geoarrow_rust_io-0.6.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:15540b29e18d43ef38b22a2451c5ac0bbd9b8e4c16493ea799bf800c7624a70f", size = 13299716 },
-    { url = "https://files.pythonhosted.org/packages/16/94/dfbfd2af284313370b1664c204afa943ce31ad5b711dd2e42a464816fb20/geoarrow_rust_io-0.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5d39dadd0faaa7fe95314f22c4aab79eda0ea6b072a331157571430206e4d9e", size = 10499797 },
-    { url = "https://files.pythonhosted.org/packages/b6/8a/c0e851de7f492ab10640ab30b58caabe19945cb009c1cdad9801f7620153/geoarrow_rust_io-0.6.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:72112ec449f5dff041736ab1010f4908bb0e3a50785be6199ec9753d8d35b3d1", size = 10398502 },
-    { url = "https://files.pythonhosted.org/packages/d7/bd/8eb48f63a6e3dffff5cb0e9f06ec80aada0a8fa38642b88ea2d4db85e7d3/geoarrow_rust_io-0.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:132dcbff42fd6f6f2b92738cf7590b9dce204fbdaad4badb1717a1edf651c099", size = 8989856 },
-    { url = "https://files.pythonhosted.org/packages/e9/2f/805b1b899543b71190bb7f3ee4a04c7319b62a3b17b48f0d0890a63992fa/geoarrow_rust_io-0.6.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:2f678fb0cf628236f55ec0b910f0a18edaf687487e135e3a7917afb413553cb8", size = 9761553 },
-    { url = "https://files.pythonhosted.org/packages/37/13/aaa2be1f840254a7c33f747653400bb22d4a3afbff7dbacc754d55af5ef0/geoarrow_rust_io-0.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7f1ec8c4b869de9dc7e7f9a5704e7bf7b74f8103c79b89c1b9e02340d93387ab", size = 9298952 },
-    { url = "https://files.pythonhosted.org/packages/c7/4f/e560d94218fe807cc09bea66d2c37258c819a4a6d48d8785952773cab06a/geoarrow_rust_io-0.6.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:76d124beea3044827fbc21f428e2fd3c1bb9520e339c43ffd1aa2f4bc5a7a203", size = 10307162 },
-    { url = "https://files.pythonhosted.org/packages/45/9c/0b3438534c5c96db4e4a65a33b0b29d374f02dfa127937c6f8213fc34420/geoarrow_rust_io-0.6.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00f2de6d1d531236a8fedd8aba7b8b8cfcc8499a4c9e4f2f958175e72617d970", size = 11290399 },
-    { url = "https://files.pythonhosted.org/packages/5f/f4/2b1621b1c9775bb0f82834806df553431c17cd788c49bce84a24ce7f5324/geoarrow_rust_io-0.6.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ca201a88976e2cb3e6fbe3e26dc20f4882de30c0611ba2ebd117be60a4f30cc4", size = 13297437 },
-    { url = "https://files.pythonhosted.org/packages/d0/9a/f406b73d1d149c24f02350c4d4671fa8c901341872ca2841aca1a5bf7296/geoarrow_rust_io-0.6.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:080da478ee833c9888c65f48da09f6ef5952f29fd1c4848a0781ecb8ea03a1a9", size = 10499789 },
-    { url = "https://files.pythonhosted.org/packages/d2/5a/f6801ec91da5cbc16f37604054d5419c9c69e9e9c2ea753aaf8dec72527f/geoarrow_rust_io-0.6.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:b0d837a8b94c5f7fd7e52dfa44b8a6b088ccf8e07836d6f935c74528a700a596", size = 10399015 },
-    { url = "https://files.pythonhosted.org/packages/c4/e3/1868fd3e90d33040555e34b5293b406acc3f12d0bbf9e9a99d7bfd270dba/geoarrow_rust_io-0.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:b2d4211f75893416b6ed26b07d7b94e04360055e902d9158abd65e3859a530f1", size = 8988931 },
-    { url = "https://files.pythonhosted.org/packages/dc/9f/7eab9987bdcd96e6a567e6f3d06a1374dada00f1446471fb6ec15b103a55/geoarrow_rust_io-0.6.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:5034be7ec038116116fdb1b2f133ee9b44d721209aeb4af9a0fc0557a0b74626", size = 9717838 },
-    { url = "https://files.pythonhosted.org/packages/a5/5b/e04aa6d8852cefcc0644353dc00ed3b1ed7f27e16c57bb2a84ba437127cf/geoarrow_rust_io-0.6.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7bdae04ed50f9a9a4672e7dffeac6cac11fdc106b02afc1af39b78e71d38e0a4", size = 9295370 },
-    { url = "https://files.pythonhosted.org/packages/f8/61/ef2386c1fc7ac9b607c07cfdc33e6f0dd4f84b15a7c9738d823413a81afc/geoarrow_rust_io-0.6.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac673d84b7e85f400d500d5e21e35632b9e91542c45d489506d986ec3d1c2586", size = 10296902 },
-    { url = "https://files.pythonhosted.org/packages/3b/02/559acc3db5408b346d5d0bf7104943ff03ea1fdb5484b5cfb35b3c3e111c/geoarrow_rust_io-0.6.1-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34c6d50383a17391f29407a314c7600573440cef718acf3fa3974cc53c79ee4d", size = 11291753 },
-    { url = "https://files.pythonhosted.org/packages/1d/2e/6149fe6141a49a554a355b3cdf09d65511e26e101aa16b784af302cd33fb/geoarrow_rust_io-0.6.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d11004b95a4ec75c733ab57ddc57ff2e31992c32f958bc8c016deb58688992c", size = 13283008 },
-    { url = "https://files.pythonhosted.org/packages/41/14/1ec1ba4df851b477d802285e8b770f65e6774f0d6272e4e8548c8758892c/geoarrow_rust_io-0.6.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a10e67d95a134dbb5f657fe3436ea645c6760a4ffef44df211f7d9b8fb687e6", size = 10499137 },
-    { url = "https://files.pythonhosted.org/packages/a5/66/7ad618415790671664e76596c000e812e0bd39e8f347f4eb7b8e3f519a55/geoarrow_rust_io-0.6.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:61ccbb528bbe4834849c501e5990a4a6f4b87976ca6a22df7859f16760c79590", size = 10394123 },
-    { url = "https://files.pythonhosted.org/packages/43/4b/4520af8c694ca0932f995c91d604837741522bd02b66414fdff4521abc98/geoarrow_rust_io-0.6.1-cp314-cp314-win_amd64.whl", hash = "sha256:aa46f6beda6c267f420ea390f071fadd0161094c1db8d71ad54002c006fe7f21", size = 8989484 },
-    { url = "https://files.pythonhosted.org/packages/e6/9f/32059400bb853eafe5d37d8c4ae9e48cd9c43820287e435cc1566f42208e/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ef94f84ba4efb42d63588241733e1b62bbdb4edeac5513baeb7bfb07db4f204a", size = 10303111 },
-    { url = "https://files.pythonhosted.org/packages/6c/a2/7db0a685eafa41e9565a3c4e441f41d2630c084f616d2669c5fe8f5805ef/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:872dd92c52b2df342d34ac42d1b710c91c58e9dd93f5c88098816f9cd9dc8a84", size = 11299498 },
-    { url = "https://files.pythonhosted.org/packages/13/b4/1bfbfbe828ca51b4f314d9f70514c2ff19923714aa7d51ef1b0ec8600aed/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:235a7ea94faa95a4699f6577765a5e5a88bee079828c3d9015d9d5c6c240459c", size = 13299230 },
-    { url = "https://files.pythonhosted.org/packages/69/a0/8ff1c2143757e4e9f499992a837d9990db5f4379cdd4a1573a1f7c22e1ff/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f74a6c0137e6fc8c5fde329c0ed85fd4cfc349fe85b2250b7aef974547427d57", size = 10499411 },
-    { url = "https://files.pythonhosted.org/packages/6b/7e/6196a7b6c63c0875474a2c2319f2a2d92bb4acd4a8d260e1e10726ccff2b/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:209ddc68c06a2f8577deaf4d744eac21696872f21d367a3ec0b15dc7cf824d5b", size = 10404698 },
+    { url = "https://files.pythonhosted.org/packages/bf/30/34858dfea53d05ccc4222cd1a40e4a8cd67a0db26dc4571c23b17184de04/geoarrow_rust_io-0.6.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3d9da006559ef26bb51f76a292182ded022516792dd44e925fb96d164f29b710", size = 9779187, upload-time = "2025-11-21T02:11:22.98Z" },
+    { url = "https://files.pythonhosted.org/packages/66/57/989ff25af2edb552047f725a4538fd2e3581e06c5a01f1928a93722b7e38/geoarrow_rust_io-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c1485af3a34d8d04077c14b259c0d2c28bd34d73d0a09e1e57b6784fd851618a", size = 9315328, upload-time = "2025-11-21T02:11:13.758Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/bc/726bc19080b16b485ba7d657b8fd8f7a90b54c2a4669fd5a68fa3562cca6/geoarrow_rust_io-0.6.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:460441ce78ebe348ace2508618c0bf04b8ffb78d6b64d7f64223c439b19677ab", size = 10307443, upload-time = "2025-11-21T02:10:25.314Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/5b/488cd94412bf10d250fe0073cc77891507f4dbbd02a2ca166ad178e3cded/geoarrow_rust_io-0.6.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9f3c86ddc6ef66de5f4a2107202f326defd50c91a11e61cfc3051ce53325eaac", size = 11287758, upload-time = "2025-11-21T02:10:50.338Z" },
+    { url = "https://files.pythonhosted.org/packages/05/97/4f1a8809a4b5f51cc69537c0b0990d1fa32a10eef76255093383c1999422/geoarrow_rust_io-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e097a8990c85b8d449286ad495acc0bd1fd7eeabfe168787bbd5d8100a9fa5e8", size = 13300003, upload-time = "2025-11-21T02:10:37.84Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/35/bc4e80c3553a9fd8c2227bf850a9c2a6b9756623196b17f29a2f394c4304/geoarrow_rust_io-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bcb9788461a2e41a1b5f9041cf797b6395d010f52a5c35012c0be9f0a02ddc8", size = 10485708, upload-time = "2025-11-21T02:11:02.337Z" },
+    { url = "https://files.pythonhosted.org/packages/af/d8/86b3e8e34b9a999d7c44945a49bb09ea58f6c0d7c5600102e63d1b9a4d2d/geoarrow_rust_io-0.6.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:d9836cd0469d0fabcd5b64ea85fed6ce0c4c2f508e16ee8eab7c3aad82fb351d", size = 10393066, upload-time = "2025-11-21T02:10:11.659Z" },
+    { url = "https://files.pythonhosted.org/packages/43/54/f24a08a1a9a2eafc798125c9c5897041471032d566de3d3de80244987096/geoarrow_rust_io-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:6030616355e023e18212f2593c0a0f84f31a47fa08799d343081ecce9b1011a5", size = 8987500, upload-time = "2025-11-21T02:11:31.939Z" },
+    { url = "https://files.pythonhosted.org/packages/05/2d/54a854ded5d1a233a0a13974b0abdcbd8d9bdf48ea1788b321d88500bc0c/geoarrow_rust_io-0.6.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d269f3b20176d8a54c86db2352a35a0a2b8275306d2e72cfa234691bf4d566d8", size = 9780467, upload-time = "2025-11-21T02:11:24.77Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/15/d816532f335c747c724d7fdd912de1553aaf6c81b642c176d6fdc105f2ca/geoarrow_rust_io-0.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c8066aedd3ae559c4f3d4958adf05ba11f7341ec4f50858bec3360f478263978", size = 9315289, upload-time = "2025-11-21T02:11:15.504Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/01/a28c42424ec6932f74ec1a1372f4d34ab2f5d557ff7d0b0b1a2a67281e10/geoarrow_rust_io-0.6.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:30658102f96006a8caa9b8ec7ad75e9eb50002b4c51017bd639d56473ec1b807", size = 10304195, upload-time = "2025-11-21T02:10:27.658Z" },
+    { url = "https://files.pythonhosted.org/packages/87/5e/c689e7095832a2304d91074579bf5c9cef5c6554c9dd15f2c32a346e9977/geoarrow_rust_io-0.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee76e806ddd1273acf0bec46bca045bcd70b1ab679c90c205d1ade9f70f966c4", size = 11281109, upload-time = "2025-11-21T02:10:52.168Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/9e/3f81c54336ec59c96734889e107a2d11a21dbbebefbab445b133a04b804e/geoarrow_rust_io-0.6.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87deb98f87ec83bfbd2a639a9736ba3079f7e831951c6e67fafc10ca2f95b463", size = 13298204, upload-time = "2025-11-21T02:10:39.727Z" },
+    { url = "https://files.pythonhosted.org/packages/74/f2/0f3c261a85c8fb999866fdf47c6054b0238826e07209a90205abf953794e/geoarrow_rust_io-0.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:512ace6604c9244eaa91110016e318f6a76ef483e2038ceb3d62006cf7940432", size = 10484580, upload-time = "2025-11-21T02:11:04.15Z" },
+    { url = "https://files.pythonhosted.org/packages/58/df/37570d23d463b1d2be8d1b8db4d60e17f976340ced55c051d24d81dc573f/geoarrow_rust_io-0.6.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:1458e1f6b96bef79b966c4b360cae8a78d0aaf4e7e05d029fde227e1cbc4bd34", size = 10393813, upload-time = "2025-11-21T02:10:13.695Z" },
+    { url = "https://files.pythonhosted.org/packages/de/57/0fb5b7414c1f8bb356fb536fbb080e564bed25f1cbe38dd3b19bf67ab5a0/geoarrow_rust_io-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:fdf00469b710b1d59c6c0e14f5ca9c4c2753e14c3de3148f4c9a84415f16723a", size = 8987671, upload-time = "2025-11-21T02:11:33.833Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/ee/7c841b38c9eaedbb830e1eab077c2d2f86e69f7bef3327167dd142a0d950/geoarrow_rust_io-0.6.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ee3386abfe1302b761a8436b27d45e040fc9f429820c9421793cf2575c90d3f5", size = 9761654, upload-time = "2025-11-21T02:11:26.598Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/d2/aa36ab40563d95562f75e707fca2ae8e92ed5adbe77517d7b8e12ebfda44/geoarrow_rust_io-0.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bc1449699d41db7c88e85eb3d8248773ad06613094c31ffdcc1e2d08aea8cc58", size = 9299644, upload-time = "2025-11-21T02:11:17.454Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/ef/dd9fa56248048dd5d971a54272496731d464ecd19833b9336ec0c1bd6dc9/geoarrow_rust_io-0.6.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:182c57b481fefe2a5cd0981a9233aa501445ed8353d907189574996b571ebc8e", size = 10307787, upload-time = "2025-11-21T02:10:29.713Z" },
+    { url = "https://files.pythonhosted.org/packages/38/d6/211c7d5534a346a91033b29cd24e279956f48ec5497bb0710811121f9be0/geoarrow_rust_io-0.6.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:766e319cd2d12dcb8c00fd8e37c5577667fe916ca79ad3378c8f0f30318fb886", size = 11291384, upload-time = "2025-11-21T02:10:53.939Z" },
+    { url = "https://files.pythonhosted.org/packages/91/d0/6aefa98a808910645d96d366bde1c72bd0ccda707ec1f0a46cdbfb8c83fd/geoarrow_rust_io-0.6.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:15540b29e18d43ef38b22a2451c5ac0bbd9b8e4c16493ea799bf800c7624a70f", size = 13299716, upload-time = "2025-11-21T02:10:41.88Z" },
+    { url = "https://files.pythonhosted.org/packages/16/94/dfbfd2af284313370b1664c204afa943ce31ad5b711dd2e42a464816fb20/geoarrow_rust_io-0.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5d39dadd0faaa7fe95314f22c4aab79eda0ea6b072a331157571430206e4d9e", size = 10499797, upload-time = "2025-11-21T02:11:05.989Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/8a/c0e851de7f492ab10640ab30b58caabe19945cb009c1cdad9801f7620153/geoarrow_rust_io-0.6.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:72112ec449f5dff041736ab1010f4908bb0e3a50785be6199ec9753d8d35b3d1", size = 10398502, upload-time = "2025-11-21T02:10:15.766Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/bd/8eb48f63a6e3dffff5cb0e9f06ec80aada0a8fa38642b88ea2d4db85e7d3/geoarrow_rust_io-0.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:132dcbff42fd6f6f2b92738cf7590b9dce204fbdaad4badb1717a1edf651c099", size = 8989856, upload-time = "2025-11-21T02:11:35.563Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/2f/805b1b899543b71190bb7f3ee4a04c7319b62a3b17b48f0d0890a63992fa/geoarrow_rust_io-0.6.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:2f678fb0cf628236f55ec0b910f0a18edaf687487e135e3a7917afb413553cb8", size = 9761553, upload-time = "2025-11-21T02:11:28.377Z" },
+    { url = "https://files.pythonhosted.org/packages/37/13/aaa2be1f840254a7c33f747653400bb22d4a3afbff7dbacc754d55af5ef0/geoarrow_rust_io-0.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7f1ec8c4b869de9dc7e7f9a5704e7bf7b74f8103c79b89c1b9e02340d93387ab", size = 9298952, upload-time = "2025-11-21T02:11:19.489Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/4f/e560d94218fe807cc09bea66d2c37258c819a4a6d48d8785952773cab06a/geoarrow_rust_io-0.6.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:76d124beea3044827fbc21f428e2fd3c1bb9520e339c43ffd1aa2f4bc5a7a203", size = 10307162, upload-time = "2025-11-21T02:10:31.933Z" },
+    { url = "https://files.pythonhosted.org/packages/45/9c/0b3438534c5c96db4e4a65a33b0b29d374f02dfa127937c6f8213fc34420/geoarrow_rust_io-0.6.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00f2de6d1d531236a8fedd8aba7b8b8cfcc8499a4c9e4f2f958175e72617d970", size = 11290399, upload-time = "2025-11-21T02:10:55.991Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/f4/2b1621b1c9775bb0f82834806df553431c17cd788c49bce84a24ce7f5324/geoarrow_rust_io-0.6.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ca201a88976e2cb3e6fbe3e26dc20f4882de30c0611ba2ebd117be60a4f30cc4", size = 13297437, upload-time = "2025-11-21T02:10:43.76Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/9a/f406b73d1d149c24f02350c4d4671fa8c901341872ca2841aca1a5bf7296/geoarrow_rust_io-0.6.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:080da478ee833c9888c65f48da09f6ef5952f29fd1c4848a0781ecb8ea03a1a9", size = 10499789, upload-time = "2025-11-21T02:11:07.744Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/5a/f6801ec91da5cbc16f37604054d5419c9c69e9e9c2ea753aaf8dec72527f/geoarrow_rust_io-0.6.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:b0d837a8b94c5f7fd7e52dfa44b8a6b088ccf8e07836d6f935c74528a700a596", size = 10399015, upload-time = "2025-11-21T02:10:18.767Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/e3/1868fd3e90d33040555e34b5293b406acc3f12d0bbf9e9a99d7bfd270dba/geoarrow_rust_io-0.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:b2d4211f75893416b6ed26b07d7b94e04360055e902d9158abd65e3859a530f1", size = 8988931, upload-time = "2025-11-21T02:11:37.387Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/9f/7eab9987bdcd96e6a567e6f3d06a1374dada00f1446471fb6ec15b103a55/geoarrow_rust_io-0.6.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:5034be7ec038116116fdb1b2f133ee9b44d721209aeb4af9a0fc0557a0b74626", size = 9717838, upload-time = "2025-12-03T19:02:35.488Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/5b/e04aa6d8852cefcc0644353dc00ed3b1ed7f27e16c57bb2a84ba437127cf/geoarrow_rust_io-0.6.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7bdae04ed50f9a9a4672e7dffeac6cac11fdc106b02afc1af39b78e71d38e0a4", size = 9295370, upload-time = "2025-12-03T19:02:29.737Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/61/ef2386c1fc7ac9b607c07cfdc33e6f0dd4f84b15a7c9738d823413a81afc/geoarrow_rust_io-0.6.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac673d84b7e85f400d500d5e21e35632b9e91542c45d489506d986ec3d1c2586", size = 10296902, upload-time = "2025-12-03T19:02:06.874Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/02/559acc3db5408b346d5d0bf7104943ff03ea1fdb5484b5cfb35b3c3e111c/geoarrow_rust_io-0.6.1-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34c6d50383a17391f29407a314c7600573440cef718acf3fa3974cc53c79ee4d", size = 11291753, upload-time = "2025-12-03T19:02:18.591Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/2e/6149fe6141a49a554a355b3cdf09d65511e26e101aa16b784af302cd33fb/geoarrow_rust_io-0.6.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d11004b95a4ec75c733ab57ddc57ff2e31992c32f958bc8c016deb58688992c", size = 13283008, upload-time = "2025-12-03T19:02:12.619Z" },
+    { url = "https://files.pythonhosted.org/packages/41/14/1ec1ba4df851b477d802285e8b770f65e6774f0d6272e4e8548c8758892c/geoarrow_rust_io-0.6.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a10e67d95a134dbb5f657fe3436ea645c6760a4ffef44df211f7d9b8fb687e6", size = 10499137, upload-time = "2025-12-03T19:02:24.514Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/66/7ad618415790671664e76596c000e812e0bd39e8f347f4eb7b8e3f519a55/geoarrow_rust_io-0.6.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:61ccbb528bbe4834849c501e5990a4a6f4b87976ca6a22df7859f16760c79590", size = 10394123, upload-time = "2025-12-03T19:02:01.248Z" },
+    { url = "https://files.pythonhosted.org/packages/43/4b/4520af8c694ca0932f995c91d604837741522bd02b66414fdff4521abc98/geoarrow_rust_io-0.6.1-cp314-cp314-win_amd64.whl", hash = "sha256:aa46f6beda6c267f420ea390f071fadd0161094c1db8d71ad54002c006fe7f21", size = 8989484, upload-time = "2025-12-03T19:02:40.081Z" },
+    { url = "https://files.pythonhosted.org/packages/69/87/efadbf1bb9d359f55791f7198cf9aa87f0272be6a2d373f5844f5e59cd1e/geoarrow_rust_io-0.6.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:46e3e41b726b250b44a829ab41489e5008280acb8af8e68001230babf04bafd8", size = 9780411, upload-time = "2025-11-21T02:11:30.128Z" },
+    { url = "https://files.pythonhosted.org/packages/95/73/5e108b286b219d3a46042cfa0830e0f075f4addd01f83f7c851a933919ae/geoarrow_rust_io-0.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bb95364b726c34c23fb93ebc9c08b8fa1d52062a4a9c1ac614ff8761a339ba7a", size = 9316307, upload-time = "2025-11-21T02:11:21.195Z" },
+    { url = "https://files.pythonhosted.org/packages/06/76/89c387d6d4d303feef328fc9c63df76cea52963e2046f2c092b434fb04a9/geoarrow_rust_io-0.6.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:691a67ef3a5214fb704d1a19d33a9ddf173483c3943056fb965101c19b0edd28", size = 10309182, upload-time = "2025-11-21T02:10:34.063Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/08/34ed2d76ebfb34ed6bf3312defad16b2b5246e40d59e46443a6fe19e85dd/geoarrow_rust_io-0.6.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91c82e9cbae6759798a8e4a87adb13ea617090a5498f384fc56c44775653d7f0", size = 11291230, upload-time = "2025-11-21T02:10:57.771Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/f5/9c25512c1f31101125555367e55ff28f72f449c8f56ff06c5be9e3feb9e5/geoarrow_rust_io-0.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9c2b609addc7a810eab5cd573243710d95afe8486f829edd05b311d51bbb5af", size = 13300664, upload-time = "2025-11-21T02:10:46.082Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/aa/14be165b439d3a3ffc6ced96f971b02df255e86b82c7e1f9f340d35689c3/geoarrow_rust_io-0.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6abdc80e130f472f55598543a4bb9ba522d6502a5d80017a952027a9e9c1d1ce", size = 10486589, upload-time = "2025-11-21T02:11:09.681Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/df/1c36bae723561785ce47e463f6366a3c52994795a168d7c4ed5e457e9a37/geoarrow_rust_io-0.6.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:c4638a89d61629110dde474b3d410ee2e71c89d2035ab2f2557857e7eee4ea30", size = 10395106, upload-time = "2025-11-21T02:10:20.832Z" },
+    { url = "https://files.pythonhosted.org/packages/47/d4/4e9cffad7647c07a5cd1cce68c97102dd011652168e3e09a2dedc1253a5e/geoarrow_rust_io-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:4811e96b1777fcf12ac2416872407b1e4717f9a59fe5b80ce02b1e9a087d1b5e", size = 8988735, upload-time = "2025-11-21T02:11:39.164Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/9f/32059400bb853eafe5d37d8c4ae9e48cd9c43820287e435cc1566f42208e/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ef94f84ba4efb42d63588241733e1b62bbdb4edeac5513baeb7bfb07db4f204a", size = 10303111, upload-time = "2025-11-21T02:10:36.067Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/a2/7db0a685eafa41e9565a3c4e441f41d2630c084f616d2669c5fe8f5805ef/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:872dd92c52b2df342d34ac42d1b710c91c58e9dd93f5c88098816f9cd9dc8a84", size = 11299498, upload-time = "2025-11-21T02:11:00.19Z" },
+    { url = "https://files.pythonhosted.org/packages/13/b4/1bfbfbe828ca51b4f314d9f70514c2ff19923714aa7d51ef1b0ec8600aed/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:235a7ea94faa95a4699f6577765a5e5a88bee079828c3d9015d9d5c6c240459c", size = 13299230, upload-time = "2025-11-21T02:10:48.12Z" },
+    { url = "https://files.pythonhosted.org/packages/69/a0/8ff1c2143757e4e9f499992a837d9990db5f4379cdd4a1573a1f7c22e1ff/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f74a6c0137e6fc8c5fde329c0ed85fd4cfc349fe85b2250b7aef974547427d57", size = 10499411, upload-time = "2025-11-21T02:11:11.783Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/7e/6196a7b6c63c0875474a2c2319f2a2d92bb4acd4a8d260e1e10726ccff2b/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:209ddc68c06a2f8577deaf4d744eac21696872f21d367a3ec0b15dc7cf824d5b", size = 10404698, upload-time = "2025-11-21T02:10:23.556Z" },
 ]
 
 [[package]]
@@ -690,9 +864,9 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "six" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/35/4a/0bd53b36ff0323d10d5f24ebd67af2de10a1117f5cf4d7add90df92756f1/google-pasta-0.2.0.tar.gz", hash = "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e", size = 40430 }
+sdist = { url = "https://files.pythonhosted.org/packages/35/4a/0bd53b36ff0323d10d5f24ebd67af2de10a1117f5cf4d7add90df92756f1/google-pasta-0.2.0.tar.gz", hash = "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e", size = 40430, upload-time = "2020-03-13T18:57:50.34Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a3/de/c648ef6835192e6e2cc03f40b19eeda4382c49b5bafb43d88b931c4c74ac/google_pasta-0.2.0-py3-none-any.whl", hash = "sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed", size = 57471 },
+    { url = "https://files.pythonhosted.org/packages/a3/de/c648ef6835192e6e2cc03f40b19eeda4382c49b5bafb43d88b931c4c74ac/google_pasta-0.2.0-py3-none-any.whl", hash = "sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed", size = 57471, upload-time = "2020-03-13T18:57:48.872Z" },
 ]
 
 [[package]]
@@ -702,43 +876,50 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/9d/f7/8963848164c7604efb3a3e6ee457fdb3a469653e19002bd24742473254f8/grpcio-1.75.1.tar.gz", hash = "sha256:3e81d89ece99b9ace23a6916880baca613c03a799925afb2857887efa8b1b3d2", size = 12731327 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/51/57/89fd829fb00a6d0bee3fbcb2c8a7aa0252d908949b6ab58bfae99d39d77e/grpcio-1.75.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:1712b5890b22547dd29f3215c5788d8fc759ce6dd0b85a6ba6e2731f2d04c088", size = 5705534 },
-    { url = "https://files.pythonhosted.org/packages/9a/3d/affe2fb897804c98d56361138e73786af8f4dd876b9d9851cfe6342b53c8/grpcio-1.75.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:683cfc70be0c1383449097cba637317e4737a357cfc185d887fd984206380403", size = 6289953 },
-    { url = "https://files.pythonhosted.org/packages/87/aa/0f40b7f47a0ff10d7e482bc3af22dac767c7ff27205915f08962d5ca87a2/grpcio-1.75.1-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:491444c081a54dcd5e6ada57314321ae526377f498d4aa09d975c3241c5b9e1c", size = 6949785 },
-    { url = "https://files.pythonhosted.org/packages/a5/45/b04407e44050781821c84f26df71b3f7bc469923f92f9f8bc27f1406dbcc/grpcio-1.75.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ce08d4e112d0d38487c2b631ec8723deac9bc404e9c7b1011426af50a79999e4", size = 6465708 },
-    { url = "https://files.pythonhosted.org/packages/09/3e/4ae3ec0a4d20dcaafbb6e597defcde06399ccdc5b342f607323f3b47f0a3/grpcio-1.75.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5a2acda37fc926ccc4547977ac3e56b1df48fe200de968e8c8421f6e3093df6c", size = 7100912 },
-    { url = "https://files.pythonhosted.org/packages/34/3f/a9085dab5c313bb0cb853f222d095e2477b9b8490a03634cdd8d19daa5c3/grpcio-1.75.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:745c5fe6bf05df6a04bf2d11552c7d867a2690759e7ab6b05c318a772739bd75", size = 8042497 },
-    { url = "https://files.pythonhosted.org/packages/c3/87/ea54eba931ab9ed3f999ba95f5d8d01a20221b664725bab2fe93e3dee848/grpcio-1.75.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:259526a7159d39e2db40d566fe3e8f8e034d0fb2db5bf9c00e09aace655a4c2b", size = 7493284 },
-    { url = "https://files.pythonhosted.org/packages/0c/3c/35ca9747473a306bfad0cee04504953f7098527cd112a4ab55c55af9e7bd/grpcio-1.75.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:573855ca2e58e35032aff30bfbd1ee103fbcf4472e4b28d4010757700918e326", size = 5709761 },
-    { url = "https://files.pythonhosted.org/packages/81/40/bc07aee2911f0d426fa53fe636216100c31a8ea65a400894f280274cb023/grpcio-1.75.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b1ea1bbe77ecbc1be00af2769f4ae4a88ce93be57a4f3eebd91087898ed749f9", size = 6296084 },
-    { url = "https://files.pythonhosted.org/packages/b8/d1/10c067f6c67396cbf46448b80f27583b5e8c4b46cdfbe18a2a02c2c2f290/grpcio-1.75.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:e5b425aee54cc5e3e3c58f00731e8a33f5567965d478d516d35ef99fd648ab68", size = 6950403 },
-    { url = "https://files.pythonhosted.org/packages/3f/42/5f628abe360b84dfe8dd8f32be6b0606dc31dc04d3358eef27db791ea4d5/grpcio-1.75.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0049a7bf547dafaeeb1db17079ce79596c298bfe308fc084d023c8907a845b9a", size = 6470166 },
-    { url = "https://files.pythonhosted.org/packages/c3/93/a24035080251324019882ee2265cfde642d6476c0cf8eb207fc693fcebdc/grpcio-1.75.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b8ea230c7f77c0a1a3208a04a1eda164633fb0767b4cefd65a01079b65e5b1f", size = 7107828 },
-    { url = "https://files.pythonhosted.org/packages/e4/f8/d18b984c1c9ba0318e3628dbbeb6af77a5007f02abc378c845070f2d3edd/grpcio-1.75.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:36990d629c3c9fb41e546414e5af52d0a7af37ce7113d9682c46d7e2919e4cca", size = 8045421 },
-    { url = "https://files.pythonhosted.org/packages/7e/b6/4bf9aacff45deca5eac5562547ed212556b831064da77971a4e632917da3/grpcio-1.75.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b10ad908118d38c2453ade7ff790e5bce36580c3742919007a2a78e3a1e521ca", size = 7503290 },
-    { url = "https://files.pythonhosted.org/packages/3a/81/42be79e73a50aaa20af66731c2defeb0e8c9008d9935a64dd8ea8e8c44eb/grpcio-1.75.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:7b888b33cd14085d86176b1628ad2fcbff94cfbbe7809465097aa0132e58b018", size = 5668314 },
-    { url = "https://files.pythonhosted.org/packages/14/85/21c71d674f03345ab183c634ecd889d3330177e27baea8d5d247a89b6442/grpcio-1.75.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bb658f703468d7fbb5dcc4037c65391b7dc34f808ac46ed9136c24fc5eeb041d", size = 6246335 },
-    { url = "https://files.pythonhosted.org/packages/fd/db/3beb661bc56a385ae4fa6b0e70f6b91ac99d47afb726fe76aaff87ebb116/grpcio-1.75.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4b7177a1cdb3c51b02b0c0a256b0a72fdab719600a693e0e9037949efffb200b", size = 6916309 },
-    { url = "https://files.pythonhosted.org/packages/1e/9c/eda9fe57f2b84343d44c1b66cf3831c973ba29b078b16a27d4587a1fdd47/grpcio-1.75.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d4fa6ccc3ec2e68a04f7b883d354d7fea22a34c44ce535a2f0c0049cf626ddf", size = 6435419 },
-    { url = "https://files.pythonhosted.org/packages/c3/b8/090c98983e0a9d602e3f919a6e2d4e470a8b489452905f9a0fa472cac059/grpcio-1.75.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d86880ecaeb5b2f0a8afa63824de93adb8ebe4e49d0e51442532f4e08add7d6", size = 7064893 },
-    { url = "https://files.pythonhosted.org/packages/ec/c0/6d53d4dbbd00f8bd81571f5478d8a95528b716e0eddb4217cc7cb45aae5f/grpcio-1.75.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a8041d2f9e8a742aeae96f4b047ee44e73619f4f9d24565e84d5446c623673b6", size = 8011922 },
-    { url = "https://files.pythonhosted.org/packages/f2/7c/48455b2d0c5949678d6982c3e31ea4d89df4e16131b03f7d5c590811cbe9/grpcio-1.75.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3652516048bf4c314ce12be37423c79829f46efffb390ad64149a10c6071e8de", size = 7466181 },
-    { url = "https://files.pythonhosted.org/packages/46/74/bac4ab9f7722164afdf263ae31ba97b8174c667153510322a5eba4194c32/grpcio-1.75.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:3bed22e750d91d53d9e31e0af35a7b0b51367e974e14a4ff229db5b207647884", size = 5672779 },
-    { url = "https://files.pythonhosted.org/packages/cf/e4/d1954dce2972e32384db6a30273275e8c8ea5a44b80347f9055589333b3f/grpcio-1.75.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5bf4001d3293e3414d0cf99ff9b1139106e57c3a66dfff0c5f60b2a6286ec133", size = 6248838 },
-    { url = "https://files.pythonhosted.org/packages/06/43/073363bf63826ba8077c335d797a8d026f129dc0912b69c42feaf8f0cd26/grpcio-1.75.1-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:9f82ff474103e26351dacfe8d50214e7c9322960d8d07ba7fa1d05ff981c8b2d", size = 6922663 },
-    { url = "https://files.pythonhosted.org/packages/c2/6f/076ac0df6c359117676cacfa8a377e2abcecec6a6599a15a672d331f6680/grpcio-1.75.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0ee119f4f88d9f75414217823d21d75bfe0e6ed40135b0cbbfc6376bc9f7757d", size = 6436149 },
-    { url = "https://files.pythonhosted.org/packages/6b/27/1d08824f1d573fcb1fa35ede40d6020e68a04391709939e1c6f4193b445f/grpcio-1.75.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:664eecc3abe6d916fa6cf8dd6b778e62fb264a70f3430a3180995bf2da935446", size = 7067989 },
-    { url = "https://files.pythonhosted.org/packages/c6/98/98594cf97b8713feb06a8cb04eeef60b4757e3e2fb91aa0d9161da769843/grpcio-1.75.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c32193fa08b2fbebf08fe08e84f8a0aad32d87c3ad42999c65e9449871b1c66e", size = 8010717 },
-    { url = "https://files.pythonhosted.org/packages/8c/7e/bb80b1bba03c12158f9254762cdf5cced4a9bc2e8ed51ed335915a5a06ef/grpcio-1.75.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5cebe13088b9254f6e615bcf1da9131d46cfa4e88039454aca9cb65f639bd3bc", size = 7463822 },
-    { url = "https://files.pythonhosted.org/packages/f2/1b/9a0a5cecd24302b9fdbcd55d15ed6267e5f3d5b898ff9ac8cbe17ee76129/grpcio-1.75.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:c05da79068dd96723793bffc8d0e64c45f316248417515f28d22204d9dae51c7", size = 5673319 },
-    { url = "https://files.pythonhosted.org/packages/09/7a/26da709e42c4565c3d7bf999a9569da96243ce34a8271a968dee810a7cf1/grpcio-1.75.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4484f4b7287bdaa7a5b3980f3c7224c3c622669405d20f69549f5fb956ad0421", size = 6254706 },
-    { url = "https://files.pythonhosted.org/packages/f1/08/dcb26a319d3725f199c97e671d904d84ee5680de57d74c566a991cfab632/grpcio-1.75.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2720c239c1180eee69f7883c1d4c83fc1a495a2535b5fa322887c70bf02b16e8", size = 6922501 },
-    { url = "https://files.pythonhosted.org/packages/78/66/044d412c98408a5e23cb348845979a2d17a2e2b6c3c34c1ec91b920f49d0/grpcio-1.75.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:07a554fa31c668cf0e7a188678ceeca3cb8fead29bbe455352e712ec33ca701c", size = 6437492 },
-    { url = "https://files.pythonhosted.org/packages/4e/9d/5e3e362815152aa1afd8b26ea613effa005962f9da0eec6e0e4527e7a7d1/grpcio-1.75.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3e71a2105210366bfc398eef7f57a664df99194f3520edb88b9c3a7e46ee0d64", size = 7081061 },
-    { url = "https://files.pythonhosted.org/packages/1e/1a/46615682a19e100f46e31ddba9ebc297c5a5ab9ddb47b35443ffadb8776c/grpcio-1.75.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:8679aa8a5b67976776d3c6b0521e99d1c34db8a312a12bcfd78a7085cb9b604e", size = 8010849 },
-    { url = "https://files.pythonhosted.org/packages/67/8e/3204b94ac30b0f675ab1c06540ab5578660dc8b690db71854d3116f20d00/grpcio-1.75.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aad1c774f4ebf0696a7f148a56d39a3432550612597331792528895258966dc0", size = 7464478 },
+sdist = { url = "https://files.pythonhosted.org/packages/9d/f7/8963848164c7604efb3a3e6ee457fdb3a469653e19002bd24742473254f8/grpcio-1.75.1.tar.gz", hash = "sha256:3e81d89ece99b9ace23a6916880baca613c03a799925afb2857887efa8b1b3d2", size = 12731327, upload-time = "2025-09-26T09:03:36.887Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/51/57/89fd829fb00a6d0bee3fbcb2c8a7aa0252d908949b6ab58bfae99d39d77e/grpcio-1.75.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:1712b5890b22547dd29f3215c5788d8fc759ce6dd0b85a6ba6e2731f2d04c088", size = 5705534, upload-time = "2025-09-26T09:00:52.225Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/3d/affe2fb897804c98d56361138e73786af8f4dd876b9d9851cfe6342b53c8/grpcio-1.75.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:683cfc70be0c1383449097cba637317e4737a357cfc185d887fd984206380403", size = 6289953, upload-time = "2025-09-26T09:01:03.699Z" },
+    { url = "https://files.pythonhosted.org/packages/87/aa/0f40b7f47a0ff10d7e482bc3af22dac767c7ff27205915f08962d5ca87a2/grpcio-1.75.1-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:491444c081a54dcd5e6ada57314321ae526377f498d4aa09d975c3241c5b9e1c", size = 6949785, upload-time = "2025-09-26T09:01:07.504Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/45/b04407e44050781821c84f26df71b3f7bc469923f92f9f8bc27f1406dbcc/grpcio-1.75.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ce08d4e112d0d38487c2b631ec8723deac9bc404e9c7b1011426af50a79999e4", size = 6465708, upload-time = "2025-09-26T09:01:11.028Z" },
+    { url = "https://files.pythonhosted.org/packages/09/3e/4ae3ec0a4d20dcaafbb6e597defcde06399ccdc5b342f607323f3b47f0a3/grpcio-1.75.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5a2acda37fc926ccc4547977ac3e56b1df48fe200de968e8c8421f6e3093df6c", size = 7100912, upload-time = "2025-09-26T09:01:14.393Z" },
+    { url = "https://files.pythonhosted.org/packages/34/3f/a9085dab5c313bb0cb853f222d095e2477b9b8490a03634cdd8d19daa5c3/grpcio-1.75.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:745c5fe6bf05df6a04bf2d11552c7d867a2690759e7ab6b05c318a772739bd75", size = 8042497, upload-time = "2025-09-26T09:01:17.759Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/87/ea54eba931ab9ed3f999ba95f5d8d01a20221b664725bab2fe93e3dee848/grpcio-1.75.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:259526a7159d39e2db40d566fe3e8f8e034d0fb2db5bf9c00e09aace655a4c2b", size = 7493284, upload-time = "2025-09-26T09:01:20.896Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/3c/35ca9747473a306bfad0cee04504953f7098527cd112a4ab55c55af9e7bd/grpcio-1.75.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:573855ca2e58e35032aff30bfbd1ee103fbcf4472e4b28d4010757700918e326", size = 5709761, upload-time = "2025-09-26T09:01:28.528Z" },
+    { url = "https://files.pythonhosted.org/packages/81/40/bc07aee2911f0d426fa53fe636216100c31a8ea65a400894f280274cb023/grpcio-1.75.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b1ea1bbe77ecbc1be00af2769f4ae4a88ce93be57a4f3eebd91087898ed749f9", size = 6296084, upload-time = "2025-09-26T09:01:34.596Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/d1/10c067f6c67396cbf46448b80f27583b5e8c4b46cdfbe18a2a02c2c2f290/grpcio-1.75.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:e5b425aee54cc5e3e3c58f00731e8a33f5567965d478d516d35ef99fd648ab68", size = 6950403, upload-time = "2025-09-26T09:01:36.736Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/42/5f628abe360b84dfe8dd8f32be6b0606dc31dc04d3358eef27db791ea4d5/grpcio-1.75.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0049a7bf547dafaeeb1db17079ce79596c298bfe308fc084d023c8907a845b9a", size = 6470166, upload-time = "2025-09-26T09:01:39.474Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/93/a24035080251324019882ee2265cfde642d6476c0cf8eb207fc693fcebdc/grpcio-1.75.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b8ea230c7f77c0a1a3208a04a1eda164633fb0767b4cefd65a01079b65e5b1f", size = 7107828, upload-time = "2025-09-26T09:01:41.782Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/f8/d18b984c1c9ba0318e3628dbbeb6af77a5007f02abc378c845070f2d3edd/grpcio-1.75.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:36990d629c3c9fb41e546414e5af52d0a7af37ce7113d9682c46d7e2919e4cca", size = 8045421, upload-time = "2025-09-26T09:01:45.835Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/b6/4bf9aacff45deca5eac5562547ed212556b831064da77971a4e632917da3/grpcio-1.75.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b10ad908118d38c2453ade7ff790e5bce36580c3742919007a2a78e3a1e521ca", size = 7503290, upload-time = "2025-09-26T09:01:49.28Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/81/42be79e73a50aaa20af66731c2defeb0e8c9008d9935a64dd8ea8e8c44eb/grpcio-1.75.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:7b888b33cd14085d86176b1628ad2fcbff94cfbbe7809465097aa0132e58b018", size = 5668314, upload-time = "2025-09-26T09:01:55.424Z" },
+    { url = "https://files.pythonhosted.org/packages/14/85/21c71d674f03345ab183c634ecd889d3330177e27baea8d5d247a89b6442/grpcio-1.75.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bb658f703468d7fbb5dcc4037c65391b7dc34f808ac46ed9136c24fc5eeb041d", size = 6246335, upload-time = "2025-09-26T09:02:00.76Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/db/3beb661bc56a385ae4fa6b0e70f6b91ac99d47afb726fe76aaff87ebb116/grpcio-1.75.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4b7177a1cdb3c51b02b0c0a256b0a72fdab719600a693e0e9037949efffb200b", size = 6916309, upload-time = "2025-09-26T09:02:02.894Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/9c/eda9fe57f2b84343d44c1b66cf3831c973ba29b078b16a27d4587a1fdd47/grpcio-1.75.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d4fa6ccc3ec2e68a04f7b883d354d7fea22a34c44ce535a2f0c0049cf626ddf", size = 6435419, upload-time = "2025-09-26T09:02:05.055Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/b8/090c98983e0a9d602e3f919a6e2d4e470a8b489452905f9a0fa472cac059/grpcio-1.75.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d86880ecaeb5b2f0a8afa63824de93adb8ebe4e49d0e51442532f4e08add7d6", size = 7064893, upload-time = "2025-09-26T09:02:07.275Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/c0/6d53d4dbbd00f8bd81571f5478d8a95528b716e0eddb4217cc7cb45aae5f/grpcio-1.75.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a8041d2f9e8a742aeae96f4b047ee44e73619f4f9d24565e84d5446c623673b6", size = 8011922, upload-time = "2025-09-26T09:02:09.527Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/7c/48455b2d0c5949678d6982c3e31ea4d89df4e16131b03f7d5c590811cbe9/grpcio-1.75.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3652516048bf4c314ce12be37423c79829f46efffb390ad64149a10c6071e8de", size = 7466181, upload-time = "2025-09-26T09:02:12.279Z" },
+    { url = "https://files.pythonhosted.org/packages/46/74/bac4ab9f7722164afdf263ae31ba97b8174c667153510322a5eba4194c32/grpcio-1.75.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:3bed22e750d91d53d9e31e0af35a7b0b51367e974e14a4ff229db5b207647884", size = 5672779, upload-time = "2025-09-26T09:02:19.11Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/e4/d1954dce2972e32384db6a30273275e8c8ea5a44b80347f9055589333b3f/grpcio-1.75.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5bf4001d3293e3414d0cf99ff9b1139106e57c3a66dfff0c5f60b2a6286ec133", size = 6248838, upload-time = "2025-09-26T09:02:26.426Z" },
+    { url = "https://files.pythonhosted.org/packages/06/43/073363bf63826ba8077c335d797a8d026f129dc0912b69c42feaf8f0cd26/grpcio-1.75.1-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:9f82ff474103e26351dacfe8d50214e7c9322960d8d07ba7fa1d05ff981c8b2d", size = 6922663, upload-time = "2025-09-26T09:02:28.724Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/6f/076ac0df6c359117676cacfa8a377e2abcecec6a6599a15a672d331f6680/grpcio-1.75.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0ee119f4f88d9f75414217823d21d75bfe0e6ed40135b0cbbfc6376bc9f7757d", size = 6436149, upload-time = "2025-09-26T09:02:30.971Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/27/1d08824f1d573fcb1fa35ede40d6020e68a04391709939e1c6f4193b445f/grpcio-1.75.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:664eecc3abe6d916fa6cf8dd6b778e62fb264a70f3430a3180995bf2da935446", size = 7067989, upload-time = "2025-09-26T09:02:33.233Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/98/98594cf97b8713feb06a8cb04eeef60b4757e3e2fb91aa0d9161da769843/grpcio-1.75.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c32193fa08b2fbebf08fe08e84f8a0aad32d87c3ad42999c65e9449871b1c66e", size = 8010717, upload-time = "2025-09-26T09:02:36.011Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/7e/bb80b1bba03c12158f9254762cdf5cced4a9bc2e8ed51ed335915a5a06ef/grpcio-1.75.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5cebe13088b9254f6e615bcf1da9131d46cfa4e88039454aca9cb65f639bd3bc", size = 7463822, upload-time = "2025-09-26T09:02:38.26Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/1b/9a0a5cecd24302b9fdbcd55d15ed6267e5f3d5b898ff9ac8cbe17ee76129/grpcio-1.75.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:c05da79068dd96723793bffc8d0e64c45f316248417515f28d22204d9dae51c7", size = 5673319, upload-time = "2025-09-26T09:02:44.742Z" },
+    { url = "https://files.pythonhosted.org/packages/09/7a/26da709e42c4565c3d7bf999a9569da96243ce34a8271a968dee810a7cf1/grpcio-1.75.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4484f4b7287bdaa7a5b3980f3c7224c3c622669405d20f69549f5fb956ad0421", size = 6254706, upload-time = "2025-09-26T09:02:50.4Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/08/dcb26a319d3725f199c97e671d904d84ee5680de57d74c566a991cfab632/grpcio-1.75.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2720c239c1180eee69f7883c1d4c83fc1a495a2535b5fa322887c70bf02b16e8", size = 6922501, upload-time = "2025-09-26T09:02:52.711Z" },
+    { url = "https://files.pythonhosted.org/packages/78/66/044d412c98408a5e23cb348845979a2d17a2e2b6c3c34c1ec91b920f49d0/grpcio-1.75.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:07a554fa31c668cf0e7a188678ceeca3cb8fead29bbe455352e712ec33ca701c", size = 6437492, upload-time = "2025-09-26T09:02:55.542Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/9d/5e3e362815152aa1afd8b26ea613effa005962f9da0eec6e0e4527e7a7d1/grpcio-1.75.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3e71a2105210366bfc398eef7f57a664df99194f3520edb88b9c3a7e46ee0d64", size = 7081061, upload-time = "2025-09-26T09:02:58.261Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/1a/46615682a19e100f46e31ddba9ebc297c5a5ab9ddb47b35443ffadb8776c/grpcio-1.75.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:8679aa8a5b67976776d3c6b0521e99d1c34db8a312a12bcfd78a7085cb9b604e", size = 8010849, upload-time = "2025-09-26T09:03:00.548Z" },
+    { url = "https://files.pythonhosted.org/packages/67/8e/3204b94ac30b0f675ab1c06540ab5578660dc8b690db71854d3116f20d00/grpcio-1.75.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aad1c774f4ebf0696a7f148a56d39a3432550612597331792528895258966dc0", size = 7464478, upload-time = "2025-09-26T09:03:03.096Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/e2/33efd823a879dc7b60c10192df1900ee5c200f8e782663a41a3b2aecd143/grpcio-1.75.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:c09fba33327c3ac11b5c33dbdd8218eef8990d78f83b1656d628831812a8c0fb", size = 5706679, upload-time = "2025-09-26T09:03:10.218Z" },
+    { url = "https://files.pythonhosted.org/packages/77/90/b80e75f8cce758425b2772742eed4e9db765a965d902ba4b7f239b2513de/grpcio-1.75.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c12121e509b9f8b0914d10054d24120237d19e870b1cd82acbb8a9b9ddd198a3", size = 6291926, upload-time = "2025-09-26T09:03:16.282Z" },
+    { url = "https://files.pythonhosted.org/packages/40/5f/e6033d8f99063350e20873a46225468b73045b9ef2c8cba73d66a87c3fd5/grpcio-1.75.1-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:73577a93e692b3474b1bfe84285d098de36705dbd838bb4d6a056d326e4dc880", size = 6950040, upload-time = "2025-09-26T09:03:18.874Z" },
+    { url = "https://files.pythonhosted.org/packages/01/12/34076c079b45af5aed40f037fffe388d7fbe90dd539ed01e4744c926d227/grpcio-1.75.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e19e7dfa0d7ca7dea22be464339e18ac608fd75d88c56770c646cdabe54bc724", size = 6465780, upload-time = "2025-09-26T09:03:21.219Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/c5/ee6fd69a9f6e7288d04da010ad7480a0566d2aac81097ff4dafbc5ffa9b6/grpcio-1.75.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4e1c28f51c1cf67eccdfc1065e8e866c9ed622f09773ca60947089c117f848a1", size = 7098308, upload-time = "2025-09-26T09:03:23.875Z" },
+    { url = "https://files.pythonhosted.org/packages/78/32/f2be13f13035361768923159fe20470a7d22db2c7c692b952e21284f56e5/grpcio-1.75.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:030a6164bc2ca726052778c0cf8e3249617a34e368354f9e6107c27ad4af8c28", size = 8042268, upload-time = "2025-09-26T09:03:26.268Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/2d/1bb0572f0a2eaab100b4635c6c2cd0d37e3cda5554037e3f90b1bc428d56/grpcio-1.75.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:67697efef5a98d46d5db7b1720fa4043536f8b8e5072a5d61cfca762f287e939", size = 7491470, upload-time = "2025-09-26T09:03:28.906Z" },
 ]
 
 [[package]]
@@ -746,34 +927,37 @@ name = "h5py"
 version = "3.14.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/5d/57/dfb3c5c3f1bf5f5ef2e59a22dec4ff1f3d7408b55bfcefcfb0ea69ef21c6/h5py-3.14.0.tar.gz", hash = "sha256:2372116b2e0d5d3e5e705b7f663f7c8d96fa79a4052d250484ef91d24d6a08f4", size = 424323 }
+sdist = { url = "https://files.pythonhosted.org/packages/5d/57/dfb3c5c3f1bf5f5ef2e59a22dec4ff1f3d7408b55bfcefcfb0ea69ef21c6/h5py-3.14.0.tar.gz", hash = "sha256:2372116b2e0d5d3e5e705b7f663f7c8d96fa79a4052d250484ef91d24d6a08f4", size = 424323, upload-time = "2025-06-06T14:06:15.01Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/fa/cd/3dd38cdb7cc9266dc4d85f27f0261680cb62f553f1523167ad7454e32b11/h5py-3.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:016e89d3be4c44f8d5e115fab60548e518ecd9efe9fa5c5324505a90773e6f03", size = 4324677 },
-    { url = "https://files.pythonhosted.org/packages/b1/45/e1a754dc7cd465ba35e438e28557119221ac89b20aaebef48282654e3dc7/h5py-3.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1223b902ef0b5d90bcc8a4778218d6d6cd0f5561861611eda59fa6c52b922f4d", size = 4557272 },
-    { url = "https://files.pythonhosted.org/packages/08/0c/5e6aaf221557314bc15ba0e0da92e40b24af97ab162076c8ae009320a42b/h5py-3.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c497600c0496548810047257e36360ff551df8b59156d3a4181072eed47d8ad", size = 4298002 },
-    { url = "https://files.pythonhosted.org/packages/21/d4/d461649cafd5137088fb7f8e78fdc6621bb0c4ff2c090a389f68e8edc136/h5py-3.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:723a40ee6505bd354bfd26385f2dae7bbfa87655f4e61bab175a49d72ebfc06b", size = 4516618 },
-    { url = "https://files.pythonhosted.org/packages/3f/19/c8bfe8543bfdd7ccfafd46d8cfd96fce53d6c33e9c7921f375530ee1d39a/h5py-3.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554ef0ced3571366d4d383427c00c966c360e178b5fb5ee5bb31a435c424db0c", size = 4708455 },
-    { url = "https://files.pythonhosted.org/packages/86/f9/f00de11c82c88bfc1ef22633557bfba9e271e0cb3189ad704183fc4a2644/h5py-3.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cbd41f4e3761f150aa5b662df991868ca533872c95467216f2bec5fcad84882", size = 4929422 },
-    { url = "https://files.pythonhosted.org/packages/0d/ce/3a21d87896bc7e3e9255e0ad5583ae31ae9e6b4b00e0bcb2a67e2b6acdbc/h5py-3.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8cbaf6910fa3983c46172666b0b8da7b7bd90d764399ca983236f2400436eeb", size = 4700675 },
-    { url = "https://files.pythonhosted.org/packages/e7/ec/86f59025306dcc6deee5fda54d980d077075b8d9889aac80f158bd585f1b/h5py-3.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d90e6445ab7c146d7f7981b11895d70bc1dd91278a4f9f9028bc0c95e4a53f13", size = 4921632 },
+    { url = "https://files.pythonhosted.org/packages/fa/cd/3dd38cdb7cc9266dc4d85f27f0261680cb62f553f1523167ad7454e32b11/h5py-3.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:016e89d3be4c44f8d5e115fab60548e518ecd9efe9fa5c5324505a90773e6f03", size = 4324677, upload-time = "2025-06-06T14:04:23.438Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/45/e1a754dc7cd465ba35e438e28557119221ac89b20aaebef48282654e3dc7/h5py-3.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1223b902ef0b5d90bcc8a4778218d6d6cd0f5561861611eda59fa6c52b922f4d", size = 4557272, upload-time = "2025-06-06T14:04:28.863Z" },
+    { url = "https://files.pythonhosted.org/packages/08/0c/5e6aaf221557314bc15ba0e0da92e40b24af97ab162076c8ae009320a42b/h5py-3.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c497600c0496548810047257e36360ff551df8b59156d3a4181072eed47d8ad", size = 4298002, upload-time = "2025-06-06T14:04:47.106Z" },
+    { url = "https://files.pythonhosted.org/packages/21/d4/d461649cafd5137088fb7f8e78fdc6621bb0c4ff2c090a389f68e8edc136/h5py-3.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:723a40ee6505bd354bfd26385f2dae7bbfa87655f4e61bab175a49d72ebfc06b", size = 4516618, upload-time = "2025-06-06T14:04:52.467Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/19/c8bfe8543bfdd7ccfafd46d8cfd96fce53d6c33e9c7921f375530ee1d39a/h5py-3.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554ef0ced3571366d4d383427c00c966c360e178b5fb5ee5bb31a435c424db0c", size = 4708455, upload-time = "2025-06-06T14:05:11.528Z" },
+    { url = "https://files.pythonhosted.org/packages/86/f9/f00de11c82c88bfc1ef22633557bfba9e271e0cb3189ad704183fc4a2644/h5py-3.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cbd41f4e3761f150aa5b662df991868ca533872c95467216f2bec5fcad84882", size = 4929422, upload-time = "2025-06-06T14:05:18.399Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/ce/3a21d87896bc7e3e9255e0ad5583ae31ae9e6b4b00e0bcb2a67e2b6acdbc/h5py-3.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8cbaf6910fa3983c46172666b0b8da7b7bd90d764399ca983236f2400436eeb", size = 4700675, upload-time = "2025-06-06T14:05:37.38Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/ec/86f59025306dcc6deee5fda54d980d077075b8d9889aac80f158bd585f1b/h5py-3.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d90e6445ab7c146d7f7981b11895d70bc1dd91278a4f9f9028bc0c95e4a53f13", size = 4921632, upload-time = "2025-06-06T14:05:43.464Z" },
+    { url = "https://files.pythonhosted.org/packages/66/40/b423b57696514e05aa7bb06150ef96667d0e0006cc6de7ab52c71734ab51/h5py-3.14.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:573c33ad056ac7c1ab6d567b6db9df3ffc401045e3f605736218f96c1e0490c6", size = 4326368, upload-time = "2025-06-06T14:06:00.782Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/07/e088f89f04fdbe57ddf9de377f857158d3daa38cf5d0fb20ef9bd489e313/h5py-3.14.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccbe17dc187c0c64178f1a10aa274ed3a57d055117588942b8a08793cc448216", size = 4559686, upload-time = "2025-06-06T14:06:07.416Z" },
 ]
 
 [[package]]
 name = "hf-xet"
 version = "1.1.10"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/74/31/feeddfce1748c4a233ec1aa5b7396161c07ae1aa9b7bdbc9a72c3c7dd768/hf_xet-1.1.10.tar.gz", hash = "sha256:408aef343800a2102374a883f283ff29068055c111f003ff840733d3b715bb97", size = 487910 }
+sdist = { url = "https://files.pythonhosted.org/packages/74/31/feeddfce1748c4a233ec1aa5b7396161c07ae1aa9b7bdbc9a72c3c7dd768/hf_xet-1.1.10.tar.gz", hash = "sha256:408aef343800a2102374a883f283ff29068055c111f003ff840733d3b715bb97", size = 487910, upload-time = "2025-09-12T20:10:27.12Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f7/a2/343e6d05de96908366bdc0081f2d8607d61200be2ac802769c4284cc65bd/hf_xet-1.1.10-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:686083aca1a6669bc85c21c0563551cbcdaa5cf7876a91f3d074a030b577231d", size = 2761466 },
-    { url = "https://files.pythonhosted.org/packages/31/f9/6215f948ac8f17566ee27af6430ea72045e0418ce757260248b483f4183b/hf_xet-1.1.10-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:71081925383b66b24eedff3013f8e6bbd41215c3338be4b94ba75fd75b21513b", size = 2623807 },
-    { url = "https://files.pythonhosted.org/packages/15/07/86397573efefff941e100367bbda0b21496ffcdb34db7ab51912994c32a2/hf_xet-1.1.10-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b6bceb6361c80c1cc42b5a7b4e3efd90e64630bcf11224dcac50ef30a47e435", size = 3186960 },
-    { url = "https://files.pythonhosted.org/packages/01/a7/0b2e242b918cc30e1f91980f3c4b026ff2eedaf1e2ad96933bca164b2869/hf_xet-1.1.10-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eae7c1fc8a664e54753ffc235e11427ca61f4b0477d757cc4eb9ae374b69f09c", size = 3087167 },
-    { url = "https://files.pythonhosted.org/packages/4a/25/3e32ab61cc7145b11eee9d745988e2f0f4fafda81b25980eebf97d8cff15/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0a0005fd08f002180f7a12d4e13b22be277725bc23ed0529f8add5c7a6309c06", size = 3248612 },
-    { url = "https://files.pythonhosted.org/packages/2c/3d/ab7109e607ed321afaa690f557a9ada6d6d164ec852fd6bf9979665dc3d6/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f900481cf6e362a6c549c61ff77468bd59d6dd082f3170a36acfef2eb6a6793f", size = 3353360 },
-    { url = "https://files.pythonhosted.org/packages/ee/0e/471f0a21db36e71a2f1752767ad77e92d8cde24e974e03d662931b1305ec/hf_xet-1.1.10-cp37-abi3-win_amd64.whl", hash = "sha256:5f54b19cc347c13235ae7ee98b330c26dd65ef1df47e5316ffb1e87713ca7045", size = 2804691 },
+    { url = "https://files.pythonhosted.org/packages/f7/a2/343e6d05de96908366bdc0081f2d8607d61200be2ac802769c4284cc65bd/hf_xet-1.1.10-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:686083aca1a6669bc85c21c0563551cbcdaa5cf7876a91f3d074a030b577231d", size = 2761466, upload-time = "2025-09-12T20:10:22.836Z" },
+    { url = "https://files.pythonhosted.org/packages/31/f9/6215f948ac8f17566ee27af6430ea72045e0418ce757260248b483f4183b/hf_xet-1.1.10-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:71081925383b66b24eedff3013f8e6bbd41215c3338be4b94ba75fd75b21513b", size = 2623807, upload-time = "2025-09-12T20:10:21.118Z" },
+    { url = "https://files.pythonhosted.org/packages/15/07/86397573efefff941e100367bbda0b21496ffcdb34db7ab51912994c32a2/hf_xet-1.1.10-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b6bceb6361c80c1cc42b5a7b4e3efd90e64630bcf11224dcac50ef30a47e435", size = 3186960, upload-time = "2025-09-12T20:10:19.336Z" },
+    { url = "https://files.pythonhosted.org/packages/01/a7/0b2e242b918cc30e1f91980f3c4b026ff2eedaf1e2ad96933bca164b2869/hf_xet-1.1.10-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eae7c1fc8a664e54753ffc235e11427ca61f4b0477d757cc4eb9ae374b69f09c", size = 3087167, upload-time = "2025-09-12T20:10:17.255Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/25/3e32ab61cc7145b11eee9d745988e2f0f4fafda81b25980eebf97d8cff15/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0a0005fd08f002180f7a12d4e13b22be277725bc23ed0529f8add5c7a6309c06", size = 3248612, upload-time = "2025-09-12T20:10:24.093Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/3d/ab7109e607ed321afaa690f557a9ada6d6d164ec852fd6bf9979665dc3d6/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f900481cf6e362a6c549c61ff77468bd59d6dd082f3170a36acfef2eb6a6793f", size = 3353360, upload-time = "2025-09-12T20:10:25.563Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/0e/471f0a21db36e71a2f1752767ad77e92d8cde24e974e03d662931b1305ec/hf_xet-1.1.10-cp37-abi3-win_amd64.whl", hash = "sha256:5f54b19cc347c13235ae7ee98b330c26dd65ef1df47e5316ffb1e87713ca7045", size = 2804691, upload-time = "2025-09-12T20:10:28.433Z" },
 ]
 
 [[package]]
@@ -781,36 +965,48 @@ name = "huggingface-hub"
 version = "0.35.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "filelock" },
-    { name = "fsspec" },
-    { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" },
-    { name = "packaging" },
-    { name = "pyyaml" },
-    { name = "requests" },
-    { name = "tqdm" },
-    { name = "typing-extensions" },
+    { name = "filelock", marker = "python_full_version >= '3.10'" },
+    { name = "fsspec", marker = "python_full_version >= '3.10'" },
+    { name = "hf-xet", marker = "(python_full_version >= '3.10' and platform_machine == 'aarch64') or (python_full_version >= '3.10' and platform_machine == 'amd64') or (python_full_version >= '3.10' and platform_machine == 'arm64') or (python_full_version >= '3.10' and platform_machine == 'x86_64')" },
+    { name = "packaging", marker = "python_full_version >= '3.10'" },
+    { name = "pyyaml", marker = "python_full_version >= '3.10'" },
+    { name = "requests", marker = "python_full_version >= '3.10'" },
+    { name = "tqdm", marker = "python_full_version >= '3.10'" },
+    { name = "typing-extensions", marker = "python_full_version >= '3.10'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/10/7e/a0a97de7c73671863ca6b3f61fa12518caf35db37825e43d63a70956738c/huggingface_hub-0.35.3.tar.gz", hash = "sha256:350932eaa5cc6a4747efae85126ee220e4ef1b54e29d31c3b45c5612ddf0b32a", size = 461798 }
+sdist = { url = "https://files.pythonhosted.org/packages/10/7e/a0a97de7c73671863ca6b3f61fa12518caf35db37825e43d63a70956738c/huggingface_hub-0.35.3.tar.gz", hash = "sha256:350932eaa5cc6a4747efae85126ee220e4ef1b54e29d31c3b45c5612ddf0b32a", size = 461798, upload-time = "2025-09-29T14:29:58.625Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/31/a0/651f93d154cb72323358bf2bbae3e642bdb5d2f1bfc874d096f7cb159fa0/huggingface_hub-0.35.3-py3-none-any.whl", hash = "sha256:0e3a01829c19d86d03793e4577816fe3bdfc1602ac62c7fb220d593d351224ba", size = 564262 },
+    { url = "https://files.pythonhosted.org/packages/31/a0/651f93d154cb72323358bf2bbae3e642bdb5d2f1bfc874d096f7cb159fa0/huggingface_hub-0.35.3-py3-none-any.whl", hash = "sha256:0e3a01829c19d86d03793e4577816fe3bdfc1602ac62c7fb220d593d351224ba", size = 564262, upload-time = "2025-09-29T14:29:55.813Z" },
 ]
 
 [[package]]
 name = "idna"
 version = "3.10"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490 }
+sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" },
+]
+
+[[package]]
+name = "importlib-metadata"
+version = "8.7.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "zipp", marker = "python_full_version < '3.10'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 },
+    { url = "https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151", size = 27865, upload-time = "2025-12-21T10:00:18.329Z" },
 ]
 
 [[package]]
 name = "iniconfig"
 version = "2.1.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793 }
+sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050 },
+    { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" },
 ]
 
 [[package]]
@@ -820,86 +1016,121 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "markupsafe" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115 }
+sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899 },
+    { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" },
 ]
 
 [[package]]
 name = "jmespath"
 version = "1.0.1"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843 }
+sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256 },
+    { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" },
+]
+
+[[package]]
+name = "keras"
+version = "2.7.0"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/6b/8b/065f94ba03282fa41b2d76942b87a180a9913312c4611ea7d6508fbbc114/keras-2.7.0-py2.py3-none-any.whl", hash = "sha256:0c33ae1f728064ca0d35dfba999e9c316f03623bf5688c82fb83cc74a80ea248", size = 1332171, upload-time = "2021-11-03T16:16:34.318Z" },
 ]
 
 [[package]]
 name = "keras"
 version = "3.11.3"
 source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version >= '3.14'",
+    "python_full_version == '3.13.*'",
+    "python_full_version == '3.12.*'",
+    "python_full_version == '3.11.*'",
+    "python_full_version == '3.10.*'",
+]
 dependencies = [
-    { name = "absl-py" },
-    { name = "h5py" },
-    { name = "ml-dtypes" },
-    { name = "namex" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "absl-py", marker = "python_full_version >= '3.10'" },
+    { name = "h5py", marker = "python_full_version >= '3.10'" },
+    { name = "ml-dtypes", marker = "python_full_version >= '3.10'" },
+    { name = "namex", marker = "python_full_version >= '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "optree" },
-    { name = "packaging" },
-    { name = "rich" },
+    { name = "optree", marker = "python_full_version >= '3.10'" },
+    { name = "packaging", marker = "python_full_version >= '3.10'" },
+    { name = "rich", marker = "python_full_version >= '3.10'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/6a/89/646425fe9a46f9053430e1271f817c36041c6f33469950a3caafc3d2591e/keras-3.11.3.tar.gz", hash = "sha256:efda616835c31b7d916d72303ef9adec1257320bc9fd4b2b0138840fc65fb5b7", size = 1065906, upload-time = "2025-08-21T22:08:57.643Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/94/5b/4c778cc921ce4b864b238f63f8e3ff6e954ab19b80c9fa680593ad8093d4/keras-3.11.3-py3-none-any.whl", hash = "sha256:f484f050e05ee400455b05ec8c36ed35edc34de94256b6073f56cfe68f65491f", size = 1408438, upload-time = "2025-08-21T22:08:55.858Z" },
+]
+
+[[package]]
+name = "keras-preprocessing"
+version = "1.1.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "six", marker = "python_full_version < '3.10'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/6a/89/646425fe9a46f9053430e1271f817c36041c6f33469950a3caafc3d2591e/keras-3.11.3.tar.gz", hash = "sha256:efda616835c31b7d916d72303ef9adec1257320bc9fd4b2b0138840fc65fb5b7", size = 1065906 }
+sdist = { url = "https://files.pythonhosted.org/packages/5e/f1/b44337faca48874333769a29398fe4666686733c8880aa160b9fd5dfe600/Keras_Preprocessing-1.1.2.tar.gz", hash = "sha256:add82567c50c8bc648c14195bf544a5ce7c1f76761536956c3d2978970179ef3", size = 163598, upload-time = "2020-05-14T03:53:48.526Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/94/5b/4c778cc921ce4b864b238f63f8e3ff6e954ab19b80c9fa680593ad8093d4/keras-3.11.3-py3-none-any.whl", hash = "sha256:f484f050e05ee400455b05ec8c36ed35edc34de94256b6073f56cfe68f65491f", size = 1408438 },
+    { url = "https://files.pythonhosted.org/packages/79/4c/7c3275a01e12ef9368a892926ab932b33bb13d55794881e3573482b378a7/Keras_Preprocessing-1.1.2-py2.py3-none-any.whl", hash = "sha256:7b82029b130ff61cc99b55f3bd27427df4838576838c5b2f65940e4fcec99a7b", size = 42581, upload-time = "2020-05-14T03:53:47.192Z" },
 ]
 
 [[package]]
 name = "lance-namespace"
-version = "0.5.2"
+version = "0.6.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "lance-namespace-urllib3-client" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/2b/c6/aec0d7752e15536564b50cf9a8926f0e5d7780aa3ab8ce8bca46daa55659/lance_namespace-0.5.2.tar.gz", hash = "sha256:566cc33091b5631793ab411f095d46c66391db0a62343cd6b4470265bb04d577", size = 10274 }
+sdist = { url = "https://files.pythonhosted.org/packages/28/9f/7906ba4117df8d965510285eaf07264a77de2fd283b9d44ec7fc63a4a57a/lance_namespace-0.6.1.tar.gz", hash = "sha256:f0deea442bd3f1056a8e2fed056ae2778e3356517ec2e680db049058b824d131", size = 10666, upload-time = "2026-03-17T17:55:44.977Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/d6/3d/737c008d8fb2861e7ce260e2ffab0d5058eae41556181f80f1a1c3b52ef5/lance_namespace-0.5.2-py3-none-any.whl", hash = "sha256:6ccaf5649bf6ee6aa92eed9c535a114b7b4eb08e89f40426f58bc1466cbcffa3", size = 12087 },
+    { url = "https://files.pythonhosted.org/packages/d1/91/aee1c0a04d17f2810173bd304bd444eb78332045df1b0c1b07cebd01f530/lance_namespace-0.6.1-py3-none-any.whl", hash = "sha256:9699c9e3f12236e5e08ea979cc4e036a8e3c67ed2f37ae6f25c5353ab908e1be", size = 12498, upload-time = "2026-03-17T17:55:44.062Z" },
 ]
 
 [[package]]
 name = "lance-namespace-urllib3-client"
-version = "0.5.2"
+version = "0.6.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "pydantic" },
     { name = "python-dateutil" },
     { name = "typing-extensions" },
-    { name = "urllib3" },
+    { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "urllib3", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/e9/64/51622c93ec8c164483c83b68764e5e76e52286c0137a8247bc6a7fac25f4/lance_namespace_urllib3_client-0.5.2.tar.gz", hash = "sha256:8a3a238006e6eabc01fc9d385ac3de22ba933aef0ae8987558f3c3199c9b3799", size = 172578 }
+sdist = { url = "https://files.pythonhosted.org/packages/63/a1/8706a2be25bd184acccc411e48f1a42a4cbf3b6556cba15b9fcf4c15cfcc/lance_namespace_urllib3_client-0.6.1.tar.gz", hash = "sha256:31fbd058ce1ea0bf49045cdeaa756360ece0bc61e9e10276f41af6d217debe87", size = 182567, upload-time = "2026-03-17T17:55:46.87Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/2a/10/f86d994498b37f7f35d0b8c2f7626a16fe4cb1949b518c1e5d5052ecf95f/lance_namespace_urllib3_client-0.5.2-py3-none-any.whl", hash = "sha256:83cefb6fd6e5df0b99b5e866ee3d46300d375b75e8af32c27bc16fbf7c1a5978", size = 300351 },
+    { url = "https://files.pythonhosted.org/packages/cd/c7/cb9580602dec25f0fdd6005c1c9ba1d4c8c0c3dc8d543107e5a9f248bba8/lance_namespace_urllib3_client-0.6.1-py3-none-any.whl", hash = "sha256:b9c103e1377ad46d2bd70eec894bfec0b1e2133dae0964d7e4de543c6e16293b", size = 317111, upload-time = "2026-03-17T17:55:45.546Z" },
 ]
 
 [[package]]
 name = "libclang"
 version = "18.1.1"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/6e/5c/ca35e19a4f142adffa27e3d652196b7362fa612243e2b916845d801454fc/libclang-18.1.1.tar.gz", hash = "sha256:a1214966d08d73d971287fc3ead8dfaf82eb07fb197680d8b3859dbbbbf78250", size = 39612 }
+sdist = { url = "https://files.pythonhosted.org/packages/6e/5c/ca35e19a4f142adffa27e3d652196b7362fa612243e2b916845d801454fc/libclang-18.1.1.tar.gz", hash = "sha256:a1214966d08d73d971287fc3ead8dfaf82eb07fb197680d8b3859dbbbbf78250", size = 39612, upload-time = "2024-03-17T16:04:37.434Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/1d/fc/716c1e62e512ef1c160e7984a73a5fc7df45166f2ff3f254e71c58076f7c/libclang-18.1.1-py2.py3-none-manylinux2010_x86_64.whl", hash = "sha256:c533091d8a3bbf7460a00cb6c1a71da93bffe148f172c7d03b1c31fbf8aa2a0b", size = 24515943 },
-    { url = "https://files.pythonhosted.org/packages/3c/3d/f0ac1150280d8d20d059608cf2d5ff61b7c3b7f7bcf9c0f425ab92df769a/libclang-18.1.1-py2.py3-none-manylinux2014_aarch64.whl", hash = "sha256:54dda940a4a0491a9d1532bf071ea3ef26e6dbaf03b5000ed94dd7174e8f9592", size = 23784972 },
-    { url = "https://files.pythonhosted.org/packages/fe/2f/d920822c2b1ce9326a4c78c0c2b4aa3fde610c7ee9f631b600acb5376c26/libclang-18.1.1-py2.py3-none-manylinux2014_armv7l.whl", hash = "sha256:cf4a99b05376513717ab5d82a0db832c56ccea4fd61a69dbb7bccf2dfb207dbe", size = 20259606 },
-    { url = "https://files.pythonhosted.org/packages/2d/c2/de1db8c6d413597076a4259cea409b83459b2db997c003578affdd32bf66/libclang-18.1.1-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:69f8eb8f65c279e765ffd28aaa7e9e364c776c17618af8bff22a8df58677ff4f", size = 24921494 },
+    { url = "https://files.pythonhosted.org/packages/1d/fc/716c1e62e512ef1c160e7984a73a5fc7df45166f2ff3f254e71c58076f7c/libclang-18.1.1-py2.py3-none-manylinux2010_x86_64.whl", hash = "sha256:c533091d8a3bbf7460a00cb6c1a71da93bffe148f172c7d03b1c31fbf8aa2a0b", size = 24515943, upload-time = "2024-03-17T16:03:45.942Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/3d/f0ac1150280d8d20d059608cf2d5ff61b7c3b7f7bcf9c0f425ab92df769a/libclang-18.1.1-py2.py3-none-manylinux2014_aarch64.whl", hash = "sha256:54dda940a4a0491a9d1532bf071ea3ef26e6dbaf03b5000ed94dd7174e8f9592", size = 23784972, upload-time = "2024-03-17T16:12:47.677Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/2f/d920822c2b1ce9326a4c78c0c2b4aa3fde610c7ee9f631b600acb5376c26/libclang-18.1.1-py2.py3-none-manylinux2014_armv7l.whl", hash = "sha256:cf4a99b05376513717ab5d82a0db832c56ccea4fd61a69dbb7bccf2dfb207dbe", size = 20259606, upload-time = "2024-03-17T16:17:42.437Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/c2/de1db8c6d413597076a4259cea409b83459b2db997c003578affdd32bf66/libclang-18.1.1-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:69f8eb8f65c279e765ffd28aaa7e9e364c776c17618af8bff22a8df58677ff4f", size = 24921494, upload-time = "2024-03-17T16:14:20.132Z" },
 ]
 
 [[package]]
 name = "markdown"
 version = "3.9"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/8d/37/02347f6d6d8279247a5837082ebc26fc0d5aaeaf75aa013fcbb433c777ab/markdown-3.9.tar.gz", hash = "sha256:d2900fe1782bd33bdbbd56859defef70c2e78fc46668f8eb9df3128138f2cb6a", size = 364585 }
+dependencies = [
+    { name = "importlib-metadata", marker = "python_full_version < '3.10'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8d/37/02347f6d6d8279247a5837082ebc26fc0d5aaeaf75aa013fcbb433c777ab/markdown-3.9.tar.gz", hash = "sha256:d2900fe1782bd33bdbbd56859defef70c2e78fc46668f8eb9df3128138f2cb6a", size = 364585, upload-time = "2025-09-04T20:25:22.885Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl", hash = "sha256:9f4d91ed810864ea88a6f32c07ba8bee1346c0cc1f6b1f9f6c822f2a9667d280", size = 107441 },
+    { url = "https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl", hash = "sha256:9f4d91ed810864ea88a6f32c07ba8bee1346c0cc1f6b1f9f6c822f2a9667d280", size = 107441, upload-time = "2025-09-04T20:25:21.784Z" },
 ]
 
 [[package]]
@@ -907,105 +1138,116 @@ name = "markdown-it-py"
 version = "4.0.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "mdurl" },
+    { name = "mdurl", marker = "python_full_version >= '3.10'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070 }
+sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321 },
+    { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" },
 ]
 
 [[package]]
 name = "markupsafe"
 version = "3.0.3"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/e8/4b/3541d44f3937ba468b75da9eebcae497dcf67adb65caa16760b0a6807ebb/markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559", size = 11631 },
-    { url = "https://files.pythonhosted.org/packages/98/1b/fbd8eed11021cabd9226c37342fa6ca4e8a98d8188a8d9b66740494960e4/markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419", size = 12057 },
-    { url = "https://files.pythonhosted.org/packages/40/01/e560d658dc0bb8ab762670ece35281dec7b6c1b33f5fbc09ebb57a185519/markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695", size = 22050 },
-    { url = "https://files.pythonhosted.org/packages/af/cd/ce6e848bbf2c32314c9b237839119c5a564a59725b53157c856e90937b7a/markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591", size = 20681 },
-    { url = "https://files.pythonhosted.org/packages/c9/2a/b5c12c809f1c3045c4d580b035a743d12fcde53cf685dbc44660826308da/markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c", size = 20705 },
-    { url = "https://files.pythonhosted.org/packages/cf/e3/9427a68c82728d0a88c50f890d0fc072a1484de2f3ac1ad0bfc1a7214fd5/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f", size = 21524 },
-    { url = "https://files.pythonhosted.org/packages/bc/36/23578f29e9e582a4d0278e009b38081dbe363c5e7165113fad546918a232/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6", size = 20282 },
-    { url = "https://files.pythonhosted.org/packages/56/21/dca11354e756ebd03e036bd8ad58d6d7168c80ce1fe5e75218e4945cbab7/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1", size = 20745 },
-    { url = "https://files.pythonhosted.org/packages/87/99/faba9369a7ad6e4d10b6a5fbf71fa2a188fe4a593b15f0963b73859a1bbd/markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa", size = 14571 },
-    { url = "https://files.pythonhosted.org/packages/d6/25/55dc3ab959917602c96985cb1253efaa4ff42f71194bddeb61eb7278b8be/markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8", size = 15056 },
-    { url = "https://files.pythonhosted.org/packages/d0/9e/0a02226640c255d1da0b8d12e24ac2aa6734da68bff14c05dd53b94a0fc3/markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1", size = 13932 },
-    { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631 },
-    { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058 },
-    { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287 },
-    { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940 },
-    { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887 },
-    { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692 },
-    { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471 },
-    { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923 },
-    { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572 },
-    { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077 },
-    { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876 },
-    { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615 },
-    { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020 },
-    { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332 },
-    { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947 },
-    { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962 },
-    { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760 },
-    { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529 },
-    { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015 },
-    { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540 },
-    { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105 },
-    { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906 },
-    { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622 },
-    { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029 },
-    { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374 },
-    { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980 },
-    { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990 },
-    { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784 },
-    { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588 },
-    { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041 },
-    { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543 },
-    { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113 },
-    { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911 },
-    { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658 },
-    { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066 },
-    { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639 },
-    { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569 },
-    { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284 },
-    { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801 },
-    { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769 },
-    { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642 },
-    { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612 },
-    { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200 },
-    { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973 },
-    { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619 },
-    { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029 },
-    { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408 },
-    { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005 },
-    { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048 },
-    { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821 },
-    { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606 },
-    { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043 },
-    { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747 },
-    { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341 },
-    { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073 },
-    { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661 },
-    { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069 },
-    { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670 },
-    { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598 },
-    { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261 },
-    { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835 },
-    { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733 },
-    { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672 },
-    { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819 },
-    { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426 },
-    { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146 },
+sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e8/4b/3541d44f3937ba468b75da9eebcae497dcf67adb65caa16760b0a6807ebb/markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559", size = 11631, upload-time = "2025-09-27T18:36:05.558Z" },
+    { url = "https://files.pythonhosted.org/packages/98/1b/fbd8eed11021cabd9226c37342fa6ca4e8a98d8188a8d9b66740494960e4/markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419", size = 12057, upload-time = "2025-09-27T18:36:07.165Z" },
+    { url = "https://files.pythonhosted.org/packages/40/01/e560d658dc0bb8ab762670ece35281dec7b6c1b33f5fbc09ebb57a185519/markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695", size = 22050, upload-time = "2025-09-27T18:36:08.005Z" },
+    { url = "https://files.pythonhosted.org/packages/af/cd/ce6e848bbf2c32314c9b237839119c5a564a59725b53157c856e90937b7a/markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591", size = 20681, upload-time = "2025-09-27T18:36:08.881Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/2a/b5c12c809f1c3045c4d580b035a743d12fcde53cf685dbc44660826308da/markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c", size = 20705, upload-time = "2025-09-27T18:36:10.131Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/e3/9427a68c82728d0a88c50f890d0fc072a1484de2f3ac1ad0bfc1a7214fd5/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f", size = 21524, upload-time = "2025-09-27T18:36:11.324Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/36/23578f29e9e582a4d0278e009b38081dbe363c5e7165113fad546918a232/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6", size = 20282, upload-time = "2025-09-27T18:36:12.573Z" },
+    { url = "https://files.pythonhosted.org/packages/56/21/dca11354e756ebd03e036bd8ad58d6d7168c80ce1fe5e75218e4945cbab7/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1", size = 20745, upload-time = "2025-09-27T18:36:13.504Z" },
+    { url = "https://files.pythonhosted.org/packages/87/99/faba9369a7ad6e4d10b6a5fbf71fa2a188fe4a593b15f0963b73859a1bbd/markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa", size = 14571, upload-time = "2025-09-27T18:36:14.779Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/25/55dc3ab959917602c96985cb1253efaa4ff42f71194bddeb61eb7278b8be/markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8", size = 15056, upload-time = "2025-09-27T18:36:16.125Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/9e/0a02226640c255d1da0b8d12e24ac2aa6734da68bff14c05dd53b94a0fc3/markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1", size = 13932, upload-time = "2025-09-27T18:36:17.311Z" },
+    { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" },
+    { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" },
+    { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" },
+    { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" },
+    { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" },
+    { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" },
+    { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" },
+    { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" },
+    { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" },
+    { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" },
+    { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" },
+    { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" },
+    { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" },
+    { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" },
+    { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" },
+    { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" },
+    { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" },
+    { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" },
+    { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" },
+    { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" },
+    { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" },
+    { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" },
+    { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" },
+    { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" },
+    { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" },
+    { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" },
+    { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" },
+    { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" },
+    { url = "https://files.pythonhosted.org/packages/56/23/0d8c13a44bde9154821586520840643467aee574d8ce79a17da539ee7fed/markupsafe-3.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26", size = 11623, upload-time = "2025-09-27T18:37:29.296Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/23/07a2cb9a8045d5f3f0890a8c3bc0859d7a47bfd9a560b563899bec7b72ed/markupsafe-3.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc", size = 12049, upload-time = "2025-09-27T18:37:30.234Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/e4/6be85eb81503f8e11b61c0b6369b6e077dcf0a74adbd9ebf6b349937b4e9/markupsafe-3.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c", size = 21923, upload-time = "2025-09-27T18:37:31.177Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/bc/4dc914ead3fe6ddaef035341fee0fc956949bbd27335b611829292b89ee2/markupsafe-3.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42", size = 20543, upload-time = "2025-09-27T18:37:32.168Z" },
+    { url = "https://files.pythonhosted.org/packages/89/6e/5fe81fbcfba4aef4093d5f856e5c774ec2057946052d18d168219b7bd9f9/markupsafe-3.0.3-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b", size = 20585, upload-time = "2025-09-27T18:37:33.166Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/f6/e0e5a3d3ae9c4020f696cd055f940ef86b64fe88de26f3a0308b9d3d048c/markupsafe-3.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758", size = 21387, upload-time = "2025-09-27T18:37:34.185Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/25/651753ef4dea08ea790f4fbb65146a9a44a014986996ca40102e237aa49a/markupsafe-3.0.3-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2", size = 20133, upload-time = "2025-09-27T18:37:35.138Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/0a/c3cf2b4fef5f0426e8a6d7fce3cb966a17817c568ce59d76b92a233fdbec/markupsafe-3.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d", size = 20588, upload-time = "2025-09-27T18:37:36.096Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/1b/a7782984844bd519ad4ffdbebbba2671ec5d0ebbeac34736c15fb86399e8/markupsafe-3.0.3-cp39-cp39-win32.whl", hash = "sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7", size = 14566, upload-time = "2025-09-27T18:37:37.09Z" },
+    { url = "https://files.pythonhosted.org/packages/18/1f/8d9c20e1c9440e215a44be5ab64359e207fcb4f675543f1cf9a2a7f648d0/markupsafe-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e", size = 15053, upload-time = "2025-09-27T18:37:38.054Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/d3/fe08482b5cd995033556d45041a4f4e76e7f0521112a9c9991d40d39825f/markupsafe-3.0.3-cp39-cp39-win_arm64.whl", hash = "sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8", size = 13928, upload-time = "2025-09-27T18:37:39.037Z" },
 ]
 
 [[package]]
 name = "mdurl"
 version = "0.1.2"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729 }
+sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979 },
+    { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
 ]
 
 [[package]]
@@ -1013,50 +1255,55 @@ name = "ml-dtypes"
 version = "0.5.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/78/a7/aad060393123cfb383956dca68402aff3db1e1caffd5764887ed5153f41b/ml_dtypes-0.5.3.tar.gz", hash = "sha256:95ce33057ba4d05df50b1f3cfefab22e351868a843b3b15a46c65836283670c9", size = 692316 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ac/bb/1f32124ab6d3a279ea39202fe098aea95b2d81ef0ce1d48612b6bf715e82/ml_dtypes-0.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0a1d68a7cb53e3f640b2b6a34d12c0542da3dd935e560fdf463c0c77f339fc20", size = 667409 },
-    { url = "https://files.pythonhosted.org/packages/1d/ac/e002d12ae19136e25bb41c7d14d7e1a1b08f3c0e99a44455ff6339796507/ml_dtypes-0.5.3-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cd5a6c711b5350f3cbc2ac28def81cd1c580075ccb7955e61e9d8f4bfd40d24", size = 4960702 },
-    { url = "https://files.pythonhosted.org/packages/dd/12/79e9954e6b3255a4b1becb191a922d6e2e94d03d16a06341ae9261963ae8/ml_dtypes-0.5.3-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bdcf26c2dbc926b8a35ec8cbfad7eff1a8bd8239e12478caca83a1fc2c400dc2", size = 4933471 },
-    { url = "https://files.pythonhosted.org/packages/d5/aa/d1eff619e83cd1ddf6b561d8240063d978e5d887d1861ba09ef01778ec3a/ml_dtypes-0.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:aecbd7c5272c82e54d5b99d8435fd10915d1bc704b7df15e4d9ca8dc3902be61", size = 206330 },
-    { url = "https://files.pythonhosted.org/packages/af/f1/720cb1409b5d0c05cff9040c0e9fba73fa4c67897d33babf905d5d46a070/ml_dtypes-0.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4a177b882667c69422402df6ed5c3428ce07ac2c1f844d8a1314944651439458", size = 667412 },
-    { url = "https://files.pythonhosted.org/packages/6a/d5/05861ede5d299f6599f86e6bc1291714e2116d96df003cfe23cc54bcc568/ml_dtypes-0.5.3-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9849ce7267444c0a717c80c6900997de4f36e2815ce34ac560a3edb2d9a64cd2", size = 4964606 },
-    { url = "https://files.pythonhosted.org/packages/db/dc/72992b68de367741bfab8df3b3fe7c29f982b7279d341aa5bf3e7ef737ea/ml_dtypes-0.5.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c3f5ae0309d9f888fd825c2e9d0241102fadaca81d888f26f845bc8c13c1e4ee", size = 4938435 },
-    { url = "https://files.pythonhosted.org/packages/81/1c/d27a930bca31fb07d975a2d7eaf3404f9388114463b9f15032813c98f893/ml_dtypes-0.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:58e39349d820b5702bb6f94ea0cb2dc8ec62ee81c0267d9622067d8333596a46", size = 206334 },
-    { url = "https://files.pythonhosted.org/packages/1a/d8/6922499effa616012cb8dc445280f66d100a7ff39b35c864cfca019b3f89/ml_dtypes-0.5.3-cp311-cp311-win_arm64.whl", hash = "sha256:66c2756ae6cfd7f5224e355c893cfd617fa2f747b8bbd8996152cbdebad9a184", size = 157584 },
-    { url = "https://files.pythonhosted.org/packages/0d/eb/bc07c88a6ab002b4635e44585d80fa0b350603f11a2097c9d1bfacc03357/ml_dtypes-0.5.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:156418abeeda48ea4797db6776db3c5bdab9ac7be197c1233771e0880c304057", size = 663864 },
-    { url = "https://files.pythonhosted.org/packages/cf/89/11af9b0f21b99e6386b6581ab40fb38d03225f9de5f55cf52097047e2826/ml_dtypes-0.5.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1db60c154989af253f6c4a34e8a540c2c9dce4d770784d426945e09908fbb177", size = 4951313 },
-    { url = "https://files.pythonhosted.org/packages/d8/a9/b98b86426c24900b0c754aad006dce2863df7ce0bb2bcc2c02f9cc7e8489/ml_dtypes-0.5.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1b255acada256d1fa8c35ed07b5f6d18bc21d1556f842fbc2d5718aea2cd9e55", size = 4928805 },
-    { url = "https://files.pythonhosted.org/packages/50/c1/85e6be4fc09c6175f36fb05a45917837f30af9a5146a5151cb3a3f0f9e09/ml_dtypes-0.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:da65e5fd3eea434ccb8984c3624bc234ddcc0d9f4c81864af611aaebcc08a50e", size = 208182 },
-    { url = "https://files.pythonhosted.org/packages/9e/17/cf5326d6867be057f232d0610de1458f70a8ce7b6290e4b4a277ea62b4cd/ml_dtypes-0.5.3-cp312-cp312-win_arm64.whl", hash = "sha256:8bb9cd1ce63096567f5f42851f5843b5a0ea11511e50039a7649619abfb4ba6d", size = 161560 },
-    { url = "https://files.pythonhosted.org/packages/2d/87/1bcc98a66de7b2455dfb292f271452cac9edc4e870796e0d87033524d790/ml_dtypes-0.5.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5103856a225465371fe119f2fef737402b705b810bd95ad5f348e6e1a6ae21af", size = 663781 },
-    { url = "https://files.pythonhosted.org/packages/fd/2c/bd2a79ba7c759ee192b5601b675b180a3fd6ccf48ffa27fe1782d280f1a7/ml_dtypes-0.5.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4cae435a68861660af81fa3c5af16b70ca11a17275c5b662d9c6f58294e0f113", size = 4956217 },
-    { url = "https://files.pythonhosted.org/packages/14/f3/091ba84e5395d7fe5b30c081a44dec881cd84b408db1763ee50768b2ab63/ml_dtypes-0.5.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6936283b56d74fbec431ca57ce58a90a908fdbd14d4e2d22eea6d72bb208a7b7", size = 4933109 },
-    { url = "https://files.pythonhosted.org/packages/bc/24/054036dbe32c43295382c90a1363241684c4d6aaa1ecc3df26bd0c8d5053/ml_dtypes-0.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:d0f730a17cf4f343b2c7ad50cee3bd19e969e793d2be6ed911f43086460096e4", size = 208187 },
-    { url = "https://files.pythonhosted.org/packages/a6/3d/7dc3ec6794a4a9004c765e0c341e32355840b698f73fd2daff46f128afc1/ml_dtypes-0.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:2db74788fc01914a3c7f7da0763427280adfc9cd377e9604b6b64eb8097284bd", size = 161559 },
-    { url = "https://files.pythonhosted.org/packages/12/91/e6c7a0d67a152b9330445f9f0cf8ae6eee9b83f990b8c57fe74631e42a90/ml_dtypes-0.5.3-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:93c36a08a6d158db44f2eb9ce3258e53f24a9a4a695325a689494f0fdbc71770", size = 689321 },
-    { url = "https://files.pythonhosted.org/packages/9e/6c/b7b94b84a104a5be1883305b87d4c6bd6ae781504474b4cca067cb2340ec/ml_dtypes-0.5.3-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e44a3761f64bc009d71ddb6d6c71008ba21b53ab6ee588dadab65e2fa79eafc", size = 5274495 },
-    { url = "https://files.pythonhosted.org/packages/5b/38/6266604dffb43378055394ea110570cf261a49876fc48f548dfe876f34cc/ml_dtypes-0.5.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bdf40d2aaabd3913dec11840f0d0ebb1b93134f99af6a0a4fd88ffe924928ab4", size = 5285422 },
-    { url = "https://files.pythonhosted.org/packages/7c/88/8612ff177d043a474b9408f0382605d881eeb4125ba89d4d4b3286573a83/ml_dtypes-0.5.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:aec640bd94c4c85c0d11e2733bd13cbb10438fb004852996ec0efbc6cacdaf70", size = 661182 },
-    { url = "https://files.pythonhosted.org/packages/6f/2b/0569a5e88b29240d373e835107c94ae9256fb2191d3156b43b2601859eff/ml_dtypes-0.5.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bda32ce212baa724e03c68771e5c69f39e584ea426bfe1a701cb01508ffc7035", size = 4956187 },
-    { url = "https://files.pythonhosted.org/packages/51/66/273c2a06ae44562b104b61e6b14444da00061fd87652506579d7eb2c40b1/ml_dtypes-0.5.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c205cac07d24a29840c163d6469f61069ce4b065518519216297fc2f261f8db9", size = 4930911 },
-    { url = "https://files.pythonhosted.org/packages/93/ab/606be3e87dc0821bd360c8c1ee46108025c31a4f96942b63907bb441b87d/ml_dtypes-0.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:cd7c0bb22d4ff86d65ad61b5dd246812e8993fbc95b558553624c33e8b6903ea", size = 216664 },
-    { url = "https://files.pythonhosted.org/packages/30/a2/e900690ca47d01dffffd66375c5de8c4f8ced0f1ef809ccd3b25b3e6b8fa/ml_dtypes-0.5.3-cp314-cp314-win_arm64.whl", hash = "sha256:9d55ea7f7baf2aed61bf1872116cefc9d0c3693b45cae3916897ee27ef4b835e", size = 160203 },
-    { url = "https://files.pythonhosted.org/packages/53/21/783dfb51f40d2660afeb9bccf3612b99f6a803d980d2a09132b0f9d216ab/ml_dtypes-0.5.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:e12e29764a0e66a7a31e9b8bf1de5cc0423ea72979f45909acd4292de834ccd3", size = 689324 },
-    { url = "https://files.pythonhosted.org/packages/09/f7/a82d249c711abf411ac027b7163f285487f5e615c3e0716c61033ce996ab/ml_dtypes-0.5.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19f6c3a4f635c2fc9e2aa7d91416bd7a3d649b48350c51f7f715a09370a90d93", size = 5275917 },
-    { url = "https://files.pythonhosted.org/packages/7f/3c/541c4b30815ab90ebfbb51df15d0b4254f2f9f1e2b4907ab229300d5e6f2/ml_dtypes-0.5.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ab039ffb40f3dc0aeeeba84fd6c3452781b5e15bef72e2d10bcb33e4bbffc39", size = 5285284 },
+sdist = { url = "https://files.pythonhosted.org/packages/78/a7/aad060393123cfb383956dca68402aff3db1e1caffd5764887ed5153f41b/ml_dtypes-0.5.3.tar.gz", hash = "sha256:95ce33057ba4d05df50b1f3cfefab22e351868a843b3b15a46c65836283670c9", size = 692316, upload-time = "2025-07-29T18:39:19.454Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ac/bb/1f32124ab6d3a279ea39202fe098aea95b2d81ef0ce1d48612b6bf715e82/ml_dtypes-0.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0a1d68a7cb53e3f640b2b6a34d12c0542da3dd935e560fdf463c0c77f339fc20", size = 667409, upload-time = "2025-07-29T18:38:17.321Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/ac/e002d12ae19136e25bb41c7d14d7e1a1b08f3c0e99a44455ff6339796507/ml_dtypes-0.5.3-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cd5a6c711b5350f3cbc2ac28def81cd1c580075ccb7955e61e9d8f4bfd40d24", size = 4960702, upload-time = "2025-07-29T18:38:19.616Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/12/79e9954e6b3255a4b1becb191a922d6e2e94d03d16a06341ae9261963ae8/ml_dtypes-0.5.3-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bdcf26c2dbc926b8a35ec8cbfad7eff1a8bd8239e12478caca83a1fc2c400dc2", size = 4933471, upload-time = "2025-07-29T18:38:21.809Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/aa/d1eff619e83cd1ddf6b561d8240063d978e5d887d1861ba09ef01778ec3a/ml_dtypes-0.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:aecbd7c5272c82e54d5b99d8435fd10915d1bc704b7df15e4d9ca8dc3902be61", size = 206330, upload-time = "2025-07-29T18:38:23.663Z" },
+    { url = "https://files.pythonhosted.org/packages/af/f1/720cb1409b5d0c05cff9040c0e9fba73fa4c67897d33babf905d5d46a070/ml_dtypes-0.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4a177b882667c69422402df6ed5c3428ce07ac2c1f844d8a1314944651439458", size = 667412, upload-time = "2025-07-29T18:38:25.275Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/d5/05861ede5d299f6599f86e6bc1291714e2116d96df003cfe23cc54bcc568/ml_dtypes-0.5.3-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9849ce7267444c0a717c80c6900997de4f36e2815ce34ac560a3edb2d9a64cd2", size = 4964606, upload-time = "2025-07-29T18:38:27.045Z" },
+    { url = "https://files.pythonhosted.org/packages/db/dc/72992b68de367741bfab8df3b3fe7c29f982b7279d341aa5bf3e7ef737ea/ml_dtypes-0.5.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c3f5ae0309d9f888fd825c2e9d0241102fadaca81d888f26f845bc8c13c1e4ee", size = 4938435, upload-time = "2025-07-29T18:38:29.193Z" },
+    { url = "https://files.pythonhosted.org/packages/81/1c/d27a930bca31fb07d975a2d7eaf3404f9388114463b9f15032813c98f893/ml_dtypes-0.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:58e39349d820b5702bb6f94ea0cb2dc8ec62ee81c0267d9622067d8333596a46", size = 206334, upload-time = "2025-07-29T18:38:30.687Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/d8/6922499effa616012cb8dc445280f66d100a7ff39b35c864cfca019b3f89/ml_dtypes-0.5.3-cp311-cp311-win_arm64.whl", hash = "sha256:66c2756ae6cfd7f5224e355c893cfd617fa2f747b8bbd8996152cbdebad9a184", size = 157584, upload-time = "2025-07-29T18:38:32.187Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/eb/bc07c88a6ab002b4635e44585d80fa0b350603f11a2097c9d1bfacc03357/ml_dtypes-0.5.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:156418abeeda48ea4797db6776db3c5bdab9ac7be197c1233771e0880c304057", size = 663864, upload-time = "2025-07-29T18:38:33.777Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/89/11af9b0f21b99e6386b6581ab40fb38d03225f9de5f55cf52097047e2826/ml_dtypes-0.5.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1db60c154989af253f6c4a34e8a540c2c9dce4d770784d426945e09908fbb177", size = 4951313, upload-time = "2025-07-29T18:38:36.45Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/a9/b98b86426c24900b0c754aad006dce2863df7ce0bb2bcc2c02f9cc7e8489/ml_dtypes-0.5.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1b255acada256d1fa8c35ed07b5f6d18bc21d1556f842fbc2d5718aea2cd9e55", size = 4928805, upload-time = "2025-07-29T18:38:38.29Z" },
+    { url = "https://files.pythonhosted.org/packages/50/c1/85e6be4fc09c6175f36fb05a45917837f30af9a5146a5151cb3a3f0f9e09/ml_dtypes-0.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:da65e5fd3eea434ccb8984c3624bc234ddcc0d9f4c81864af611aaebcc08a50e", size = 208182, upload-time = "2025-07-29T18:38:39.72Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/17/cf5326d6867be057f232d0610de1458f70a8ce7b6290e4b4a277ea62b4cd/ml_dtypes-0.5.3-cp312-cp312-win_arm64.whl", hash = "sha256:8bb9cd1ce63096567f5f42851f5843b5a0ea11511e50039a7649619abfb4ba6d", size = 161560, upload-time = "2025-07-29T18:38:41.072Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/87/1bcc98a66de7b2455dfb292f271452cac9edc4e870796e0d87033524d790/ml_dtypes-0.5.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5103856a225465371fe119f2fef737402b705b810bd95ad5f348e6e1a6ae21af", size = 663781, upload-time = "2025-07-29T18:38:42.984Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/2c/bd2a79ba7c759ee192b5601b675b180a3fd6ccf48ffa27fe1782d280f1a7/ml_dtypes-0.5.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4cae435a68861660af81fa3c5af16b70ca11a17275c5b662d9c6f58294e0f113", size = 4956217, upload-time = "2025-07-29T18:38:44.65Z" },
+    { url = "https://files.pythonhosted.org/packages/14/f3/091ba84e5395d7fe5b30c081a44dec881cd84b408db1763ee50768b2ab63/ml_dtypes-0.5.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6936283b56d74fbec431ca57ce58a90a908fdbd14d4e2d22eea6d72bb208a7b7", size = 4933109, upload-time = "2025-07-29T18:38:46.405Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/24/054036dbe32c43295382c90a1363241684c4d6aaa1ecc3df26bd0c8d5053/ml_dtypes-0.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:d0f730a17cf4f343b2c7ad50cee3bd19e969e793d2be6ed911f43086460096e4", size = 208187, upload-time = "2025-07-29T18:38:48.24Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/3d/7dc3ec6794a4a9004c765e0c341e32355840b698f73fd2daff46f128afc1/ml_dtypes-0.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:2db74788fc01914a3c7f7da0763427280adfc9cd377e9604b6b64eb8097284bd", size = 161559, upload-time = "2025-07-29T18:38:50.493Z" },
+    { url = "https://files.pythonhosted.org/packages/12/91/e6c7a0d67a152b9330445f9f0cf8ae6eee9b83f990b8c57fe74631e42a90/ml_dtypes-0.5.3-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:93c36a08a6d158db44f2eb9ce3258e53f24a9a4a695325a689494f0fdbc71770", size = 689321, upload-time = "2025-07-29T18:38:52.03Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/6c/b7b94b84a104a5be1883305b87d4c6bd6ae781504474b4cca067cb2340ec/ml_dtypes-0.5.3-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e44a3761f64bc009d71ddb6d6c71008ba21b53ab6ee588dadab65e2fa79eafc", size = 5274495, upload-time = "2025-07-29T18:38:53.797Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/38/6266604dffb43378055394ea110570cf261a49876fc48f548dfe876f34cc/ml_dtypes-0.5.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bdf40d2aaabd3913dec11840f0d0ebb1b93134f99af6a0a4fd88ffe924928ab4", size = 5285422, upload-time = "2025-07-29T18:38:56.603Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/88/8612ff177d043a474b9408f0382605d881eeb4125ba89d4d4b3286573a83/ml_dtypes-0.5.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:aec640bd94c4c85c0d11e2733bd13cbb10438fb004852996ec0efbc6cacdaf70", size = 661182, upload-time = "2025-07-29T18:38:58.414Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/2b/0569a5e88b29240d373e835107c94ae9256fb2191d3156b43b2601859eff/ml_dtypes-0.5.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bda32ce212baa724e03c68771e5c69f39e584ea426bfe1a701cb01508ffc7035", size = 4956187, upload-time = "2025-07-29T18:39:00.611Z" },
+    { url = "https://files.pythonhosted.org/packages/51/66/273c2a06ae44562b104b61e6b14444da00061fd87652506579d7eb2c40b1/ml_dtypes-0.5.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c205cac07d24a29840c163d6469f61069ce4b065518519216297fc2f261f8db9", size = 4930911, upload-time = "2025-07-29T18:39:02.405Z" },
+    { url = "https://files.pythonhosted.org/packages/93/ab/606be3e87dc0821bd360c8c1ee46108025c31a4f96942b63907bb441b87d/ml_dtypes-0.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:cd7c0bb22d4ff86d65ad61b5dd246812e8993fbc95b558553624c33e8b6903ea", size = 216664, upload-time = "2025-07-29T18:39:03.927Z" },
+    { url = "https://files.pythonhosted.org/packages/30/a2/e900690ca47d01dffffd66375c5de8c4f8ced0f1ef809ccd3b25b3e6b8fa/ml_dtypes-0.5.3-cp314-cp314-win_arm64.whl", hash = "sha256:9d55ea7f7baf2aed61bf1872116cefc9d0c3693b45cae3916897ee27ef4b835e", size = 160203, upload-time = "2025-07-29T18:39:05.671Z" },
+    { url = "https://files.pythonhosted.org/packages/53/21/783dfb51f40d2660afeb9bccf3612b99f6a803d980d2a09132b0f9d216ab/ml_dtypes-0.5.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:e12e29764a0e66a7a31e9b8bf1de5cc0423ea72979f45909acd4292de834ccd3", size = 689324, upload-time = "2025-07-29T18:39:07.567Z" },
+    { url = "https://files.pythonhosted.org/packages/09/f7/a82d249c711abf411ac027b7163f285487f5e615c3e0716c61033ce996ab/ml_dtypes-0.5.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19f6c3a4f635c2fc9e2aa7d91416bd7a3d649b48350c51f7f715a09370a90d93", size = 5275917, upload-time = "2025-07-29T18:39:09.339Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/3c/541c4b30815ab90ebfbb51df15d0b4254f2f9f1e2b4907ab229300d5e6f2/ml_dtypes-0.5.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ab039ffb40f3dc0aeeeba84fd6c3452781b5e15bef72e2d10bcb33e4bbffc39", size = 5285284, upload-time = "2025-07-29T18:39:11.532Z" },
+    { url = "https://files.pythonhosted.org/packages/19/2d/c61af51173083bbf2a3b0f1a1a01d50ef1830436880027433d1b75271083/ml_dtypes-0.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5ee72568d46b9533ad54f78b1e1f3067c0534c5065120ea8ecc6f210d22748b3", size = 663552, upload-time = "2025-07-29T18:39:13.102Z" },
+    { url = "https://files.pythonhosted.org/packages/61/0e/a628f2aefd719745e8a13492375a55cedea77c0cfc917b1ce11bde435c68/ml_dtypes-0.5.3-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01de48de4537dc3c46e684b969a40ec36594e7eeb7c69e9a093e7239f030a28a", size = 4952704, upload-time = "2025-07-29T18:39:14.829Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/2e/5ba92f1f99d1f5f62bffec614a5b8161e55c3961257c902fa26dbe909baa/ml_dtypes-0.5.3-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8b1a6e231b0770f2894910f1dce6d2f31d65884dbf7668f9b08d73623cdca909", size = 4923538, upload-time = "2025-07-29T18:39:16.581Z" },
+    { url = "https://files.pythonhosted.org/packages/70/3b/f801c69027866ea6e387224551185fedef62ad8e2e71181ec0d9dda905f7/ml_dtypes-0.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:a4f39b9bf6555fab9bfb536cf5fdd1c1c727e8d22312078702e9ff005354b37f", size = 206567, upload-time = "2025-07-29T18:39:18.047Z" },
 ]
 
 [[package]]
 name = "mpmath"
 version = "1.3.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106 }
+sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198 },
+    { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" },
 ]
 
 [[package]]
@@ -1064,101 +1311,119 @@ name = "multidict"
 version = "6.6.4"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "typing-extensions", marker = "python_full_version < '3.11'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/69/7f/0652e6ed47ab288e3756ea9c0df8b14950781184d4bd7883f4d87dd41245/multidict-6.6.4.tar.gz", hash = "sha256:d2d4e4787672911b48350df02ed3fa3fffdc2f2e8ca06dd6afdf34189b76a9dd", size = 101843 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/eb/6b/86f353088c1358e76fd30b0146947fddecee812703b604ee901e85cd2a80/multidict-6.6.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b8aa6f0bd8125ddd04a6593437bad6a7e70f300ff4180a531654aa2ab3f6d58f", size = 77054 },
-    { url = "https://files.pythonhosted.org/packages/19/5d/c01dc3d3788bb877bd7f5753ea6eb23c1beeca8044902a8f5bfb54430f63/multidict-6.6.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b9e5853bbd7264baca42ffc53391b490d65fe62849bf2c690fa3f6273dbcd0cb", size = 44914 },
-    { url = "https://files.pythonhosted.org/packages/46/44/964dae19ea42f7d3e166474d8205f14bb811020e28bc423d46123ddda763/multidict-6.6.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0af5f9dee472371e36d6ae38bde009bd8ce65ac7335f55dcc240379d7bed1495", size = 44601 },
-    { url = "https://files.pythonhosted.org/packages/31/20/0616348a1dfb36cb2ab33fc9521de1f27235a397bf3f59338e583afadd17/multidict-6.6.4-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:d24f351e4d759f5054b641c81e8291e5d122af0fca5c72454ff77f7cbe492de8", size = 224821 },
-    { url = "https://files.pythonhosted.org/packages/14/26/5d8923c69c110ff51861af05bd27ca6783011b96725d59ccae6d9daeb627/multidict-6.6.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:db6a3810eec08280a172a6cd541ff4a5f6a97b161d93ec94e6c4018917deb6b7", size = 242608 },
-    { url = "https://files.pythonhosted.org/packages/5c/cc/e2ad3ba9459aa34fa65cf1f82a5c4a820a2ce615aacfb5143b8817f76504/multidict-6.6.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a1b20a9d56b2d81e2ff52ecc0670d583eaabaa55f402e8d16dd062373dbbe796", size = 222324 },
-    { url = "https://files.pythonhosted.org/packages/19/db/4ed0f65701afbc2cb0c140d2d02928bb0fe38dd044af76e58ad7c54fd21f/multidict-6.6.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8c9854df0eaa610a23494c32a6f44a3a550fb398b6b51a56e8c6b9b3689578db", size = 253234 },
-    { url = "https://files.pythonhosted.org/packages/94/c1/5160c9813269e39ae14b73debb907bfaaa1beee1762da8c4fb95df4764ed/multidict-6.6.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4bb7627fd7a968f41905a4d6343b0d63244a0623f006e9ed989fa2b78f4438a0", size = 251613 },
-    { url = "https://files.pythonhosted.org/packages/05/a9/48d1bd111fc2f8fb98b2ed7f9a115c55a9355358432a19f53c0b74d8425d/multidict-6.6.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:caebafea30ed049c57c673d0b36238b1748683be2593965614d7b0e99125c877", size = 241649 },
-    { url = "https://files.pythonhosted.org/packages/85/2a/f7d743df0019408768af8a70d2037546a2be7b81fbb65f040d76caafd4c5/multidict-6.6.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ad887a8250eb47d3ab083d2f98db7f48098d13d42eb7a3b67d8a5c795f224ace", size = 239238 },
-    { url = "https://files.pythonhosted.org/packages/cb/b8/4f4bb13323c2d647323f7919201493cf48ebe7ded971717bfb0f1a79b6bf/multidict-6.6.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:ed8358ae7d94ffb7c397cecb62cbac9578a83ecefc1eba27b9090ee910e2efb6", size = 233517 },
-    { url = "https://files.pythonhosted.org/packages/33/29/4293c26029ebfbba4f574febd2ed01b6f619cfa0d2e344217d53eef34192/multidict-6.6.4-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ecab51ad2462197a4c000b6d5701fc8585b80eecb90583635d7e327b7b6923eb", size = 243122 },
-    { url = "https://files.pythonhosted.org/packages/20/60/a1c53628168aa22447bfde3a8730096ac28086704a0d8c590f3b63388d0c/multidict-6.6.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c5c97aa666cf70e667dfa5af945424ba1329af5dd988a437efeb3a09430389fb", size = 248992 },
-    { url = "https://files.pythonhosted.org/packages/a3/3b/55443a0c372f33cae5d9ec37a6a973802884fa0ab3586659b197cf8cc5e9/multidict-6.6.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:9a950b7cf54099c1209f455ac5970b1ea81410f2af60ed9eb3c3f14f0bfcf987", size = 243708 },
-    { url = "https://files.pythonhosted.org/packages/7c/60/a18c6900086769312560b2626b18e8cca22d9e85b1186ba77f4755b11266/multidict-6.6.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:163c7ea522ea9365a8a57832dea7618e6cbdc3cd75f8c627663587459a4e328f", size = 237498 },
-    { url = "https://files.pythonhosted.org/packages/11/3d/8bdd8bcaff2951ce2affccca107a404925a2beafedd5aef0b5e4a71120a6/multidict-6.6.4-cp310-cp310-win32.whl", hash = "sha256:17d2cbbfa6ff20821396b25890f155f40c986f9cfbce5667759696d83504954f", size = 41415 },
-    { url = "https://files.pythonhosted.org/packages/c0/53/cab1ad80356a4cd1b685a254b680167059b433b573e53872fab245e9fc95/multidict-6.6.4-cp310-cp310-win_amd64.whl", hash = "sha256:ce9a40fbe52e57e7edf20113a4eaddfacac0561a0879734e636aa6d4bb5e3fb0", size = 46046 },
-    { url = "https://files.pythonhosted.org/packages/cf/9a/874212b6f5c1c2d870d0a7adc5bb4cfe9b0624fa15cdf5cf757c0f5087ae/multidict-6.6.4-cp310-cp310-win_arm64.whl", hash = "sha256:01d0959807a451fe9fdd4da3e139cb5b77f7328baf2140feeaf233e1d777b729", size = 43147 },
-    { url = "https://files.pythonhosted.org/packages/6b/7f/90a7f01e2d005d6653c689039977f6856718c75c5579445effb7e60923d1/multidict-6.6.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c7a0e9b561e6460484318a7612e725df1145d46b0ef57c6b9866441bf6e27e0c", size = 76472 },
-    { url = "https://files.pythonhosted.org/packages/54/a3/bed07bc9e2bb302ce752f1dabc69e884cd6a676da44fb0e501b246031fdd/multidict-6.6.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6bf2f10f70acc7a2446965ffbc726e5fc0b272c97a90b485857e5c70022213eb", size = 44634 },
-    { url = "https://files.pythonhosted.org/packages/a7/4b/ceeb4f8f33cf81277da464307afeaf164fb0297947642585884f5cad4f28/multidict-6.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66247d72ed62d5dd29752ffc1d3b88f135c6a8de8b5f63b7c14e973ef5bda19e", size = 44282 },
-    { url = "https://files.pythonhosted.org/packages/03/35/436a5da8702b06866189b69f655ffdb8f70796252a8772a77815f1812679/multidict-6.6.4-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:105245cc6b76f51e408451a844a54e6823bbd5a490ebfe5bdfc79798511ceded", size = 229696 },
-    { url = "https://files.pythonhosted.org/packages/b6/0e/915160be8fecf1fca35f790c08fb74ca684d752fcba62c11daaf3d92c216/multidict-6.6.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbbc54e58b34c3bae389ef00046be0961f30fef7cb0dd9c7756aee376a4f7683", size = 246665 },
-    { url = "https://files.pythonhosted.org/packages/08/ee/2f464330acd83f77dcc346f0b1a0eaae10230291450887f96b204b8ac4d3/multidict-6.6.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:56c6b3652f945c9bc3ac6c8178cd93132b8d82dd581fcbc3a00676c51302bc1a", size = 225485 },
-    { url = "https://files.pythonhosted.org/packages/71/cc/9a117f828b4d7fbaec6adeed2204f211e9caf0a012692a1ee32169f846ae/multidict-6.6.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b95494daf857602eccf4c18ca33337dd2be705bccdb6dddbfc9d513e6addb9d9", size = 257318 },
-    { url = "https://files.pythonhosted.org/packages/25/77/62752d3dbd70e27fdd68e86626c1ae6bccfebe2bb1f84ae226363e112f5a/multidict-6.6.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e5b1413361cef15340ab9dc61523e653d25723e82d488ef7d60a12878227ed50", size = 254689 },
-    { url = "https://files.pythonhosted.org/packages/00/6e/fac58b1072a6fc59af5e7acb245e8754d3e1f97f4f808a6559951f72a0d4/multidict-6.6.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e167bf899c3d724f9662ef00b4f7fef87a19c22b2fead198a6f68b263618df52", size = 246709 },
-    { url = "https://files.pythonhosted.org/packages/01/ef/4698d6842ef5e797c6db7744b0081e36fb5de3d00002cc4c58071097fac3/multidict-6.6.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aaea28ba20a9026dfa77f4b80369e51cb767c61e33a2d4043399c67bd95fb7c6", size = 243185 },
-    { url = "https://files.pythonhosted.org/packages/aa/c9/d82e95ae1d6e4ef396934e9b0e942dfc428775f9554acf04393cce66b157/multidict-6.6.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:8c91cdb30809a96d9ecf442ec9bc45e8cfaa0f7f8bdf534e082c2443a196727e", size = 237838 },
-    { url = "https://files.pythonhosted.org/packages/57/cf/f94af5c36baaa75d44fab9f02e2a6bcfa0cd90acb44d4976a80960759dbc/multidict-6.6.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1a0ccbfe93ca114c5d65a2471d52d8829e56d467c97b0e341cf5ee45410033b3", size = 246368 },
-    { url = "https://files.pythonhosted.org/packages/4a/fe/29f23460c3d995f6a4b678cb2e9730e7277231b981f0b234702f0177818a/multidict-6.6.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:55624b3f321d84c403cb7d8e6e982f41ae233d85f85db54ba6286f7295dc8a9c", size = 253339 },
-    { url = "https://files.pythonhosted.org/packages/29/b6/fd59449204426187b82bf8a75f629310f68c6adc9559dc922d5abe34797b/multidict-6.6.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4a1fb393a2c9d202cb766c76208bd7945bc194eba8ac920ce98c6e458f0b524b", size = 246933 },
-    { url = "https://files.pythonhosted.org/packages/19/52/d5d6b344f176a5ac3606f7a61fb44dc746e04550e1a13834dff722b8d7d6/multidict-6.6.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:43868297a5759a845fa3a483fb4392973a95fb1de891605a3728130c52b8f40f", size = 242225 },
-    { url = "https://files.pythonhosted.org/packages/ec/d3/5b2281ed89ff4d5318d82478a2a2450fcdfc3300da48ff15c1778280ad26/multidict-6.6.4-cp311-cp311-win32.whl", hash = "sha256:ed3b94c5e362a8a84d69642dbeac615452e8af9b8eb825b7bc9f31a53a1051e2", size = 41306 },
-    { url = "https://files.pythonhosted.org/packages/74/7d/36b045c23a1ab98507aefd44fd8b264ee1dd5e5010543c6fccf82141ccef/multidict-6.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:d8c112f7a90d8ca5d20213aa41eac690bb50a76da153e3afb3886418e61cb22e", size = 46029 },
-    { url = "https://files.pythonhosted.org/packages/0f/5e/553d67d24432c5cd52b49047f2d248821843743ee6d29a704594f656d182/multidict-6.6.4-cp311-cp311-win_arm64.whl", hash = "sha256:3bb0eae408fa1996d87247ca0d6a57b7fc1dcf83e8a5c47ab82c558c250d4adf", size = 43017 },
-    { url = "https://files.pythonhosted.org/packages/05/f6/512ffd8fd8b37fb2680e5ac35d788f1d71bbaf37789d21a820bdc441e565/multidict-6.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0ffb87be160942d56d7b87b0fdf098e81ed565add09eaa1294268c7f3caac4c8", size = 76516 },
-    { url = "https://files.pythonhosted.org/packages/99/58/45c3e75deb8855c36bd66cc1658007589662ba584dbf423d01df478dd1c5/multidict-6.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d191de6cbab2aff5de6c5723101705fd044b3e4c7cfd587a1929b5028b9714b3", size = 45394 },
-    { url = "https://files.pythonhosted.org/packages/fd/ca/e8c4472a93a26e4507c0b8e1f0762c0d8a32de1328ef72fd704ef9cc5447/multidict-6.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38a0956dd92d918ad5feff3db8fcb4a5eb7dba114da917e1a88475619781b57b", size = 43591 },
-    { url = "https://files.pythonhosted.org/packages/05/51/edf414f4df058574a7265034d04c935aa84a89e79ce90fcf4df211f47b16/multidict-6.6.4-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6865f6d3b7900ae020b495d599fcf3765653bc927951c1abb959017f81ae8287", size = 237215 },
-    { url = "https://files.pythonhosted.org/packages/c8/45/8b3d6dbad8cf3252553cc41abea09ad527b33ce47a5e199072620b296902/multidict-6.6.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a2088c126b6f72db6c9212ad827d0ba088c01d951cee25e758c450da732c138", size = 258299 },
-    { url = "https://files.pythonhosted.org/packages/3c/e8/8ca2e9a9f5a435fc6db40438a55730a4bf4956b554e487fa1b9ae920f825/multidict-6.6.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0f37bed7319b848097085d7d48116f545985db988e2256b2e6f00563a3416ee6", size = 242357 },
-    { url = "https://files.pythonhosted.org/packages/0f/84/80c77c99df05a75c28490b2af8f7cba2a12621186e0a8b0865d8e745c104/multidict-6.6.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:01368e3c94032ba6ca0b78e7ccb099643466cf24f8dc8eefcfdc0571d56e58f9", size = 268369 },
-    { url = "https://files.pythonhosted.org/packages/0d/e9/920bfa46c27b05fb3e1ad85121fd49f441492dca2449c5bcfe42e4565d8a/multidict-6.6.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe323540c255db0bffee79ad7f048c909f2ab0edb87a597e1c17da6a54e493c", size = 269341 },
-    { url = "https://files.pythonhosted.org/packages/af/65/753a2d8b05daf496f4a9c367fe844e90a1b2cac78e2be2c844200d10cc4c/multidict-6.6.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8eb3025f17b0a4c3cd08cda49acf312a19ad6e8a4edd9dbd591e6506d999402", size = 256100 },
-    { url = "https://files.pythonhosted.org/packages/09/54/655be13ae324212bf0bc15d665a4e34844f34c206f78801be42f7a0a8aaa/multidict-6.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bbc14f0365534d35a06970d6a83478b249752e922d662dc24d489af1aa0d1be7", size = 253584 },
-    { url = "https://files.pythonhosted.org/packages/5c/74/ab2039ecc05264b5cec73eb018ce417af3ebb384ae9c0e9ed42cb33f8151/multidict-6.6.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:75aa52fba2d96bf972e85451b99d8e19cc37ce26fd016f6d4aa60da9ab2b005f", size = 251018 },
-    { url = "https://files.pythonhosted.org/packages/af/0a/ccbb244ac848e56c6427f2392741c06302bbfba49c0042f1eb3c5b606497/multidict-6.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fefd4a815e362d4f011919d97d7b4a1e566f1dde83dc4ad8cfb5b41de1df68d", size = 251477 },
-    { url = "https://files.pythonhosted.org/packages/0e/b0/0ed49bba775b135937f52fe13922bc64a7eaf0a3ead84a36e8e4e446e096/multidict-6.6.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:db9801fe021f59a5b375ab778973127ca0ac52429a26e2fd86aa9508f4d26eb7", size = 263575 },
-    { url = "https://files.pythonhosted.org/packages/3e/d9/7fb85a85e14de2e44dfb6a24f03c41e2af8697a6df83daddb0e9b7569f73/multidict-6.6.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a650629970fa21ac1fb06ba25dabfc5b8a2054fcbf6ae97c758aa956b8dba802", size = 259649 },
-    { url = "https://files.pythonhosted.org/packages/03/9e/b3a459bcf9b6e74fa461a5222a10ff9b544cb1cd52fd482fb1b75ecda2a2/multidict-6.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:452ff5da78d4720d7516a3a2abd804957532dd69296cb77319c193e3ffb87e24", size = 251505 },
-    { url = "https://files.pythonhosted.org/packages/86/a2/8022f78f041dfe6d71e364001a5cf987c30edfc83c8a5fb7a3f0974cff39/multidict-6.6.4-cp312-cp312-win32.whl", hash = "sha256:8c2fcb12136530ed19572bbba61b407f655e3953ba669b96a35036a11a485793", size = 41888 },
-    { url = "https://files.pythonhosted.org/packages/c7/eb/d88b1780d43a56db2cba24289fa744a9d216c1a8546a0dc3956563fd53ea/multidict-6.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:047d9425860a8c9544fed1b9584f0c8bcd31bcde9568b047c5e567a1025ecd6e", size = 46072 },
-    { url = "https://files.pythonhosted.org/packages/9f/16/b929320bf5750e2d9d4931835a4c638a19d2494a5b519caaaa7492ebe105/multidict-6.6.4-cp312-cp312-win_arm64.whl", hash = "sha256:14754eb72feaa1e8ae528468f24250dd997b8e2188c3d2f593f9eba259e4b364", size = 43222 },
-    { url = "https://files.pythonhosted.org/packages/3a/5d/e1db626f64f60008320aab00fbe4f23fc3300d75892a3381275b3d284580/multidict-6.6.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f46a6e8597f9bd71b31cc708195d42b634c8527fecbcf93febf1052cacc1f16e", size = 75848 },
-    { url = "https://files.pythonhosted.org/packages/4c/aa/8b6f548d839b6c13887253af4e29c939af22a18591bfb5d0ee6f1931dae8/multidict-6.6.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:22e38b2bc176c5eb9c0a0e379f9d188ae4cd8b28c0f53b52bce7ab0a9e534657", size = 45060 },
-    { url = "https://files.pythonhosted.org/packages/eb/c6/f5e97e5d99a729bc2aa58eb3ebfa9f1e56a9b517cc38c60537c81834a73f/multidict-6.6.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5df8afd26f162da59e218ac0eefaa01b01b2e6cd606cffa46608f699539246da", size = 43269 },
-    { url = "https://files.pythonhosted.org/packages/dc/31/d54eb0c62516776f36fe67f84a732f97e0b0e12f98d5685bebcc6d396910/multidict-6.6.4-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:49517449b58d043023720aa58e62b2f74ce9b28f740a0b5d33971149553d72aa", size = 237158 },
-    { url = "https://files.pythonhosted.org/packages/c4/1c/8a10c1c25b23156e63b12165a929d8eb49a6ed769fdbefb06e6f07c1e50d/multidict-6.6.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae9408439537c5afdca05edd128a63f56a62680f4b3c234301055d7a2000220f", size = 257076 },
-    { url = "https://files.pythonhosted.org/packages/ad/86/90e20b5771d6805a119e483fd3d1e8393e745a11511aebca41f0da38c3e2/multidict-6.6.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:87a32d20759dc52a9e850fe1061b6e41ab28e2998d44168a8a341b99ded1dba0", size = 240694 },
-    { url = "https://files.pythonhosted.org/packages/e7/49/484d3e6b535bc0555b52a0a26ba86e4d8d03fd5587d4936dc59ba7583221/multidict-6.6.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:52e3c8d43cdfff587ceedce9deb25e6ae77daba560b626e97a56ddcad3756879", size = 266350 },
-    { url = "https://files.pythonhosted.org/packages/bf/b4/aa4c5c379b11895083d50021e229e90c408d7d875471cb3abf721e4670d6/multidict-6.6.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ad8850921d3a8d8ff6fbef790e773cecfc260bbfa0566998980d3fa8f520bc4a", size = 267250 },
-    { url = "https://files.pythonhosted.org/packages/80/e5/5e22c5bf96a64bdd43518b1834c6d95a4922cc2066b7d8e467dae9b6cee6/multidict-6.6.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:497a2954adc25c08daff36f795077f63ad33e13f19bfff7736e72c785391534f", size = 254900 },
-    { url = "https://files.pythonhosted.org/packages/17/38/58b27fed927c07035abc02befacab42491e7388ca105e087e6e0215ead64/multidict-6.6.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:024ce601f92d780ca1617ad4be5ac15b501cc2414970ffa2bb2bbc2bd5a68fa5", size = 252355 },
-    { url = "https://files.pythonhosted.org/packages/d0/a1/dad75d23a90c29c02b5d6f3d7c10ab36c3197613be5d07ec49c7791e186c/multidict-6.6.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a693fc5ed9bdd1c9e898013e0da4dcc640de7963a371c0bd458e50e046bf6438", size = 250061 },
-    { url = "https://files.pythonhosted.org/packages/b8/1a/ac2216b61c7f116edab6dc3378cca6c70dc019c9a457ff0d754067c58b20/multidict-6.6.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:190766dac95aab54cae5b152a56520fd99298f32a1266d66d27fdd1b5ac00f4e", size = 249675 },
-    { url = "https://files.pythonhosted.org/packages/d4/79/1916af833b800d13883e452e8e0977c065c4ee3ab7a26941fbfdebc11895/multidict-6.6.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:34d8f2a5ffdceab9dcd97c7a016deb2308531d5f0fced2bb0c9e1df45b3363d7", size = 261247 },
-    { url = "https://files.pythonhosted.org/packages/c5/65/d1f84fe08ac44a5fc7391cbc20a7cedc433ea616b266284413fd86062f8c/multidict-6.6.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:59e8d40ab1f5a8597abcef00d04845155a5693b5da00d2c93dbe88f2050f2812", size = 257960 },
-    { url = "https://files.pythonhosted.org/packages/13/b5/29ec78057d377b195ac2c5248c773703a6b602e132a763e20ec0457e7440/multidict-6.6.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:467fe64138cfac771f0e949b938c2e1ada2b5af22f39692aa9258715e9ea613a", size = 250078 },
-    { url = "https://files.pythonhosted.org/packages/c4/0e/7e79d38f70a872cae32e29b0d77024bef7834b0afb406ddae6558d9e2414/multidict-6.6.4-cp313-cp313-win32.whl", hash = "sha256:14616a30fe6d0a48d0a48d1a633ab3b8bec4cf293aac65f32ed116f620adfd69", size = 41708 },
-    { url = "https://files.pythonhosted.org/packages/9d/34/746696dffff742e97cd6a23da953e55d0ea51fa601fa2ff387b3edcfaa2c/multidict-6.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:40cd05eaeb39e2bc8939451f033e57feaa2ac99e07dbca8afe2be450a4a3b6cf", size = 45912 },
-    { url = "https://files.pythonhosted.org/packages/c7/87/3bac136181e271e29170d8d71929cdeddeb77f3e8b6a0c08da3a8e9da114/multidict-6.6.4-cp313-cp313-win_arm64.whl", hash = "sha256:f6eb37d511bfae9e13e82cb4d1af36b91150466f24d9b2b8a9785816deb16605", size = 43076 },
-    { url = "https://files.pythonhosted.org/packages/64/94/0a8e63e36c049b571c9ae41ee301ada29c3fee9643d9c2548d7d558a1d99/multidict-6.6.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:6c84378acd4f37d1b507dfa0d459b449e2321b3ba5f2338f9b085cf7a7ba95eb", size = 82812 },
-    { url = "https://files.pythonhosted.org/packages/25/1a/be8e369dfcd260d2070a67e65dd3990dd635cbd735b98da31e00ea84cd4e/multidict-6.6.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0e0558693063c75f3d952abf645c78f3c5dfdd825a41d8c4d8156fc0b0da6e7e", size = 48313 },
-    { url = "https://files.pythonhosted.org/packages/26/5a/dd4ade298674b2f9a7b06a32c94ffbc0497354df8285f27317c66433ce3b/multidict-6.6.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3f8e2384cb83ebd23fd07e9eada8ba64afc4c759cd94817433ab8c81ee4b403f", size = 46777 },
-    { url = "https://files.pythonhosted.org/packages/89/db/98aa28bc7e071bfba611ac2ae803c24e96dd3a452b4118c587d3d872c64c/multidict-6.6.4-cp313-cp313t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f996b87b420995a9174b2a7c1a8daf7db4750be6848b03eb5e639674f7963773", size = 229321 },
-    { url = "https://files.pythonhosted.org/packages/c7/bc/01ddda2a73dd9d167bd85d0e8ef4293836a8f82b786c63fb1a429bc3e678/multidict-6.6.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc356250cffd6e78416cf5b40dc6a74f1edf3be8e834cf8862d9ed5265cf9b0e", size = 249954 },
-    { url = "https://files.pythonhosted.org/packages/06/78/6b7c0f020f9aa0acf66d0ab4eb9f08375bac9a50ff5e3edb1c4ccd59eafc/multidict-6.6.4-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:dadf95aa862714ea468a49ad1e09fe00fcc9ec67d122f6596a8d40caf6cec7d0", size = 228612 },
-    { url = "https://files.pythonhosted.org/packages/00/44/3faa416f89b2d5d76e9d447296a81521e1c832ad6e40b92f990697b43192/multidict-6.6.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dd57515bebffd8ebd714d101d4c434063322e4fe24042e90ced41f18b6d3395", size = 257528 },
-    { url = "https://files.pythonhosted.org/packages/05/5f/77c03b89af0fcb16f018f668207768191fb9dcfb5e3361a5e706a11db2c9/multidict-6.6.4-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:967af5f238ebc2eb1da4e77af5492219fbd9b4b812347da39a7b5f5c72c0fa45", size = 256329 },
-    { url = "https://files.pythonhosted.org/packages/cf/e9/ed750a2a9afb4f8dc6f13dc5b67b514832101b95714f1211cd42e0aafc26/multidict-6.6.4-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a4c6875c37aae9794308ec43e3530e4aa0d36579ce38d89979bbf89582002bb", size = 247928 },
-    { url = "https://files.pythonhosted.org/packages/1f/b5/e0571bc13cda277db7e6e8a532791d4403dacc9850006cb66d2556e649c0/multidict-6.6.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:7f683a551e92bdb7fac545b9c6f9fa2aebdeefa61d607510b3533286fcab67f5", size = 245228 },
-    { url = "https://files.pythonhosted.org/packages/f3/a3/69a84b0eccb9824491f06368f5b86e72e4af54c3067c37c39099b6687109/multidict-6.6.4-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:3ba5aaf600edaf2a868a391779f7a85d93bed147854925f34edd24cc70a3e141", size = 235869 },
-    { url = "https://files.pythonhosted.org/packages/a9/9d/28802e8f9121a6a0804fa009debf4e753d0a59969ea9f70be5f5fdfcb18f/multidict-6.6.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:580b643b7fd2c295d83cad90d78419081f53fd532d1f1eb67ceb7060f61cff0d", size = 243446 },
-    { url = "https://files.pythonhosted.org/packages/38/ea/6c98add069b4878c1d66428a5f5149ddb6d32b1f9836a826ac764b9940be/multidict-6.6.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:37b7187197da6af3ee0b044dbc9625afd0c885f2800815b228a0e70f9a7f473d", size = 252299 },
-    { url = "https://files.pythonhosted.org/packages/3a/09/8fe02d204473e14c0af3affd50af9078839dfca1742f025cca765435d6b4/multidict-6.6.4-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:e1b93790ed0bc26feb72e2f08299691ceb6da5e9e14a0d13cc74f1869af327a0", size = 246926 },
-    { url = "https://files.pythonhosted.org/packages/37/3d/7b1e10d774a6df5175ecd3c92bff069e77bed9ec2a927fdd4ff5fe182f67/multidict-6.6.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a506a77ddee1efcca81ecbeae27ade3e09cdf21a8ae854d766c2bb4f14053f92", size = 243383 },
-    { url = "https://files.pythonhosted.org/packages/50/b0/a6fae46071b645ae98786ab738447de1ef53742eaad949f27e960864bb49/multidict-6.6.4-cp313-cp313t-win32.whl", hash = "sha256:f93b2b2279883d1d0a9e1bd01f312d6fc315c5e4c1f09e112e4736e2f650bc4e", size = 47775 },
-    { url = "https://files.pythonhosted.org/packages/b2/0a/2436550b1520091af0600dff547913cb2d66fbac27a8c33bc1b1bccd8d98/multidict-6.6.4-cp313-cp313t-win_amd64.whl", hash = "sha256:6d46a180acdf6e87cc41dc15d8f5c2986e1e8739dc25dbb7dac826731ef381a4", size = 53100 },
-    { url = "https://files.pythonhosted.org/packages/97/ea/43ac51faff934086db9c072a94d327d71b7d8b40cd5dcb47311330929ef0/multidict-6.6.4-cp313-cp313t-win_arm64.whl", hash = "sha256:756989334015e3335d087a27331659820d53ba432befdef6a718398b0a8493ad", size = 45501 },
-    { url = "https://files.pythonhosted.org/packages/fd/69/b547032297c7e63ba2af494edba695d781af8a0c6e89e4d06cf848b21d80/multidict-6.6.4-py3-none-any.whl", hash = "sha256:27d8f8e125c07cb954e54d75d04905a9bba8a439c1d84aca94949d4d03d8601c", size = 12313 },
+    { name = "typing-extensions", marker = "python_full_version == '3.10.*'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/69/7f/0652e6ed47ab288e3756ea9c0df8b14950781184d4bd7883f4d87dd41245/multidict-6.6.4.tar.gz", hash = "sha256:d2d4e4787672911b48350df02ed3fa3fffdc2f2e8ca06dd6afdf34189b76a9dd", size = 101843, upload-time = "2025-08-11T12:08:48.217Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/eb/6b/86f353088c1358e76fd30b0146947fddecee812703b604ee901e85cd2a80/multidict-6.6.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b8aa6f0bd8125ddd04a6593437bad6a7e70f300ff4180a531654aa2ab3f6d58f", size = 77054, upload-time = "2025-08-11T12:06:02.99Z" },
+    { url = "https://files.pythonhosted.org/packages/19/5d/c01dc3d3788bb877bd7f5753ea6eb23c1beeca8044902a8f5bfb54430f63/multidict-6.6.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b9e5853bbd7264baca42ffc53391b490d65fe62849bf2c690fa3f6273dbcd0cb", size = 44914, upload-time = "2025-08-11T12:06:05.264Z" },
+    { url = "https://files.pythonhosted.org/packages/46/44/964dae19ea42f7d3e166474d8205f14bb811020e28bc423d46123ddda763/multidict-6.6.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0af5f9dee472371e36d6ae38bde009bd8ce65ac7335f55dcc240379d7bed1495", size = 44601, upload-time = "2025-08-11T12:06:06.627Z" },
+    { url = "https://files.pythonhosted.org/packages/31/20/0616348a1dfb36cb2ab33fc9521de1f27235a397bf3f59338e583afadd17/multidict-6.6.4-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:d24f351e4d759f5054b641c81e8291e5d122af0fca5c72454ff77f7cbe492de8", size = 224821, upload-time = "2025-08-11T12:06:08.06Z" },
+    { url = "https://files.pythonhosted.org/packages/14/26/5d8923c69c110ff51861af05bd27ca6783011b96725d59ccae6d9daeb627/multidict-6.6.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:db6a3810eec08280a172a6cd541ff4a5f6a97b161d93ec94e6c4018917deb6b7", size = 242608, upload-time = "2025-08-11T12:06:09.697Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/cc/e2ad3ba9459aa34fa65cf1f82a5c4a820a2ce615aacfb5143b8817f76504/multidict-6.6.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a1b20a9d56b2d81e2ff52ecc0670d583eaabaa55f402e8d16dd062373dbbe796", size = 222324, upload-time = "2025-08-11T12:06:10.905Z" },
+    { url = "https://files.pythonhosted.org/packages/19/db/4ed0f65701afbc2cb0c140d2d02928bb0fe38dd044af76e58ad7c54fd21f/multidict-6.6.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8c9854df0eaa610a23494c32a6f44a3a550fb398b6b51a56e8c6b9b3689578db", size = 253234, upload-time = "2025-08-11T12:06:12.658Z" },
+    { url = "https://files.pythonhosted.org/packages/94/c1/5160c9813269e39ae14b73debb907bfaaa1beee1762da8c4fb95df4764ed/multidict-6.6.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4bb7627fd7a968f41905a4d6343b0d63244a0623f006e9ed989fa2b78f4438a0", size = 251613, upload-time = "2025-08-11T12:06:13.97Z" },
+    { url = "https://files.pythonhosted.org/packages/05/a9/48d1bd111fc2f8fb98b2ed7f9a115c55a9355358432a19f53c0b74d8425d/multidict-6.6.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:caebafea30ed049c57c673d0b36238b1748683be2593965614d7b0e99125c877", size = 241649, upload-time = "2025-08-11T12:06:15.204Z" },
+    { url = "https://files.pythonhosted.org/packages/85/2a/f7d743df0019408768af8a70d2037546a2be7b81fbb65f040d76caafd4c5/multidict-6.6.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ad887a8250eb47d3ab083d2f98db7f48098d13d42eb7a3b67d8a5c795f224ace", size = 239238, upload-time = "2025-08-11T12:06:16.467Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/b8/4f4bb13323c2d647323f7919201493cf48ebe7ded971717bfb0f1a79b6bf/multidict-6.6.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:ed8358ae7d94ffb7c397cecb62cbac9578a83ecefc1eba27b9090ee910e2efb6", size = 233517, upload-time = "2025-08-11T12:06:18.107Z" },
+    { url = "https://files.pythonhosted.org/packages/33/29/4293c26029ebfbba4f574febd2ed01b6f619cfa0d2e344217d53eef34192/multidict-6.6.4-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ecab51ad2462197a4c000b6d5701fc8585b80eecb90583635d7e327b7b6923eb", size = 243122, upload-time = "2025-08-11T12:06:19.361Z" },
+    { url = "https://files.pythonhosted.org/packages/20/60/a1c53628168aa22447bfde3a8730096ac28086704a0d8c590f3b63388d0c/multidict-6.6.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c5c97aa666cf70e667dfa5af945424ba1329af5dd988a437efeb3a09430389fb", size = 248992, upload-time = "2025-08-11T12:06:20.661Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/3b/55443a0c372f33cae5d9ec37a6a973802884fa0ab3586659b197cf8cc5e9/multidict-6.6.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:9a950b7cf54099c1209f455ac5970b1ea81410f2af60ed9eb3c3f14f0bfcf987", size = 243708, upload-time = "2025-08-11T12:06:21.891Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/60/a18c6900086769312560b2626b18e8cca22d9e85b1186ba77f4755b11266/multidict-6.6.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:163c7ea522ea9365a8a57832dea7618e6cbdc3cd75f8c627663587459a4e328f", size = 237498, upload-time = "2025-08-11T12:06:23.206Z" },
+    { url = "https://files.pythonhosted.org/packages/11/3d/8bdd8bcaff2951ce2affccca107a404925a2beafedd5aef0b5e4a71120a6/multidict-6.6.4-cp310-cp310-win32.whl", hash = "sha256:17d2cbbfa6ff20821396b25890f155f40c986f9cfbce5667759696d83504954f", size = 41415, upload-time = "2025-08-11T12:06:24.77Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/53/cab1ad80356a4cd1b685a254b680167059b433b573e53872fab245e9fc95/multidict-6.6.4-cp310-cp310-win_amd64.whl", hash = "sha256:ce9a40fbe52e57e7edf20113a4eaddfacac0561a0879734e636aa6d4bb5e3fb0", size = 46046, upload-time = "2025-08-11T12:06:25.893Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/9a/874212b6f5c1c2d870d0a7adc5bb4cfe9b0624fa15cdf5cf757c0f5087ae/multidict-6.6.4-cp310-cp310-win_arm64.whl", hash = "sha256:01d0959807a451fe9fdd4da3e139cb5b77f7328baf2140feeaf233e1d777b729", size = 43147, upload-time = "2025-08-11T12:06:27.534Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/7f/90a7f01e2d005d6653c689039977f6856718c75c5579445effb7e60923d1/multidict-6.6.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c7a0e9b561e6460484318a7612e725df1145d46b0ef57c6b9866441bf6e27e0c", size = 76472, upload-time = "2025-08-11T12:06:29.006Z" },
+    { url = "https://files.pythonhosted.org/packages/54/a3/bed07bc9e2bb302ce752f1dabc69e884cd6a676da44fb0e501b246031fdd/multidict-6.6.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6bf2f10f70acc7a2446965ffbc726e5fc0b272c97a90b485857e5c70022213eb", size = 44634, upload-time = "2025-08-11T12:06:30.374Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/4b/ceeb4f8f33cf81277da464307afeaf164fb0297947642585884f5cad4f28/multidict-6.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66247d72ed62d5dd29752ffc1d3b88f135c6a8de8b5f63b7c14e973ef5bda19e", size = 44282, upload-time = "2025-08-11T12:06:31.958Z" },
+    { url = "https://files.pythonhosted.org/packages/03/35/436a5da8702b06866189b69f655ffdb8f70796252a8772a77815f1812679/multidict-6.6.4-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:105245cc6b76f51e408451a844a54e6823bbd5a490ebfe5bdfc79798511ceded", size = 229696, upload-time = "2025-08-11T12:06:33.087Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/0e/915160be8fecf1fca35f790c08fb74ca684d752fcba62c11daaf3d92c216/multidict-6.6.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbbc54e58b34c3bae389ef00046be0961f30fef7cb0dd9c7756aee376a4f7683", size = 246665, upload-time = "2025-08-11T12:06:34.448Z" },
+    { url = "https://files.pythonhosted.org/packages/08/ee/2f464330acd83f77dcc346f0b1a0eaae10230291450887f96b204b8ac4d3/multidict-6.6.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:56c6b3652f945c9bc3ac6c8178cd93132b8d82dd581fcbc3a00676c51302bc1a", size = 225485, upload-time = "2025-08-11T12:06:35.672Z" },
+    { url = "https://files.pythonhosted.org/packages/71/cc/9a117f828b4d7fbaec6adeed2204f211e9caf0a012692a1ee32169f846ae/multidict-6.6.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b95494daf857602eccf4c18ca33337dd2be705bccdb6dddbfc9d513e6addb9d9", size = 257318, upload-time = "2025-08-11T12:06:36.98Z" },
+    { url = "https://files.pythonhosted.org/packages/25/77/62752d3dbd70e27fdd68e86626c1ae6bccfebe2bb1f84ae226363e112f5a/multidict-6.6.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e5b1413361cef15340ab9dc61523e653d25723e82d488ef7d60a12878227ed50", size = 254689, upload-time = "2025-08-11T12:06:38.233Z" },
+    { url = "https://files.pythonhosted.org/packages/00/6e/fac58b1072a6fc59af5e7acb245e8754d3e1f97f4f808a6559951f72a0d4/multidict-6.6.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e167bf899c3d724f9662ef00b4f7fef87a19c22b2fead198a6f68b263618df52", size = 246709, upload-time = "2025-08-11T12:06:39.517Z" },
+    { url = "https://files.pythonhosted.org/packages/01/ef/4698d6842ef5e797c6db7744b0081e36fb5de3d00002cc4c58071097fac3/multidict-6.6.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aaea28ba20a9026dfa77f4b80369e51cb767c61e33a2d4043399c67bd95fb7c6", size = 243185, upload-time = "2025-08-11T12:06:40.796Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/c9/d82e95ae1d6e4ef396934e9b0e942dfc428775f9554acf04393cce66b157/multidict-6.6.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:8c91cdb30809a96d9ecf442ec9bc45e8cfaa0f7f8bdf534e082c2443a196727e", size = 237838, upload-time = "2025-08-11T12:06:42.595Z" },
+    { url = "https://files.pythonhosted.org/packages/57/cf/f94af5c36baaa75d44fab9f02e2a6bcfa0cd90acb44d4976a80960759dbc/multidict-6.6.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1a0ccbfe93ca114c5d65a2471d52d8829e56d467c97b0e341cf5ee45410033b3", size = 246368, upload-time = "2025-08-11T12:06:44.304Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/fe/29f23460c3d995f6a4b678cb2e9730e7277231b981f0b234702f0177818a/multidict-6.6.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:55624b3f321d84c403cb7d8e6e982f41ae233d85f85db54ba6286f7295dc8a9c", size = 253339, upload-time = "2025-08-11T12:06:45.597Z" },
+    { url = "https://files.pythonhosted.org/packages/29/b6/fd59449204426187b82bf8a75f629310f68c6adc9559dc922d5abe34797b/multidict-6.6.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4a1fb393a2c9d202cb766c76208bd7945bc194eba8ac920ce98c6e458f0b524b", size = 246933, upload-time = "2025-08-11T12:06:46.841Z" },
+    { url = "https://files.pythonhosted.org/packages/19/52/d5d6b344f176a5ac3606f7a61fb44dc746e04550e1a13834dff722b8d7d6/multidict-6.6.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:43868297a5759a845fa3a483fb4392973a95fb1de891605a3728130c52b8f40f", size = 242225, upload-time = "2025-08-11T12:06:48.588Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/d3/5b2281ed89ff4d5318d82478a2a2450fcdfc3300da48ff15c1778280ad26/multidict-6.6.4-cp311-cp311-win32.whl", hash = "sha256:ed3b94c5e362a8a84d69642dbeac615452e8af9b8eb825b7bc9f31a53a1051e2", size = 41306, upload-time = "2025-08-11T12:06:49.95Z" },
+    { url = "https://files.pythonhosted.org/packages/74/7d/36b045c23a1ab98507aefd44fd8b264ee1dd5e5010543c6fccf82141ccef/multidict-6.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:d8c112f7a90d8ca5d20213aa41eac690bb50a76da153e3afb3886418e61cb22e", size = 46029, upload-time = "2025-08-11T12:06:51.082Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/5e/553d67d24432c5cd52b49047f2d248821843743ee6d29a704594f656d182/multidict-6.6.4-cp311-cp311-win_arm64.whl", hash = "sha256:3bb0eae408fa1996d87247ca0d6a57b7fc1dcf83e8a5c47ab82c558c250d4adf", size = 43017, upload-time = "2025-08-11T12:06:52.243Z" },
+    { url = "https://files.pythonhosted.org/packages/05/f6/512ffd8fd8b37fb2680e5ac35d788f1d71bbaf37789d21a820bdc441e565/multidict-6.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0ffb87be160942d56d7b87b0fdf098e81ed565add09eaa1294268c7f3caac4c8", size = 76516, upload-time = "2025-08-11T12:06:53.393Z" },
+    { url = "https://files.pythonhosted.org/packages/99/58/45c3e75deb8855c36bd66cc1658007589662ba584dbf423d01df478dd1c5/multidict-6.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d191de6cbab2aff5de6c5723101705fd044b3e4c7cfd587a1929b5028b9714b3", size = 45394, upload-time = "2025-08-11T12:06:54.555Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/ca/e8c4472a93a26e4507c0b8e1f0762c0d8a32de1328ef72fd704ef9cc5447/multidict-6.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38a0956dd92d918ad5feff3db8fcb4a5eb7dba114da917e1a88475619781b57b", size = 43591, upload-time = "2025-08-11T12:06:55.672Z" },
+    { url = "https://files.pythonhosted.org/packages/05/51/edf414f4df058574a7265034d04c935aa84a89e79ce90fcf4df211f47b16/multidict-6.6.4-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6865f6d3b7900ae020b495d599fcf3765653bc927951c1abb959017f81ae8287", size = 237215, upload-time = "2025-08-11T12:06:57.213Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/45/8b3d6dbad8cf3252553cc41abea09ad527b33ce47a5e199072620b296902/multidict-6.6.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a2088c126b6f72db6c9212ad827d0ba088c01d951cee25e758c450da732c138", size = 258299, upload-time = "2025-08-11T12:06:58.946Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/e8/8ca2e9a9f5a435fc6db40438a55730a4bf4956b554e487fa1b9ae920f825/multidict-6.6.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0f37bed7319b848097085d7d48116f545985db988e2256b2e6f00563a3416ee6", size = 242357, upload-time = "2025-08-11T12:07:00.301Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/84/80c77c99df05a75c28490b2af8f7cba2a12621186e0a8b0865d8e745c104/multidict-6.6.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:01368e3c94032ba6ca0b78e7ccb099643466cf24f8dc8eefcfdc0571d56e58f9", size = 268369, upload-time = "2025-08-11T12:07:01.638Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/e9/920bfa46c27b05fb3e1ad85121fd49f441492dca2449c5bcfe42e4565d8a/multidict-6.6.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe323540c255db0bffee79ad7f048c909f2ab0edb87a597e1c17da6a54e493c", size = 269341, upload-time = "2025-08-11T12:07:02.943Z" },
+    { url = "https://files.pythonhosted.org/packages/af/65/753a2d8b05daf496f4a9c367fe844e90a1b2cac78e2be2c844200d10cc4c/multidict-6.6.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8eb3025f17b0a4c3cd08cda49acf312a19ad6e8a4edd9dbd591e6506d999402", size = 256100, upload-time = "2025-08-11T12:07:04.564Z" },
+    { url = "https://files.pythonhosted.org/packages/09/54/655be13ae324212bf0bc15d665a4e34844f34c206f78801be42f7a0a8aaa/multidict-6.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bbc14f0365534d35a06970d6a83478b249752e922d662dc24d489af1aa0d1be7", size = 253584, upload-time = "2025-08-11T12:07:05.914Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/74/ab2039ecc05264b5cec73eb018ce417af3ebb384ae9c0e9ed42cb33f8151/multidict-6.6.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:75aa52fba2d96bf972e85451b99d8e19cc37ce26fd016f6d4aa60da9ab2b005f", size = 251018, upload-time = "2025-08-11T12:07:08.301Z" },
+    { url = "https://files.pythonhosted.org/packages/af/0a/ccbb244ac848e56c6427f2392741c06302bbfba49c0042f1eb3c5b606497/multidict-6.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fefd4a815e362d4f011919d97d7b4a1e566f1dde83dc4ad8cfb5b41de1df68d", size = 251477, upload-time = "2025-08-11T12:07:10.248Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/b0/0ed49bba775b135937f52fe13922bc64a7eaf0a3ead84a36e8e4e446e096/multidict-6.6.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:db9801fe021f59a5b375ab778973127ca0ac52429a26e2fd86aa9508f4d26eb7", size = 263575, upload-time = "2025-08-11T12:07:11.928Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/d9/7fb85a85e14de2e44dfb6a24f03c41e2af8697a6df83daddb0e9b7569f73/multidict-6.6.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a650629970fa21ac1fb06ba25dabfc5b8a2054fcbf6ae97c758aa956b8dba802", size = 259649, upload-time = "2025-08-11T12:07:13.244Z" },
+    { url = "https://files.pythonhosted.org/packages/03/9e/b3a459bcf9b6e74fa461a5222a10ff9b544cb1cd52fd482fb1b75ecda2a2/multidict-6.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:452ff5da78d4720d7516a3a2abd804957532dd69296cb77319c193e3ffb87e24", size = 251505, upload-time = "2025-08-11T12:07:14.57Z" },
+    { url = "https://files.pythonhosted.org/packages/86/a2/8022f78f041dfe6d71e364001a5cf987c30edfc83c8a5fb7a3f0974cff39/multidict-6.6.4-cp312-cp312-win32.whl", hash = "sha256:8c2fcb12136530ed19572bbba61b407f655e3953ba669b96a35036a11a485793", size = 41888, upload-time = "2025-08-11T12:07:15.904Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/eb/d88b1780d43a56db2cba24289fa744a9d216c1a8546a0dc3956563fd53ea/multidict-6.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:047d9425860a8c9544fed1b9584f0c8bcd31bcde9568b047c5e567a1025ecd6e", size = 46072, upload-time = "2025-08-11T12:07:17.045Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/16/b929320bf5750e2d9d4931835a4c638a19d2494a5b519caaaa7492ebe105/multidict-6.6.4-cp312-cp312-win_arm64.whl", hash = "sha256:14754eb72feaa1e8ae528468f24250dd997b8e2188c3d2f593f9eba259e4b364", size = 43222, upload-time = "2025-08-11T12:07:18.328Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/5d/e1db626f64f60008320aab00fbe4f23fc3300d75892a3381275b3d284580/multidict-6.6.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f46a6e8597f9bd71b31cc708195d42b634c8527fecbcf93febf1052cacc1f16e", size = 75848, upload-time = "2025-08-11T12:07:19.912Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/aa/8b6f548d839b6c13887253af4e29c939af22a18591bfb5d0ee6f1931dae8/multidict-6.6.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:22e38b2bc176c5eb9c0a0e379f9d188ae4cd8b28c0f53b52bce7ab0a9e534657", size = 45060, upload-time = "2025-08-11T12:07:21.163Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/c6/f5e97e5d99a729bc2aa58eb3ebfa9f1e56a9b517cc38c60537c81834a73f/multidict-6.6.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5df8afd26f162da59e218ac0eefaa01b01b2e6cd606cffa46608f699539246da", size = 43269, upload-time = "2025-08-11T12:07:22.392Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/31/d54eb0c62516776f36fe67f84a732f97e0b0e12f98d5685bebcc6d396910/multidict-6.6.4-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:49517449b58d043023720aa58e62b2f74ce9b28f740a0b5d33971149553d72aa", size = 237158, upload-time = "2025-08-11T12:07:23.636Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/1c/8a10c1c25b23156e63b12165a929d8eb49a6ed769fdbefb06e6f07c1e50d/multidict-6.6.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae9408439537c5afdca05edd128a63f56a62680f4b3c234301055d7a2000220f", size = 257076, upload-time = "2025-08-11T12:07:25.049Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/86/90e20b5771d6805a119e483fd3d1e8393e745a11511aebca41f0da38c3e2/multidict-6.6.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:87a32d20759dc52a9e850fe1061b6e41ab28e2998d44168a8a341b99ded1dba0", size = 240694, upload-time = "2025-08-11T12:07:26.458Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/49/484d3e6b535bc0555b52a0a26ba86e4d8d03fd5587d4936dc59ba7583221/multidict-6.6.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:52e3c8d43cdfff587ceedce9deb25e6ae77daba560b626e97a56ddcad3756879", size = 266350, upload-time = "2025-08-11T12:07:27.94Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/b4/aa4c5c379b11895083d50021e229e90c408d7d875471cb3abf721e4670d6/multidict-6.6.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ad8850921d3a8d8ff6fbef790e773cecfc260bbfa0566998980d3fa8f520bc4a", size = 267250, upload-time = "2025-08-11T12:07:29.303Z" },
+    { url = "https://files.pythonhosted.org/packages/80/e5/5e22c5bf96a64bdd43518b1834c6d95a4922cc2066b7d8e467dae9b6cee6/multidict-6.6.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:497a2954adc25c08daff36f795077f63ad33e13f19bfff7736e72c785391534f", size = 254900, upload-time = "2025-08-11T12:07:30.764Z" },
+    { url = "https://files.pythonhosted.org/packages/17/38/58b27fed927c07035abc02befacab42491e7388ca105e087e6e0215ead64/multidict-6.6.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:024ce601f92d780ca1617ad4be5ac15b501cc2414970ffa2bb2bbc2bd5a68fa5", size = 252355, upload-time = "2025-08-11T12:07:32.205Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/a1/dad75d23a90c29c02b5d6f3d7c10ab36c3197613be5d07ec49c7791e186c/multidict-6.6.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a693fc5ed9bdd1c9e898013e0da4dcc640de7963a371c0bd458e50e046bf6438", size = 250061, upload-time = "2025-08-11T12:07:33.623Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/1a/ac2216b61c7f116edab6dc3378cca6c70dc019c9a457ff0d754067c58b20/multidict-6.6.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:190766dac95aab54cae5b152a56520fd99298f32a1266d66d27fdd1b5ac00f4e", size = 249675, upload-time = "2025-08-11T12:07:34.958Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/79/1916af833b800d13883e452e8e0977c065c4ee3ab7a26941fbfdebc11895/multidict-6.6.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:34d8f2a5ffdceab9dcd97c7a016deb2308531d5f0fced2bb0c9e1df45b3363d7", size = 261247, upload-time = "2025-08-11T12:07:36.588Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/65/d1f84fe08ac44a5fc7391cbc20a7cedc433ea616b266284413fd86062f8c/multidict-6.6.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:59e8d40ab1f5a8597abcef00d04845155a5693b5da00d2c93dbe88f2050f2812", size = 257960, upload-time = "2025-08-11T12:07:39.735Z" },
+    { url = "https://files.pythonhosted.org/packages/13/b5/29ec78057d377b195ac2c5248c773703a6b602e132a763e20ec0457e7440/multidict-6.6.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:467fe64138cfac771f0e949b938c2e1ada2b5af22f39692aa9258715e9ea613a", size = 250078, upload-time = "2025-08-11T12:07:41.525Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/0e/7e79d38f70a872cae32e29b0d77024bef7834b0afb406ddae6558d9e2414/multidict-6.6.4-cp313-cp313-win32.whl", hash = "sha256:14616a30fe6d0a48d0a48d1a633ab3b8bec4cf293aac65f32ed116f620adfd69", size = 41708, upload-time = "2025-08-11T12:07:43.405Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/34/746696dffff742e97cd6a23da953e55d0ea51fa601fa2ff387b3edcfaa2c/multidict-6.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:40cd05eaeb39e2bc8939451f033e57feaa2ac99e07dbca8afe2be450a4a3b6cf", size = 45912, upload-time = "2025-08-11T12:07:45.082Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/87/3bac136181e271e29170d8d71929cdeddeb77f3e8b6a0c08da3a8e9da114/multidict-6.6.4-cp313-cp313-win_arm64.whl", hash = "sha256:f6eb37d511bfae9e13e82cb4d1af36b91150466f24d9b2b8a9785816deb16605", size = 43076, upload-time = "2025-08-11T12:07:46.746Z" },
+    { url = "https://files.pythonhosted.org/packages/64/94/0a8e63e36c049b571c9ae41ee301ada29c3fee9643d9c2548d7d558a1d99/multidict-6.6.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:6c84378acd4f37d1b507dfa0d459b449e2321b3ba5f2338f9b085cf7a7ba95eb", size = 82812, upload-time = "2025-08-11T12:07:48.402Z" },
+    { url = "https://files.pythonhosted.org/packages/25/1a/be8e369dfcd260d2070a67e65dd3990dd635cbd735b98da31e00ea84cd4e/multidict-6.6.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0e0558693063c75f3d952abf645c78f3c5dfdd825a41d8c4d8156fc0b0da6e7e", size = 48313, upload-time = "2025-08-11T12:07:49.679Z" },
+    { url = "https://files.pythonhosted.org/packages/26/5a/dd4ade298674b2f9a7b06a32c94ffbc0497354df8285f27317c66433ce3b/multidict-6.6.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3f8e2384cb83ebd23fd07e9eada8ba64afc4c759cd94817433ab8c81ee4b403f", size = 46777, upload-time = "2025-08-11T12:07:51.318Z" },
+    { url = "https://files.pythonhosted.org/packages/89/db/98aa28bc7e071bfba611ac2ae803c24e96dd3a452b4118c587d3d872c64c/multidict-6.6.4-cp313-cp313t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f996b87b420995a9174b2a7c1a8daf7db4750be6848b03eb5e639674f7963773", size = 229321, upload-time = "2025-08-11T12:07:52.965Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/bc/01ddda2a73dd9d167bd85d0e8ef4293836a8f82b786c63fb1a429bc3e678/multidict-6.6.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc356250cffd6e78416cf5b40dc6a74f1edf3be8e834cf8862d9ed5265cf9b0e", size = 249954, upload-time = "2025-08-11T12:07:54.423Z" },
+    { url = "https://files.pythonhosted.org/packages/06/78/6b7c0f020f9aa0acf66d0ab4eb9f08375bac9a50ff5e3edb1c4ccd59eafc/multidict-6.6.4-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:dadf95aa862714ea468a49ad1e09fe00fcc9ec67d122f6596a8d40caf6cec7d0", size = 228612, upload-time = "2025-08-11T12:07:55.914Z" },
+    { url = "https://files.pythonhosted.org/packages/00/44/3faa416f89b2d5d76e9d447296a81521e1c832ad6e40b92f990697b43192/multidict-6.6.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dd57515bebffd8ebd714d101d4c434063322e4fe24042e90ced41f18b6d3395", size = 257528, upload-time = "2025-08-11T12:07:57.371Z" },
+    { url = "https://files.pythonhosted.org/packages/05/5f/77c03b89af0fcb16f018f668207768191fb9dcfb5e3361a5e706a11db2c9/multidict-6.6.4-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:967af5f238ebc2eb1da4e77af5492219fbd9b4b812347da39a7b5f5c72c0fa45", size = 256329, upload-time = "2025-08-11T12:07:58.844Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/e9/ed750a2a9afb4f8dc6f13dc5b67b514832101b95714f1211cd42e0aafc26/multidict-6.6.4-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a4c6875c37aae9794308ec43e3530e4aa0d36579ce38d89979bbf89582002bb", size = 247928, upload-time = "2025-08-11T12:08:01.037Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/b5/e0571bc13cda277db7e6e8a532791d4403dacc9850006cb66d2556e649c0/multidict-6.6.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:7f683a551e92bdb7fac545b9c6f9fa2aebdeefa61d607510b3533286fcab67f5", size = 245228, upload-time = "2025-08-11T12:08:02.96Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/a3/69a84b0eccb9824491f06368f5b86e72e4af54c3067c37c39099b6687109/multidict-6.6.4-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:3ba5aaf600edaf2a868a391779f7a85d93bed147854925f34edd24cc70a3e141", size = 235869, upload-time = "2025-08-11T12:08:04.746Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/9d/28802e8f9121a6a0804fa009debf4e753d0a59969ea9f70be5f5fdfcb18f/multidict-6.6.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:580b643b7fd2c295d83cad90d78419081f53fd532d1f1eb67ceb7060f61cff0d", size = 243446, upload-time = "2025-08-11T12:08:06.332Z" },
+    { url = "https://files.pythonhosted.org/packages/38/ea/6c98add069b4878c1d66428a5f5149ddb6d32b1f9836a826ac764b9940be/multidict-6.6.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:37b7187197da6af3ee0b044dbc9625afd0c885f2800815b228a0e70f9a7f473d", size = 252299, upload-time = "2025-08-11T12:08:07.931Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/09/8fe02d204473e14c0af3affd50af9078839dfca1742f025cca765435d6b4/multidict-6.6.4-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:e1b93790ed0bc26feb72e2f08299691ceb6da5e9e14a0d13cc74f1869af327a0", size = 246926, upload-time = "2025-08-11T12:08:09.467Z" },
+    { url = "https://files.pythonhosted.org/packages/37/3d/7b1e10d774a6df5175ecd3c92bff069e77bed9ec2a927fdd4ff5fe182f67/multidict-6.6.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a506a77ddee1efcca81ecbeae27ade3e09cdf21a8ae854d766c2bb4f14053f92", size = 243383, upload-time = "2025-08-11T12:08:10.981Z" },
+    { url = "https://files.pythonhosted.org/packages/50/b0/a6fae46071b645ae98786ab738447de1ef53742eaad949f27e960864bb49/multidict-6.6.4-cp313-cp313t-win32.whl", hash = "sha256:f93b2b2279883d1d0a9e1bd01f312d6fc315c5e4c1f09e112e4736e2f650bc4e", size = 47775, upload-time = "2025-08-11T12:08:12.439Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/0a/2436550b1520091af0600dff547913cb2d66fbac27a8c33bc1b1bccd8d98/multidict-6.6.4-cp313-cp313t-win_amd64.whl", hash = "sha256:6d46a180acdf6e87cc41dc15d8f5c2986e1e8739dc25dbb7dac826731ef381a4", size = 53100, upload-time = "2025-08-11T12:08:13.823Z" },
+    { url = "https://files.pythonhosted.org/packages/97/ea/43ac51faff934086db9c072a94d327d71b7d8b40cd5dcb47311330929ef0/multidict-6.6.4-cp313-cp313t-win_arm64.whl", hash = "sha256:756989334015e3335d087a27331659820d53ba432befdef6a718398b0a8493ad", size = 45501, upload-time = "2025-08-11T12:08:15.173Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/d3/f04c5db316caee9b5b2cbba66270b358c922a959855995bedde87134287c/multidict-6.6.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:af7618b591bae552b40dbb6f93f5518328a949dac626ee75927bba1ecdeea9f4", size = 76977, upload-time = "2025-08-11T12:08:16.667Z" },
+    { url = "https://files.pythonhosted.org/packages/70/39/a6200417d883e510728ab3caec02d3b66ff09e1c85e0aab2ba311abfdf06/multidict-6.6.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b6819f83aef06f560cb15482d619d0e623ce9bf155115150a85ab11b8342a665", size = 44878, upload-time = "2025-08-11T12:08:18.157Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/7e/815be31ed35571b137d65232816f61513fcd97b2717d6a9d7800b5a0c6e0/multidict-6.6.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4d09384e75788861e046330308e7af54dd306aaf20eb760eb1d0de26b2bea2cb", size = 44546, upload-time = "2025-08-11T12:08:19.694Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/f1/21b5bff6a8c3e2aff56956c241941ace6b8820e1abe6b12d3c52868a773d/multidict-6.6.4-cp39-cp39-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:a59c63061f1a07b861c004e53869eb1211ffd1a4acbca330e3322efa6dd02978", size = 223020, upload-time = "2025-08-11T12:08:21.554Z" },
+    { url = "https://files.pythonhosted.org/packages/15/59/37083f1dd3439979a0ffeb1906818d978d88b4cc7f4600a9f89b1cb6713c/multidict-6.6.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:350f6b0fe1ced61e778037fdc7613f4051c8baf64b1ee19371b42a3acdb016a0", size = 240528, upload-time = "2025-08-11T12:08:23.45Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/f0/f054d123c87784307a27324c829eb55bcfd2e261eb785fcabbd832c8dc4a/multidict-6.6.4-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0c5cbac6b55ad69cb6aa17ee9343dfbba903118fd530348c330211dc7aa756d1", size = 219540, upload-time = "2025-08-11T12:08:24.965Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/26/8f78ce17b7118149c17f238f28fba2a850b660b860f9b024a34d0191030f/multidict-6.6.4-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:630f70c32b8066ddfd920350bc236225814ad94dfa493fe1910ee17fe4365cbb", size = 251182, upload-time = "2025-08-11T12:08:26.511Z" },
+    { url = "https://files.pythonhosted.org/packages/00/c3/a21466322d69f6594fe22d9379200f99194d21c12a5bbf8c2a39a46b83b6/multidict-6.6.4-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f8d4916a81697faec6cb724a273bd5457e4c6c43d82b29f9dc02c5542fd21fc9", size = 249371, upload-time = "2025-08-11T12:08:28.075Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/8e/2e673124eb05cf8dc82e9265eccde01a36bcbd3193e27799b8377123c976/multidict-6.6.4-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e42332cf8276bb7645d310cdecca93a16920256a5b01bebf747365f86a1675b", size = 239235, upload-time = "2025-08-11T12:08:29.937Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/2d/bdd9f05e7c89e30a4b0e4faf0681a30748f8d1310f68cfdc0e3571e75bd5/multidict-6.6.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f3be27440f7644ab9a13a6fc86f09cdd90b347c3c5e30c6d6d860de822d7cb53", size = 237410, upload-time = "2025-08-11T12:08:31.872Z" },
+    { url = "https://files.pythonhosted.org/packages/46/4c/3237b83f8ca9a2673bb08fc340c15da005a80f5cc49748b587c8ae83823b/multidict-6.6.4-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:21f216669109e02ef3e2415ede07f4f8987f00de8cdfa0cc0b3440d42534f9f0", size = 232979, upload-time = "2025-08-11T12:08:33.399Z" },
+    { url = "https://files.pythonhosted.org/packages/55/a6/a765decff625ae9bc581aed303cd1837955177dafc558859a69f56f56ba8/multidict-6.6.4-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:d9890d68c45d1aeac5178ded1d1cccf3bc8d7accf1f976f79bf63099fb16e4bd", size = 240979, upload-time = "2025-08-11T12:08:35.02Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/2d/9c75975cb0c66ea33cae1443bb265b2b3cd689bffcbc68872565f401da23/multidict-6.6.4-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:edfdcae97cdc5d1a89477c436b61f472c4d40971774ac4729c613b4b133163cb", size = 246849, upload-time = "2025-08-11T12:08:37.038Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/71/d21ac0843c1d8751fb5dcf8a1f436625d39d4577bc27829799d09b419af7/multidict-6.6.4-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:0b2e886624be5773e69cf32bcb8534aecdeb38943520b240fed3d5596a430f2f", size = 241798, upload-time = "2025-08-11T12:08:38.669Z" },
+    { url = "https://files.pythonhosted.org/packages/94/3d/1d8911e53092837bd11b1c99d71de3e2a9a26f8911f864554677663242aa/multidict-6.6.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:be5bf4b3224948032a845d12ab0f69f208293742df96dc14c4ff9b09e508fc17", size = 235315, upload-time = "2025-08-11T12:08:40.266Z" },
+    { url = "https://files.pythonhosted.org/packages/86/c5/4b758df96376f73e936b1942c6c2dfc17e37ed9d5ff3b01a811496966ca0/multidict-6.6.4-cp39-cp39-win32.whl", hash = "sha256:10a68a9191f284fe9d501fef4efe93226e74df92ce7a24e301371293bd4918ae", size = 41434, upload-time = "2025-08-11T12:08:41.965Z" },
+    { url = "https://files.pythonhosted.org/packages/58/16/f1dfa2a0f25f2717a5e9e5fe8fd30613f7fe95e3530cec8d11f5de0b709c/multidict-6.6.4-cp39-cp39-win_amd64.whl", hash = "sha256:ee25f82f53262f9ac93bd7e58e47ea1bdcc3393cef815847e397cba17e284210", size = 46186, upload-time = "2025-08-11T12:08:43.367Z" },
+    { url = "https://files.pythonhosted.org/packages/88/7d/a0568bac65438c494cb6950b29f394d875a796a237536ac724879cf710c9/multidict-6.6.4-cp39-cp39-win_arm64.whl", hash = "sha256:f9867e55590e0855bcec60d4f9a092b69476db64573c9fe17e92b0c50614c16a", size = 43115, upload-time = "2025-08-11T12:08:45.126Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/69/b547032297c7e63ba2af494edba695d781af8a0c6e89e4d06cf848b21d80/multidict-6.6.4-py3-none-any.whl", hash = "sha256:27d8f8e125c07cb954e54d75d04905a9bba8a439c1d84aca94949d4d03d8601c", size = 12313, upload-time = "2025-08-11T12:08:46.891Z" },
 ]
 
 [[package]]
@@ -1166,26 +1431,40 @@ name = "multiprocess"
 version = "0.70.16"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "dill" },
+    { name = "dill", marker = "python_full_version >= '3.10'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/b5/ae/04f39c5d0d0def03247c2893d6f2b83c136bf3320a2154d7b8858f2ba72d/multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1", size = 1772603 }
+sdist = { url = "https://files.pythonhosted.org/packages/b5/ae/04f39c5d0d0def03247c2893d6f2b83c136bf3320a2154d7b8858f2ba72d/multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1", size = 1772603, upload-time = "2024-01-28T18:52:34.85Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ef/76/6e712a2623d146d314f17598df5de7224c85c0060ef63fd95cc15a25b3fa/multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee", size = 134980 },
-    { url = "https://files.pythonhosted.org/packages/0f/ab/1e6e8009e380e22254ff539ebe117861e5bdb3bff1fc977920972237c6c7/multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec", size = 134982 },
-    { url = "https://files.pythonhosted.org/packages/bc/f7/7ec7fddc92e50714ea3745631f79bd9c96424cb2702632521028e57d3a36/multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02", size = 134824 },
-    { url = "https://files.pythonhosted.org/packages/50/15/b56e50e8debaf439f44befec5b2af11db85f6e0f344c3113ae0be0593a91/multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a", size = 143519 },
-    { url = "https://files.pythonhosted.org/packages/0a/7d/a988f258104dcd2ccf1ed40fdc97e26c4ac351eeaf81d76e266c52d84e2f/multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e", size = 146741 },
-    { url = "https://files.pythonhosted.org/packages/ea/89/38df130f2c799090c978b366cfdf5b96d08de5b29a4a293df7f7429fa50b/multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435", size = 132628 },
-    { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351 },
+    { url = "https://files.pythonhosted.org/packages/ef/76/6e712a2623d146d314f17598df5de7224c85c0060ef63fd95cc15a25b3fa/multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee", size = 134980, upload-time = "2024-01-28T18:52:15.731Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/ab/1e6e8009e380e22254ff539ebe117861e5bdb3bff1fc977920972237c6c7/multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec", size = 134982, upload-time = "2024-01-28T18:52:17.783Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/94/8638a89f93c80df329116e6781a060506c7e91e1f4370dc831e9d17a041d/multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41", size = 133497, upload-time = "2024-01-28T18:52:22.644Z" },
+    { url = "https://files.pythonhosted.org/packages/89/21/222066f6bb8d8af287923ae3bd26cf4699a9ce020228ac273caca1de8250/multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a", size = 133498, upload-time = "2024-01-28T18:52:24.576Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/f7/7ec7fddc92e50714ea3745631f79bd9c96424cb2702632521028e57d3a36/multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02", size = 134824, upload-time = "2024-01-28T18:52:26.062Z" },
+    { url = "https://files.pythonhosted.org/packages/50/15/b56e50e8debaf439f44befec5b2af11db85f6e0f344c3113ae0be0593a91/multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a", size = 143519, upload-time = "2024-01-28T18:52:28.115Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/7d/a988f258104dcd2ccf1ed40fdc97e26c4ac351eeaf81d76e266c52d84e2f/multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e", size = 146741, upload-time = "2024-01-28T18:52:29.395Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/89/38df130f2c799090c978b366cfdf5b96d08de5b29a4a293df7f7429fa50b/multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435", size = 132628, upload-time = "2024-01-28T18:52:30.853Z" },
+    { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351, upload-time = "2024-01-28T18:52:31.981Z" },
 ]
 
 [[package]]
 name = "namex"
 version = "0.1.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/0c/c0/ee95b28f029c73f8d49d8f52edaed02a1d4a9acb8b69355737fdb1faa191/namex-0.1.0.tar.gz", hash = "sha256:117f03ccd302cc48e3f5c58a296838f6b89c83455ab8683a1e85f2a430aa4306", size = 6649 }
+sdist = { url = "https://files.pythonhosted.org/packages/0c/c0/ee95b28f029c73f8d49d8f52edaed02a1d4a9acb8b69355737fdb1faa191/namex-0.1.0.tar.gz", hash = "sha256:117f03ccd302cc48e3f5c58a296838f6b89c83455ab8683a1e85f2a430aa4306", size = 6649, upload-time = "2025-05-26T23:17:38.918Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/b2/bc/465daf1de06409cdd4532082806770ee0d8d7df434da79c76564d0f69741/namex-0.1.0-py3-none-any.whl", hash = "sha256:e2012a474502f1e2251267062aae3114611f07df4224b6e06334c57b0f2ce87c", size = 5905 },
+    { url = "https://files.pythonhosted.org/packages/b2/bc/465daf1de06409cdd4532082806770ee0d8d7df434da79c76564d0f69741/namex-0.1.0-py3-none-any.whl", hash = "sha256:e2012a474502f1e2251267062aae3114611f07df4224b6e06334c57b0f2ce87c", size = 5905, upload-time = "2025-05-26T23:17:37.695Z" },
+]
+
+[[package]]
+name = "networkx"
+version = "3.2.1"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c4/80/a84676339aaae2f1cfdf9f418701dd634aef9cc76f708ef55c36ff39c3ca/networkx-3.2.1.tar.gz", hash = "sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6", size = 2073928, upload-time = "2023-10-28T08:41:39.364Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d5/f0/8fbc882ca80cf077f1b246c0e3c3465f7f415439bdea6b899f6b19f61f70/networkx-3.2.1-py3-none-any.whl", hash = "sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2", size = 1647772, upload-time = "2023-10-28T08:41:36.945Z" },
 ]
 
 [[package]]
@@ -1193,11 +1472,11 @@ name = "networkx"
 version = "3.4.2"
 source = { registry = "https://pypi.org/simple" }
 resolution-markers = [
-    "python_full_version < '3.11'",
+    "python_full_version == '3.10.*'",
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368 }
+sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f", size = 1723263 },
+    { url = "https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f", size = 1723263, upload-time = "2024-10-21T12:39:36.247Z" },
 ]
 
 [[package]]
@@ -1210,18 +1489,73 @@ resolution-markers = [
     "python_full_version == '3.12.*'",
     "python_full_version == '3.11.*'",
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065 }
+sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406 },
+    { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" },
 ]
 
 [[package]]
 name = "nodeenv"
 version = "1.9.1"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437 }
+sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437, upload-time = "2024-06-04T18:44:11.171Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314 },
+    { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" },
+]
+
+[[package]]
+name = "numpy"
+version = "2.0.2"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a9/75/10dd1f8116a8b796cb2c737b674e02d02e80454bda953fa7e65d8c12b016/numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78", size = 18902015, upload-time = "2024-08-26T20:19:40.945Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/21/91/3495b3237510f79f5d81f2508f9f13fea78ebfdf07538fc7444badda173d/numpy-2.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece", size = 21165245, upload-time = "2024-08-26T20:04:14.625Z" },
+    { url = "https://files.pythonhosted.org/packages/05/33/26178c7d437a87082d11019292dce6d3fe6f0e9026b7b2309cbf3e489b1d/numpy-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04", size = 13738540, upload-time = "2024-08-26T20:04:36.784Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/31/cc46e13bf07644efc7a4bf68df2df5fb2a1a88d0cd0da9ddc84dc0033e51/numpy-2.0.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8c5713284ce4e282544c68d1c3b2c7161d38c256d2eefc93c1d683cf47683e66", size = 5300623, upload-time = "2024-08-26T20:04:46.491Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/16/7bfcebf27bb4f9d7ec67332ffebee4d1bf085c84246552d52dbb548600e7/numpy-2.0.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:becfae3ddd30736fe1889a37f1f580e245ba79a5855bff5f2a29cb3ccc22dd7b", size = 6901774, upload-time = "2024-08-26T20:04:58.173Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/a3/561c531c0e8bf082c5bef509d00d56f82e0ea7e1e3e3a7fc8fa78742a6e5/numpy-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da5960c3cf0df7eafefd806d4e612c5e19358de82cb3c343631188991566ccd", size = 13907081, upload-time = "2024-08-26T20:05:19.098Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/66/f7177ab331876200ac7563a580140643d1179c8b4b6a6b0fc9838de2a9b8/numpy-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:496f71341824ed9f3d2fd36cf3ac57ae2e0165c143b55c3a035ee219413f3318", size = 19523451, upload-time = "2024-08-26T20:05:47.479Z" },
+    { url = "https://files.pythonhosted.org/packages/25/7f/0b209498009ad6453e4efc2c65bcdf0ae08a182b2b7877d7ab38a92dc542/numpy-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a61ec659f68ae254e4d237816e33171497e978140353c0c2038d46e63282d0c8", size = 19927572, upload-time = "2024-08-26T20:06:17.137Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/df/2619393b1e1b565cd2d4c4403bdd979621e2c4dea1f8532754b2598ed63b/numpy-2.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d731a1c6116ba289c1e9ee714b08a8ff882944d4ad631fd411106a30f083c326", size = 14400722, upload-time = "2024-08-26T20:06:39.16Z" },
+    { url = "https://files.pythonhosted.org/packages/22/ad/77e921b9f256d5da36424ffb711ae79ca3f451ff8489eeca544d0701d74a/numpy-2.0.2-cp310-cp310-win32.whl", hash = "sha256:984d96121c9f9616cd33fbd0618b7f08e0cfc9600a7ee1d6fd9b239186d19d97", size = 6472170, upload-time = "2024-08-26T20:06:50.361Z" },
+    { url = "https://files.pythonhosted.org/packages/10/05/3442317535028bc29cf0c0dd4c191a4481e8376e9f0db6bcf29703cadae6/numpy-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7b0be4ef08607dd04da4092faee0b86607f111d5ae68036f16cc787e250a131", size = 15905558, upload-time = "2024-08-26T20:07:13.881Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/cf/034500fb83041aa0286e0fb16e7c76e5c8b67c0711bb6e9e9737a717d5fe/numpy-2.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448", size = 21169137, upload-time = "2024-08-26T20:07:45.345Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/d9/32de45561811a4b87fbdee23b5797394e3d1504b4a7cf40c10199848893e/numpy-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195", size = 13703552, upload-time = "2024-08-26T20:08:06.666Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/ca/2f384720020c7b244d22508cb7ab23d95f179fcfff33c31a6eeba8d6c512/numpy-2.0.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57", size = 5298957, upload-time = "2024-08-26T20:08:15.83Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/78/a3e4f9fb6aa4e6fdca0c5428e8ba039408514388cf62d89651aade838269/numpy-2.0.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a", size = 6905573, upload-time = "2024-08-26T20:08:27.185Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/72/cfc3a1beb2caf4efc9d0b38a15fe34025230da27e1c08cc2eb9bfb1c7231/numpy-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669", size = 13914330, upload-time = "2024-08-26T20:08:48.058Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/a8/c17acf65a931ce551fee11b72e8de63bf7e8a6f0e21add4c937c83563538/numpy-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951", size = 19534895, upload-time = "2024-08-26T20:09:16.536Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/86/8767f3d54f6ae0165749f84648da9dcc8cd78ab65d415494962c86fac80f/numpy-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9", size = 19937253, upload-time = "2024-08-26T20:09:46.263Z" },
+    { url = "https://files.pythonhosted.org/packages/df/87/f76450e6e1c14e5bb1eae6836478b1028e096fd02e85c1c37674606ab752/numpy-2.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15", size = 14414074, upload-time = "2024-08-26T20:10:08.483Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/ca/0f0f328e1e59f73754f06e1adfb909de43726d4f24c6a3f8805f34f2b0fa/numpy-2.0.2-cp311-cp311-win32.whl", hash = "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4", size = 6470640, upload-time = "2024-08-26T20:10:19.732Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/57/3a3f14d3a759dcf9bf6e9eda905794726b758819df4663f217d658a58695/numpy-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc", size = 15910230, upload-time = "2024-08-26T20:10:43.413Z" },
+    { url = "https://files.pythonhosted.org/packages/45/40/2e117be60ec50d98fa08c2f8c48e09b3edea93cfcabd5a9ff6925d54b1c2/numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b", size = 20895803, upload-time = "2024-08-26T20:11:13.916Z" },
+    { url = "https://files.pythonhosted.org/packages/46/92/1b8b8dee833f53cef3e0a3f69b2374467789e0bb7399689582314df02651/numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e", size = 13471835, upload-time = "2024-08-26T20:11:34.779Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/19/e2793bde475f1edaea6945be141aef6c8b4c669b90c90a300a8954d08f0a/numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c", size = 5038499, upload-time = "2024-08-26T20:11:43.902Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/ff/ddf6dac2ff0dd50a7327bcdba45cb0264d0e96bb44d33324853f781a8f3c/numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c", size = 6633497, upload-time = "2024-08-26T20:11:55.09Z" },
+    { url = "https://files.pythonhosted.org/packages/72/21/67f36eac8e2d2cd652a2e69595a54128297cdcb1ff3931cfc87838874bd4/numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692", size = 13621158, upload-time = "2024-08-26T20:12:14.95Z" },
+    { url = "https://files.pythonhosted.org/packages/39/68/e9f1126d757653496dbc096cb429014347a36b228f5a991dae2c6b6cfd40/numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a", size = 19236173, upload-time = "2024-08-26T20:12:44.049Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/e9/1f5333281e4ebf483ba1c888b1d61ba7e78d7e910fdd8e6499667041cc35/numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c", size = 19634174, upload-time = "2024-08-26T20:13:13.634Z" },
+    { url = "https://files.pythonhosted.org/packages/71/af/a469674070c8d8408384e3012e064299f7a2de540738a8e414dcfd639996/numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded", size = 14099701, upload-time = "2024-08-26T20:13:34.851Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/3d/08ea9f239d0e0e939b6ca52ad403c84a2bce1bde301a8eb4888c1c1543f1/numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5", size = 6174313, upload-time = "2024-08-26T20:13:45.653Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/b5/4ac39baebf1fdb2e72585c8352c56d063b6126be9fc95bd2bb5ef5770c20/numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a", size = 15606179, upload-time = "2024-08-26T20:14:08.786Z" },
+    { url = "https://files.pythonhosted.org/packages/43/c1/41c8f6df3162b0c6ffd4437d729115704bd43363de0090c7f913cfbc2d89/numpy-2.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9059e10581ce4093f735ed23f3b9d283b9d517ff46009ddd485f1747eb22653c", size = 21169942, upload-time = "2024-08-26T20:14:40.108Z" },
+    { url = "https://files.pythonhosted.org/packages/39/bc/fd298f308dcd232b56a4031fd6ddf11c43f9917fbc937e53762f7b5a3bb1/numpy-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:423e89b23490805d2a5a96fe40ec507407b8ee786d66f7328be214f9679df6dd", size = 13711512, upload-time = "2024-08-26T20:15:00.985Z" },
+    { url = "https://files.pythonhosted.org/packages/96/ff/06d1aa3eeb1c614eda245c1ba4fb88c483bee6520d361641331872ac4b82/numpy-2.0.2-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:2b2955fa6f11907cf7a70dab0d0755159bca87755e831e47932367fc8f2f2d0b", size = 5306976, upload-time = "2024-08-26T20:15:10.876Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/98/121996dcfb10a6087a05e54453e28e58694a7db62c5a5a29cee14c6e047b/numpy-2.0.2-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:97032a27bd9d8988b9a97a8c4d2c9f2c15a81f61e2f21404d7e8ef00cb5be729", size = 6906494, upload-time = "2024-08-26T20:15:22.055Z" },
+    { url = "https://files.pythonhosted.org/packages/15/31/9dffc70da6b9bbf7968f6551967fc21156207366272c2a40b4ed6008dc9b/numpy-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e795a8be3ddbac43274f18588329c72939870a16cae810c2b73461c40718ab1", size = 13912596, upload-time = "2024-08-26T20:15:42.452Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/14/78635daab4b07c0930c919d451b8bf8c164774e6a3413aed04a6d95758ce/numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b258c385842546006213344c50655ff1555a9338e2e5e02a0756dc3e803dd", size = 19526099, upload-time = "2024-08-26T20:16:11.048Z" },
+    { url = "https://files.pythonhosted.org/packages/26/4c/0eeca4614003077f68bfe7aac8b7496f04221865b3a5e7cb230c9d055afd/numpy-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fec9451a7789926bcf7c2b8d187292c9f93ea30284802a0ab3f5be8ab36865d", size = 19932823, upload-time = "2024-08-26T20:16:40.171Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/46/ea25b98b13dccaebddf1a803f8c748680d972e00507cd9bc6dcdb5aa2ac1/numpy-2.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9189427407d88ff25ecf8f12469d4d39d35bee1db5d39fc5c168c6f088a6956d", size = 14404424, upload-time = "2024-08-26T20:17:02.604Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/a6/177dd88d95ecf07e722d21008b1b40e681a929eb9e329684d449c36586b2/numpy-2.0.2-cp39-cp39-win32.whl", hash = "sha256:905d16e0c60200656500c95b6b8dca5d109e23cb24abc701d41c02d74c6b3afa", size = 6476809, upload-time = "2024-08-26T20:17:13.553Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/2b/7fc9f4e7ae5b507c1a3a21f0f15ed03e794c1242ea8a242ac158beb56034/numpy-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:a3f4ab0caa7f053f6797fcd4e1e25caee367db3112ef2b6ef82d749530768c73", size = 15911314, upload-time = "2024-08-26T20:17:36.72Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/3b/df5a870ac6a3be3a86856ce195ef42eec7ae50d2a202be1f5a4b3b340e14/numpy-2.0.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7f0a0c6f12e07fa94133c8a67404322845220c06a9e80e85999afe727f7438b8", size = 21025288, upload-time = "2024-08-26T20:18:07.732Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/97/51af92f18d6f6f2d9ad8b482a99fb74e142d71372da5d834b3a2747a446e/numpy-2.0.2-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:312950fdd060354350ed123c0e25a71327d3711584beaef30cdaa93320c392d4", size = 6762793, upload-time = "2024-08-26T20:18:19.125Z" },
+    { url = "https://files.pythonhosted.org/packages/12/46/de1fbd0c1b5ccaa7f9a005b66761533e2f6a3e560096682683a223631fe9/numpy-2.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26df23238872200f63518dd2aa984cfca675d82469535dc7162dc2ee52d9dd5c", size = 19334885, upload-time = "2024-08-26T20:18:47.237Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/dc/d330a6faefd92b446ec0f0dfea4c3207bb1fef3c4771d19cf4543efd2c78/numpy-2.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a46288ec55ebbd58947d31d72be2c63cbf839f0a63b49cb755022310792a3385", size = 15828784, upload-time = "2024-08-26T20:19:11.19Z" },
 ]
 
 [[package]]
@@ -1229,64 +1563,64 @@ name = "numpy"
 version = "2.2.6"
 source = { registry = "https://pypi.org/simple" }
 resolution-markers = [
-    "python_full_version < '3.11'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/9a/3e/ed6db5be21ce87955c0cbd3009f2803f59fa08df21b5df06862e2d8e2bdd/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb", size = 21165245 },
-    { url = "https://files.pythonhosted.org/packages/22/c2/4b9221495b2a132cc9d2eb862e21d42a009f5a60e45fc44b00118c174bff/numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90", size = 14360048 },
-    { url = "https://files.pythonhosted.org/packages/fd/77/dc2fcfc66943c6410e2bf598062f5959372735ffda175b39906d54f02349/numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163", size = 5340542 },
-    { url = "https://files.pythonhosted.org/packages/7a/4f/1cb5fdc353a5f5cc7feb692db9b8ec2c3d6405453f982435efc52561df58/numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf", size = 6878301 },
-    { url = "https://files.pythonhosted.org/packages/eb/17/96a3acd228cec142fcb8723bd3cc39c2a474f7dcf0a5d16731980bcafa95/numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83", size = 14297320 },
-    { url = "https://files.pythonhosted.org/packages/b4/63/3de6a34ad7ad6646ac7d2f55ebc6ad439dbbf9c4370017c50cf403fb19b5/numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915", size = 16801050 },
-    { url = "https://files.pythonhosted.org/packages/07/b6/89d837eddef52b3d0cec5c6ba0456c1bf1b9ef6a6672fc2b7873c3ec4e2e/numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680", size = 15807034 },
-    { url = "https://files.pythonhosted.org/packages/01/c8/dc6ae86e3c61cfec1f178e5c9f7858584049b6093f843bca541f94120920/numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289", size = 18614185 },
-    { url = "https://files.pythonhosted.org/packages/5b/c5/0064b1b7e7c89137b471ccec1fd2282fceaae0ab3a9550f2568782d80357/numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d", size = 6527149 },
-    { url = "https://files.pythonhosted.org/packages/a3/dd/4b822569d6b96c39d1215dbae0582fd99954dcbcf0c1a13c61783feaca3f/numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3", size = 12904620 },
-    { url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963 },
-    { url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743 },
-    { url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616 },
-    { url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579 },
-    { url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005 },
-    { url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570 },
-    { url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548 },
-    { url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521 },
-    { url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866 },
-    { url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455 },
-    { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348 },
-    { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362 },
-    { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103 },
-    { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382 },
-    { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462 },
-    { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618 },
-    { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511 },
-    { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783 },
-    { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506 },
-    { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190 },
-    { url = "https://files.pythonhosted.org/packages/f9/5c/6657823f4f594f72b5471f1db1ab12e26e890bb2e41897522d134d2a3e81/numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84", size = 20867828 },
-    { url = "https://files.pythonhosted.org/packages/dc/9e/14520dc3dadf3c803473bd07e9b2bd1b69bc583cb2497b47000fed2fa92f/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b", size = 14143006 },
-    { url = "https://files.pythonhosted.org/packages/4f/06/7e96c57d90bebdce9918412087fc22ca9851cceaf5567a45c1f404480e9e/numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d", size = 5076765 },
-    { url = "https://files.pythonhosted.org/packages/73/ed/63d920c23b4289fdac96ddbdd6132e9427790977d5457cd132f18e76eae0/numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566", size = 6617736 },
-    { url = "https://files.pythonhosted.org/packages/85/c5/e19c8f99d83fd377ec8c7e0cf627a8049746da54afc24ef0a0cb73d5dfb5/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f", size = 14010719 },
-    { url = "https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f", size = 16526072 },
-    { url = "https://files.pythonhosted.org/packages/b2/6c/04b5f47f4f32f7c2b0e7260442a8cbcf8168b0e1a41ff1495da42f42a14f/numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868", size = 15503213 },
-    { url = "https://files.pythonhosted.org/packages/17/0a/5cd92e352c1307640d5b6fec1b2ffb06cd0dabe7d7b8227f97933d378422/numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d", size = 18316632 },
-    { url = "https://files.pythonhosted.org/packages/f0/3b/5cba2b1d88760ef86596ad0f3d484b1cbff7c115ae2429678465057c5155/numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd", size = 6244532 },
-    { url = "https://files.pythonhosted.org/packages/cb/3b/d58c12eafcb298d4e6d0d40216866ab15f59e55d148a5658bb3132311fcf/numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c", size = 12610885 },
-    { url = "https://files.pythonhosted.org/packages/6b/9e/4bf918b818e516322db999ac25d00c75788ddfd2d2ade4fa66f1f38097e1/numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6", size = 20963467 },
-    { url = "https://files.pythonhosted.org/packages/61/66/d2de6b291507517ff2e438e13ff7b1e2cdbdb7cb40b3ed475377aece69f9/numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda", size = 14225144 },
-    { url = "https://files.pythonhosted.org/packages/e4/25/480387655407ead912e28ba3a820bc69af9adf13bcbe40b299d454ec011f/numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40", size = 5200217 },
-    { url = "https://files.pythonhosted.org/packages/aa/4a/6e313b5108f53dcbf3aca0c0f3e9c92f4c10ce57a0a721851f9785872895/numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8", size = 6712014 },
-    { url = "https://files.pythonhosted.org/packages/b7/30/172c2d5c4be71fdf476e9de553443cf8e25feddbe185e0bd88b096915bcc/numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f", size = 14077935 },
-    { url = "https://files.pythonhosted.org/packages/12/fb/9e743f8d4e4d3c710902cf87af3512082ae3d43b945d5d16563f26ec251d/numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa", size = 16600122 },
-    { url = "https://files.pythonhosted.org/packages/12/75/ee20da0e58d3a66f204f38916757e01e33a9737d0b22373b3eb5a27358f9/numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571", size = 15586143 },
-    { url = "https://files.pythonhosted.org/packages/76/95/bef5b37f29fc5e739947e9ce5179ad402875633308504a52d188302319c8/numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1", size = 18385260 },
-    { url = "https://files.pythonhosted.org/packages/09/04/f2f83279d287407cf36a7a8053a5abe7be3622a4363337338f2585e4afda/numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff", size = 6377225 },
-    { url = "https://files.pythonhosted.org/packages/67/0e/35082d13c09c02c011cf21570543d202ad929d961c02a147493cb0c2bdf5/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06", size = 12771374 },
-    { url = "https://files.pythonhosted.org/packages/9e/3b/d94a75f4dbf1ef5d321523ecac21ef23a3cd2ac8b78ae2aac40873590229/numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d", size = 21040391 },
-    { url = "https://files.pythonhosted.org/packages/17/f4/09b2fa1b58f0fb4f7c7963a1649c64c4d315752240377ed74d9cd878f7b5/numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db", size = 6786754 },
-    { url = "https://files.pythonhosted.org/packages/af/30/feba75f143bdc868a1cc3f44ccfa6c4b9ec522b36458e738cd00f67b573f/numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543", size = 16643476 },
-    { url = "https://files.pythonhosted.org/packages/37/48/ac2a9584402fb6c0cd5b5d1a91dcf176b15760130dd386bbafdbfe3640bf/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00", size = 12812666 },
+    "python_full_version == '3.10.*'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9a/3e/ed6db5be21ce87955c0cbd3009f2803f59fa08df21b5df06862e2d8e2bdd/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb", size = 21165245, upload-time = "2025-05-17T21:27:58.555Z" },
+    { url = "https://files.pythonhosted.org/packages/22/c2/4b9221495b2a132cc9d2eb862e21d42a009f5a60e45fc44b00118c174bff/numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90", size = 14360048, upload-time = "2025-05-17T21:28:21.406Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/77/dc2fcfc66943c6410e2bf598062f5959372735ffda175b39906d54f02349/numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163", size = 5340542, upload-time = "2025-05-17T21:28:30.931Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/4f/1cb5fdc353a5f5cc7feb692db9b8ec2c3d6405453f982435efc52561df58/numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf", size = 6878301, upload-time = "2025-05-17T21:28:41.613Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/17/96a3acd228cec142fcb8723bd3cc39c2a474f7dcf0a5d16731980bcafa95/numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83", size = 14297320, upload-time = "2025-05-17T21:29:02.78Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/63/3de6a34ad7ad6646ac7d2f55ebc6ad439dbbf9c4370017c50cf403fb19b5/numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915", size = 16801050, upload-time = "2025-05-17T21:29:27.675Z" },
+    { url = "https://files.pythonhosted.org/packages/07/b6/89d837eddef52b3d0cec5c6ba0456c1bf1b9ef6a6672fc2b7873c3ec4e2e/numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680", size = 15807034, upload-time = "2025-05-17T21:29:51.102Z" },
+    { url = "https://files.pythonhosted.org/packages/01/c8/dc6ae86e3c61cfec1f178e5c9f7858584049b6093f843bca541f94120920/numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289", size = 18614185, upload-time = "2025-05-17T21:30:18.703Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/c5/0064b1b7e7c89137b471ccec1fd2282fceaae0ab3a9550f2568782d80357/numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d", size = 6527149, upload-time = "2025-05-17T21:30:29.788Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/dd/4b822569d6b96c39d1215dbae0582fd99954dcbcf0c1a13c61783feaca3f/numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3", size = 12904620, upload-time = "2025-05-17T21:30:48.994Z" },
+    { url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963, upload-time = "2025-05-17T21:31:19.36Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743, upload-time = "2025-05-17T21:31:41.087Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616, upload-time = "2025-05-17T21:31:50.072Z" },
+    { url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579, upload-time = "2025-05-17T21:32:01.712Z" },
+    { url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005, upload-time = "2025-05-17T21:32:23.332Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570, upload-time = "2025-05-17T21:32:47.991Z" },
+    { url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548, upload-time = "2025-05-17T21:33:11.728Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521, upload-time = "2025-05-17T21:33:39.139Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866, upload-time = "2025-05-17T21:33:50.273Z" },
+    { url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455, upload-time = "2025-05-17T21:34:09.135Z" },
+    { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348, upload-time = "2025-05-17T21:34:39.648Z" },
+    { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362, upload-time = "2025-05-17T21:35:01.241Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103, upload-time = "2025-05-17T21:35:10.622Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382, upload-time = "2025-05-17T21:35:21.414Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462, upload-time = "2025-05-17T21:35:42.174Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618, upload-time = "2025-05-17T21:36:06.711Z" },
+    { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511, upload-time = "2025-05-17T21:36:29.965Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783, upload-time = "2025-05-17T21:36:56.883Z" },
+    { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506, upload-time = "2025-05-17T21:37:07.368Z" },
+    { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190, upload-time = "2025-05-17T21:37:26.213Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/5c/6657823f4f594f72b5471f1db1ab12e26e890bb2e41897522d134d2a3e81/numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84", size = 20867828, upload-time = "2025-05-17T21:37:56.699Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/9e/14520dc3dadf3c803473bd07e9b2bd1b69bc583cb2497b47000fed2fa92f/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b", size = 14143006, upload-time = "2025-05-17T21:38:18.291Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/06/7e96c57d90bebdce9918412087fc22ca9851cceaf5567a45c1f404480e9e/numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d", size = 5076765, upload-time = "2025-05-17T21:38:27.319Z" },
+    { url = "https://files.pythonhosted.org/packages/73/ed/63d920c23b4289fdac96ddbdd6132e9427790977d5457cd132f18e76eae0/numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566", size = 6617736, upload-time = "2025-05-17T21:38:38.141Z" },
+    { url = "https://files.pythonhosted.org/packages/85/c5/e19c8f99d83fd377ec8c7e0cf627a8049746da54afc24ef0a0cb73d5dfb5/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f", size = 14010719, upload-time = "2025-05-17T21:38:58.433Z" },
+    { url = "https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f", size = 16526072, upload-time = "2025-05-17T21:39:22.638Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/6c/04b5f47f4f32f7c2b0e7260442a8cbcf8168b0e1a41ff1495da42f42a14f/numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868", size = 15503213, upload-time = "2025-05-17T21:39:45.865Z" },
+    { url = "https://files.pythonhosted.org/packages/17/0a/5cd92e352c1307640d5b6fec1b2ffb06cd0dabe7d7b8227f97933d378422/numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d", size = 18316632, upload-time = "2025-05-17T21:40:13.331Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/3b/5cba2b1d88760ef86596ad0f3d484b1cbff7c115ae2429678465057c5155/numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd", size = 6244532, upload-time = "2025-05-17T21:43:46.099Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/3b/d58c12eafcb298d4e6d0d40216866ab15f59e55d148a5658bb3132311fcf/numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c", size = 12610885, upload-time = "2025-05-17T21:44:05.145Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/9e/4bf918b818e516322db999ac25d00c75788ddfd2d2ade4fa66f1f38097e1/numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6", size = 20963467, upload-time = "2025-05-17T21:40:44Z" },
+    { url = "https://files.pythonhosted.org/packages/61/66/d2de6b291507517ff2e438e13ff7b1e2cdbdb7cb40b3ed475377aece69f9/numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda", size = 14225144, upload-time = "2025-05-17T21:41:05.695Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/25/480387655407ead912e28ba3a820bc69af9adf13bcbe40b299d454ec011f/numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40", size = 5200217, upload-time = "2025-05-17T21:41:15.903Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/4a/6e313b5108f53dcbf3aca0c0f3e9c92f4c10ce57a0a721851f9785872895/numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8", size = 6712014, upload-time = "2025-05-17T21:41:27.321Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/30/172c2d5c4be71fdf476e9de553443cf8e25feddbe185e0bd88b096915bcc/numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f", size = 14077935, upload-time = "2025-05-17T21:41:49.738Z" },
+    { url = "https://files.pythonhosted.org/packages/12/fb/9e743f8d4e4d3c710902cf87af3512082ae3d43b945d5d16563f26ec251d/numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa", size = 16600122, upload-time = "2025-05-17T21:42:14.046Z" },
+    { url = "https://files.pythonhosted.org/packages/12/75/ee20da0e58d3a66f204f38916757e01e33a9737d0b22373b3eb5a27358f9/numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571", size = 15586143, upload-time = "2025-05-17T21:42:37.464Z" },
+    { url = "https://files.pythonhosted.org/packages/76/95/bef5b37f29fc5e739947e9ce5179ad402875633308504a52d188302319c8/numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1", size = 18385260, upload-time = "2025-05-17T21:43:05.189Z" },
+    { url = "https://files.pythonhosted.org/packages/09/04/f2f83279d287407cf36a7a8053a5abe7be3622a4363337338f2585e4afda/numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff", size = 6377225, upload-time = "2025-05-17T21:43:16.254Z" },
+    { url = "https://files.pythonhosted.org/packages/67/0e/35082d13c09c02c011cf21570543d202ad929d961c02a147493cb0c2bdf5/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06", size = 12771374, upload-time = "2025-05-17T21:43:35.479Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/3b/d94a75f4dbf1ef5d321523ecac21ef23a3cd2ac8b78ae2aac40873590229/numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d", size = 21040391, upload-time = "2025-05-17T21:44:35.948Z" },
+    { url = "https://files.pythonhosted.org/packages/17/f4/09b2fa1b58f0fb4f7c7963a1649c64c4d315752240377ed74d9cd878f7b5/numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db", size = 6786754, upload-time = "2025-05-17T21:44:47.446Z" },
+    { url = "https://files.pythonhosted.org/packages/af/30/feba75f143bdc868a1cc3f44ccfa6c4b9ec522b36458e738cd00f67b573f/numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543", size = 16643476, upload-time = "2025-05-17T21:45:11.871Z" },
+    { url = "https://files.pythonhosted.org/packages/37/48/ac2a9584402fb6c0cd5b5d1a91dcf176b15760130dd386bbafdbfe3640bf/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00", size = 12812666, upload-time = "2025-05-17T21:45:31.426Z" },
 ]
 
 [[package]]
@@ -1299,81 +1633,81 @@ resolution-markers = [
     "python_full_version == '3.12.*'",
     "python_full_version == '3.11.*'",
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/d0/19/95b3d357407220ed24c139018d2518fab0a61a948e68286a25f1a4d049ff/numpy-2.3.3.tar.gz", hash = "sha256:ddc7c39727ba62b80dfdbedf400d1c10ddfa8eefbd7ec8dcb118be8b56d31029", size = 20576648 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/7a/45/e80d203ef6b267aa29b22714fb558930b27960a0c5ce3c19c999232bb3eb/numpy-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ffc4f5caba7dfcbe944ed674b7eef683c7e94874046454bb79ed7ee0236f59d", size = 21259253 },
-    { url = "https://files.pythonhosted.org/packages/52/18/cf2c648fccf339e59302e00e5f2bc87725a3ce1992f30f3f78c9044d7c43/numpy-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7e946c7170858a0295f79a60214424caac2ffdb0063d4d79cb681f9aa0aa569", size = 14450980 },
-    { url = "https://files.pythonhosted.org/packages/93/fb/9af1082bec870188c42a1c239839915b74a5099c392389ff04215dcee812/numpy-2.3.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:cd4260f64bc794c3390a63bf0728220dd1a68170c169088a1e0dfa2fde1be12f", size = 5379709 },
-    { url = "https://files.pythonhosted.org/packages/75/0f/bfd7abca52bcbf9a4a65abc83fe18ef01ccdeb37bfb28bbd6ad613447c79/numpy-2.3.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:f0ddb4b96a87b6728df9362135e764eac3cfa674499943ebc44ce96c478ab125", size = 6913923 },
-    { url = "https://files.pythonhosted.org/packages/79/55/d69adad255e87ab7afda1caf93ca997859092afeb697703e2f010f7c2e55/numpy-2.3.3-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:afd07d377f478344ec6ca2b8d4ca08ae8bd44706763d1efb56397de606393f48", size = 14589591 },
-    { url = "https://files.pythonhosted.org/packages/10/a2/010b0e27ddeacab7839957d7a8f00e91206e0c2c47abbb5f35a2630e5387/numpy-2.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc92a5dedcc53857249ca51ef29f5e5f2f8c513e22cfb90faeb20343b8c6f7a6", size = 16938714 },
-    { url = "https://files.pythonhosted.org/packages/1c/6b/12ce8ede632c7126eb2762b9e15e18e204b81725b81f35176eac14dc5b82/numpy-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7af05ed4dc19f308e1d9fc759f36f21921eb7bbfc82843eeec6b2a2863a0aefa", size = 16370592 },
-    { url = "https://files.pythonhosted.org/packages/b4/35/aba8568b2593067bb6a8fe4c52babb23b4c3b9c80e1b49dff03a09925e4a/numpy-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:433bf137e338677cebdd5beac0199ac84712ad9d630b74eceeb759eaa45ddf30", size = 18884474 },
-    { url = "https://files.pythonhosted.org/packages/45/fa/7f43ba10c77575e8be7b0138d107e4f44ca4a1ef322cd16980ea3e8b8222/numpy-2.3.3-cp311-cp311-win32.whl", hash = "sha256:eb63d443d7b4ffd1e873f8155260d7f58e7e4b095961b01c91062935c2491e57", size = 6599794 },
-    { url = "https://files.pythonhosted.org/packages/0a/a2/a4f78cb2241fe5664a22a10332f2be886dcdea8784c9f6a01c272da9b426/numpy-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:ec9d249840f6a565f58d8f913bccac2444235025bbb13e9a4681783572ee3caa", size = 13088104 },
-    { url = "https://files.pythonhosted.org/packages/79/64/e424e975adbd38282ebcd4891661965b78783de893b381cbc4832fb9beb2/numpy-2.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:74c2a948d02f88c11a3c075d9733f1ae67d97c6bdb97f2bb542f980458b257e7", size = 10460772 },
-    { url = "https://files.pythonhosted.org/packages/51/5d/bb7fc075b762c96329147799e1bcc9176ab07ca6375ea976c475482ad5b3/numpy-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cfdd09f9c84a1a934cde1eec2267f0a43a7cd44b2cca4ff95b7c0d14d144b0bf", size = 20957014 },
-    { url = "https://files.pythonhosted.org/packages/6b/0e/c6211bb92af26517acd52125a237a92afe9c3124c6a68d3b9f81b62a0568/numpy-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb32e3cf0f762aee47ad1ddc6672988f7f27045b0783c887190545baba73aa25", size = 14185220 },
-    { url = "https://files.pythonhosted.org/packages/22/f2/07bb754eb2ede9073f4054f7c0286b0d9d2e23982e090a80d478b26d35ca/numpy-2.3.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:396b254daeb0a57b1fe0ecb5e3cff6fa79a380fa97c8f7781a6d08cd429418fe", size = 5113918 },
-    { url = "https://files.pythonhosted.org/packages/81/0a/afa51697e9fb74642f231ea36aca80fa17c8fb89f7a82abd5174023c3960/numpy-2.3.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:067e3d7159a5d8f8a0b46ee11148fc35ca9b21f61e3c49fbd0a027450e65a33b", size = 6647922 },
-    { url = "https://files.pythonhosted.org/packages/5d/f5/122d9cdb3f51c520d150fef6e87df9279e33d19a9611a87c0d2cf78a89f4/numpy-2.3.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c02d0629d25d426585fb2e45a66154081b9fa677bc92a881ff1d216bc9919a8", size = 14281991 },
-    { url = "https://files.pythonhosted.org/packages/51/64/7de3c91e821a2debf77c92962ea3fe6ac2bc45d0778c1cbe15d4fce2fd94/numpy-2.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9192da52b9745f7f0766531dcfa978b7763916f158bb63bdb8a1eca0068ab20", size = 16641643 },
-    { url = "https://files.pythonhosted.org/packages/30/e4/961a5fa681502cd0d68907818b69f67542695b74e3ceaa513918103b7e80/numpy-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cd7de500a5b66319db419dc3c345244404a164beae0d0937283b907d8152e6ea", size = 16056787 },
-    { url = "https://files.pythonhosted.org/packages/99/26/92c912b966e47fbbdf2ad556cb17e3a3088e2e1292b9833be1dfa5361a1a/numpy-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:93d4962d8f82af58f0b2eb85daaf1b3ca23fe0a85d0be8f1f2b7bb46034e56d7", size = 18579598 },
-    { url = "https://files.pythonhosted.org/packages/17/b6/fc8f82cb3520768718834f310c37d96380d9dc61bfdaf05fe5c0b7653e01/numpy-2.3.3-cp312-cp312-win32.whl", hash = "sha256:5534ed6b92f9b7dca6c0a19d6df12d41c68b991cef051d108f6dbff3babc4ebf", size = 6320800 },
-    { url = "https://files.pythonhosted.org/packages/32/ee/de999f2625b80d043d6d2d628c07d0d5555a677a3cf78fdf868d409b8766/numpy-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:497d7cad08e7092dba36e3d296fe4c97708c93daf26643a1ae4b03f6294d30eb", size = 12786615 },
-    { url = "https://files.pythonhosted.org/packages/49/6e/b479032f8a43559c383acb20816644f5f91c88f633d9271ee84f3b3a996c/numpy-2.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:ca0309a18d4dfea6fc6262a66d06c26cfe4640c3926ceec90e57791a82b6eee5", size = 10195936 },
-    { url = "https://files.pythonhosted.org/packages/7d/b9/984c2b1ee61a8b803bf63582b4ac4242cf76e2dbd663efeafcb620cc0ccb/numpy-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f5415fb78995644253370985342cd03572ef8620b934da27d77377a2285955bf", size = 20949588 },
-    { url = "https://files.pythonhosted.org/packages/a6/e4/07970e3bed0b1384d22af1e9912527ecbeb47d3b26e9b6a3bced068b3bea/numpy-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d00de139a3324e26ed5b95870ce63be7ec7352171bc69a4cf1f157a48e3eb6b7", size = 14177802 },
-    { url = "https://files.pythonhosted.org/packages/35/c7/477a83887f9de61f1203bad89cf208b7c19cc9fef0cebef65d5a1a0619f2/numpy-2.3.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:9dc13c6a5829610cc07422bc74d3ac083bd8323f14e2827d992f9e52e22cd6a6", size = 5106537 },
-    { url = "https://files.pythonhosted.org/packages/52/47/93b953bd5866a6f6986344d045a207d3f1cfbad99db29f534ea9cee5108c/numpy-2.3.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:d79715d95f1894771eb4e60fb23f065663b2298f7d22945d66877aadf33d00c7", size = 6640743 },
-    { url = "https://files.pythonhosted.org/packages/23/83/377f84aaeb800b64c0ef4de58b08769e782edcefa4fea712910b6f0afd3c/numpy-2.3.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:952cfd0748514ea7c3afc729a0fc639e61655ce4c55ab9acfab14bda4f402b4c", size = 14278881 },
-    { url = "https://files.pythonhosted.org/packages/9a/a5/bf3db6e66c4b160d6ea10b534c381a1955dfab34cb1017ea93aa33c70ed3/numpy-2.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b83648633d46f77039c29078751f80da65aa64d5622a3cd62aaef9d835b6c93", size = 16636301 },
-    { url = "https://files.pythonhosted.org/packages/a2/59/1287924242eb4fa3f9b3a2c30400f2e17eb2707020d1c5e3086fe7330717/numpy-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b001bae8cea1c7dfdb2ae2b017ed0a6f2102d7a70059df1e338e307a4c78a8ae", size = 16053645 },
-    { url = "https://files.pythonhosted.org/packages/e6/93/b3d47ed882027c35e94ac2320c37e452a549f582a5e801f2d34b56973c97/numpy-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8e9aced64054739037d42fb84c54dd38b81ee238816c948c8f3ed134665dcd86", size = 18578179 },
-    { url = "https://files.pythonhosted.org/packages/20/d9/487a2bccbf7cc9d4bfc5f0f197761a5ef27ba870f1e3bbb9afc4bbe3fcc2/numpy-2.3.3-cp313-cp313-win32.whl", hash = "sha256:9591e1221db3f37751e6442850429b3aabf7026d3b05542d102944ca7f00c8a8", size = 6312250 },
-    { url = "https://files.pythonhosted.org/packages/1b/b5/263ebbbbcede85028f30047eab3d58028d7ebe389d6493fc95ae66c636ab/numpy-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f0dadeb302887f07431910f67a14d57209ed91130be0adea2f9793f1a4f817cf", size = 12783269 },
-    { url = "https://files.pythonhosted.org/packages/fa/75/67b8ca554bbeaaeb3fac2e8bce46967a5a06544c9108ec0cf5cece559b6c/numpy-2.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:3c7cf302ac6e0b76a64c4aecf1a09e51abd9b01fc7feee80f6c43e3ab1b1dbc5", size = 10195314 },
-    { url = "https://files.pythonhosted.org/packages/11/d0/0d1ddec56b162042ddfafeeb293bac672de9b0cfd688383590090963720a/numpy-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:eda59e44957d272846bb407aad19f89dc6f58fecf3504bd144f4c5cf81a7eacc", size = 21048025 },
-    { url = "https://files.pythonhosted.org/packages/36/9e/1996ca6b6d00415b6acbdd3c42f7f03ea256e2c3f158f80bd7436a8a19f3/numpy-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:823d04112bc85ef5c4fda73ba24e6096c8f869931405a80aa8b0e604510a26bc", size = 14301053 },
-    { url = "https://files.pythonhosted.org/packages/05/24/43da09aa764c68694b76e84b3d3f0c44cb7c18cdc1ba80e48b0ac1d2cd39/numpy-2.3.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:40051003e03db4041aa325da2a0971ba41cf65714e65d296397cc0e32de6018b", size = 5229444 },
-    { url = "https://files.pythonhosted.org/packages/bc/14/50ffb0f22f7218ef8af28dd089f79f68289a7a05a208db9a2c5dcbe123c1/numpy-2.3.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:6ee9086235dd6ab7ae75aba5662f582a81ced49f0f1c6de4260a78d8f2d91a19", size = 6738039 },
-    { url = "https://files.pythonhosted.org/packages/55/52/af46ac0795e09657d45a7f4db961917314377edecf66db0e39fa7ab5c3d3/numpy-2.3.3-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94fcaa68757c3e2e668ddadeaa86ab05499a70725811e582b6a9858dd472fb30", size = 14352314 },
-    { url = "https://files.pythonhosted.org/packages/a7/b1/dc226b4c90eb9f07a3fff95c2f0db3268e2e54e5cce97c4ac91518aee71b/numpy-2.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da1a74b90e7483d6ce5244053399a614b1d6b7bc30a60d2f570e5071f8959d3e", size = 16701722 },
-    { url = "https://files.pythonhosted.org/packages/9d/9d/9d8d358f2eb5eced14dba99f110d83b5cd9a4460895230f3b396ad19a323/numpy-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2990adf06d1ecee3b3dcbb4977dfab6e9f09807598d647f04d385d29e7a3c3d3", size = 16132755 },
-    { url = "https://files.pythonhosted.org/packages/b6/27/b3922660c45513f9377b3fb42240bec63f203c71416093476ec9aa0719dc/numpy-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ed635ff692483b8e3f0fcaa8e7eb8a75ee71aa6d975388224f70821421800cea", size = 18651560 },
-    { url = "https://files.pythonhosted.org/packages/5b/8e/3ab61a730bdbbc201bb245a71102aa609f0008b9ed15255500a99cd7f780/numpy-2.3.3-cp313-cp313t-win32.whl", hash = "sha256:a333b4ed33d8dc2b373cc955ca57babc00cd6f9009991d9edc5ddbc1bac36bcd", size = 6442776 },
-    { url = "https://files.pythonhosted.org/packages/1c/3a/e22b766b11f6030dc2decdeff5c2fb1610768055603f9f3be88b6d192fb2/numpy-2.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:4384a169c4d8f97195980815d6fcad04933a7e1ab3b530921c3fef7a1c63426d", size = 12927281 },
-    { url = "https://files.pythonhosted.org/packages/7b/42/c2e2bc48c5e9b2a83423f99733950fbefd86f165b468a3d85d52b30bf782/numpy-2.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:75370986cc0bc66f4ce5110ad35aae6d182cc4ce6433c40ad151f53690130bf1", size = 10265275 },
-    { url = "https://files.pythonhosted.org/packages/6b/01/342ad585ad82419b99bcf7cebe99e61da6bedb89e213c5fd71acc467faee/numpy-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cd052f1fa6a78dee696b58a914b7229ecfa41f0a6d96dc663c1220a55e137593", size = 20951527 },
-    { url = "https://files.pythonhosted.org/packages/ef/d8/204e0d73fc1b7a9ee80ab1fe1983dd33a4d64a4e30a05364b0208e9a241a/numpy-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:414a97499480067d305fcac9716c29cf4d0d76db6ebf0bf3cbce666677f12652", size = 14186159 },
-    { url = "https://files.pythonhosted.org/packages/22/af/f11c916d08f3a18fb8ba81ab72b5b74a6e42ead4c2846d270eb19845bf74/numpy-2.3.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:50a5fe69f135f88a2be9b6ca0481a68a136f6febe1916e4920e12f1a34e708a7", size = 5114624 },
-    { url = "https://files.pythonhosted.org/packages/fb/11/0ed919c8381ac9d2ffacd63fd1f0c34d27e99cab650f0eb6f110e6ae4858/numpy-2.3.3-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:b912f2ed2b67a129e6a601e9d93d4fa37bef67e54cac442a2f588a54afe5c67a", size = 6642627 },
-    { url = "https://files.pythonhosted.org/packages/ee/83/deb5f77cb0f7ba6cb52b91ed388b47f8f3c2e9930d4665c600408d9b90b9/numpy-2.3.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e318ee0596d76d4cb3d78535dc005fa60e5ea348cd131a51e99d0bdbe0b54fe", size = 14296926 },
-    { url = "https://files.pythonhosted.org/packages/77/cc/70e59dcb84f2b005d4f306310ff0a892518cc0c8000a33d0e6faf7ca8d80/numpy-2.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce020080e4a52426202bdb6f7691c65bb55e49f261f31a8f506c9f6bc7450421", size = 16638958 },
-    { url = "https://files.pythonhosted.org/packages/b6/5a/b2ab6c18b4257e099587d5b7f903317bd7115333ad8d4ec4874278eafa61/numpy-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e6687dc183aa55dae4a705b35f9c0f8cb178bcaa2f029b241ac5356221d5c021", size = 16071920 },
-    { url = "https://files.pythonhosted.org/packages/b8/f1/8b3fdc44324a259298520dd82147ff648979bed085feeacc1250ef1656c0/numpy-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d8f3b1080782469fdc1718c4ed1d22549b5fb12af0d57d35e992158a772a37cf", size = 18577076 },
-    { url = "https://files.pythonhosted.org/packages/f0/a1/b87a284fb15a42e9274e7fcea0dad259d12ddbf07c1595b26883151ca3b4/numpy-2.3.3-cp314-cp314-win32.whl", hash = "sha256:cb248499b0bc3be66ebd6578b83e5acacf1d6cb2a77f2248ce0e40fbec5a76d0", size = 6366952 },
-    { url = "https://files.pythonhosted.org/packages/70/5f/1816f4d08f3b8f66576d8433a66f8fa35a5acfb3bbd0bf6c31183b003f3d/numpy-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:691808c2b26b0f002a032c73255d0bd89751425f379f7bcd22d140db593a96e8", size = 12919322 },
-    { url = "https://files.pythonhosted.org/packages/8c/de/072420342e46a8ea41c324a555fa90fcc11637583fb8df722936aed1736d/numpy-2.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:9ad12e976ca7b10f1774b03615a2a4bab8addce37ecc77394d8e986927dc0dfe", size = 10478630 },
-    { url = "https://files.pythonhosted.org/packages/d5/df/ee2f1c0a9de7347f14da5dd3cd3c3b034d1b8607ccb6883d7dd5c035d631/numpy-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9cc48e09feb11e1db00b320e9d30a4151f7369afb96bd0e48d942d09da3a0d00", size = 21047987 },
-    { url = "https://files.pythonhosted.org/packages/d6/92/9453bdc5a4e9e69cf4358463f25e8260e2ffc126d52e10038b9077815989/numpy-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:901bf6123879b7f251d3631967fd574690734236075082078e0571977c6a8e6a", size = 14301076 },
-    { url = "https://files.pythonhosted.org/packages/13/77/1447b9eb500f028bb44253105bd67534af60499588a5149a94f18f2ca917/numpy-2.3.3-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:7f025652034199c301049296b59fa7d52c7e625017cae4c75d8662e377bf487d", size = 5229491 },
-    { url = "https://files.pythonhosted.org/packages/3d/f9/d72221b6ca205f9736cb4b2ce3b002f6e45cd67cd6a6d1c8af11a2f0b649/numpy-2.3.3-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:533ca5f6d325c80b6007d4d7fb1984c303553534191024ec6a524a4c92a5935a", size = 6737913 },
-    { url = "https://files.pythonhosted.org/packages/3c/5f/d12834711962ad9c46af72f79bb31e73e416ee49d17f4c797f72c96b6ca5/numpy-2.3.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0edd58682a399824633b66885d699d7de982800053acf20be1eaa46d92009c54", size = 14352811 },
-    { url = "https://files.pythonhosted.org/packages/a1/0d/fdbec6629d97fd1bebed56cd742884e4eead593611bbe1abc3eb40d304b2/numpy-2.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:367ad5d8fbec5d9296d18478804a530f1191e24ab4d75ab408346ae88045d25e", size = 16702689 },
-    { url = "https://files.pythonhosted.org/packages/9b/09/0a35196dc5575adde1eb97ddfbc3e1687a814f905377621d18ca9bc2b7dd/numpy-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8f6ac61a217437946a1fa48d24c47c91a0c4f725237871117dea264982128097", size = 16133855 },
-    { url = "https://files.pythonhosted.org/packages/7a/ca/c9de3ea397d576f1b6753eaa906d4cdef1bf97589a6d9825a349b4729cc2/numpy-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:179a42101b845a816d464b6fe9a845dfaf308fdfc7925387195570789bb2c970", size = 18652520 },
-    { url = "https://files.pythonhosted.org/packages/fd/c2/e5ed830e08cd0196351db55db82f65bc0ab05da6ef2b72a836dcf1936d2f/numpy-2.3.3-cp314-cp314t-win32.whl", hash = "sha256:1250c5d3d2562ec4174bce2e3a1523041595f9b651065e4a4473f5f48a6bc8a5", size = 6515371 },
-    { url = "https://files.pythonhosted.org/packages/47/c7/b0f6b5b67f6788a0725f744496badbb604d226bf233ba716683ebb47b570/numpy-2.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b37a0b2e5935409daebe82c1e42274d30d9dd355852529eab91dab8dcca7419f", size = 13112576 },
-    { url = "https://files.pythonhosted.org/packages/06/b9/33bba5ff6fb679aa0b1f8a07e853f002a6b04b9394db3069a1270a7784ca/numpy-2.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:78c9f6560dc7e6b3990e32df7ea1a50bbd0e2a111e05209963f5ddcab7073b0b", size = 10545953 },
-    { url = "https://files.pythonhosted.org/packages/b8/f2/7e0a37cfced2644c9563c529f29fa28acbd0960dde32ece683aafa6f4949/numpy-2.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1e02c7159791cd481e1e6d5ddd766b62a4d5acf8df4d4d1afe35ee9c5c33a41e", size = 21131019 },
-    { url = "https://files.pythonhosted.org/packages/1a/7e/3291f505297ed63831135a6cc0f474da0c868a1f31b0dd9a9f03a7a0d2ed/numpy-2.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:dca2d0fc80b3893ae72197b39f69d55a3cd8b17ea1b50aa4c62de82419936150", size = 14376288 },
-    { url = "https://files.pythonhosted.org/packages/bf/4b/ae02e985bdeee73d7b5abdefeb98aef1207e96d4c0621ee0cf228ddfac3c/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:99683cbe0658f8271b333a1b1b4bb3173750ad59c0c61f5bbdc5b318918fffe3", size = 5305425 },
-    { url = "https://files.pythonhosted.org/packages/8b/eb/9df215d6d7250db32007941500dc51c48190be25f2401d5b2b564e467247/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:d9d537a39cc9de668e5cd0e25affb17aec17b577c6b3ae8a3d866b479fbe88d0", size = 6819053 },
-    { url = "https://files.pythonhosted.org/packages/57/62/208293d7d6b2a8998a4a1f23ac758648c3c32182d4ce4346062018362e29/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8596ba2f8af5f93b01d97563832686d20206d303024777f6dfc2e7c7c3f1850e", size = 14420354 },
-    { url = "https://files.pythonhosted.org/packages/ed/0c/8e86e0ff7072e14a71b4c6af63175e40d1e7e933ce9b9e9f765a95b4e0c3/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1ec5615b05369925bd1125f27df33f3b6c8bc10d788d5999ecd8769a1fa04db", size = 16760413 },
-    { url = "https://files.pythonhosted.org/packages/af/11/0cc63f9f321ccf63886ac203336777140011fb669e739da36d8db3c53b98/numpy-2.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2e267c7da5bf7309670523896df97f93f6e469fb931161f483cd6882b3b1a5dc", size = 12971844 },
+sdist = { url = "https://files.pythonhosted.org/packages/d0/19/95b3d357407220ed24c139018d2518fab0a61a948e68286a25f1a4d049ff/numpy-2.3.3.tar.gz", hash = "sha256:ddc7c39727ba62b80dfdbedf400d1c10ddfa8eefbd7ec8dcb118be8b56d31029", size = 20576648, upload-time = "2025-09-09T16:54:12.543Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7a/45/e80d203ef6b267aa29b22714fb558930b27960a0c5ce3c19c999232bb3eb/numpy-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ffc4f5caba7dfcbe944ed674b7eef683c7e94874046454bb79ed7ee0236f59d", size = 21259253, upload-time = "2025-09-09T15:56:02.094Z" },
+    { url = "https://files.pythonhosted.org/packages/52/18/cf2c648fccf339e59302e00e5f2bc87725a3ce1992f30f3f78c9044d7c43/numpy-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7e946c7170858a0295f79a60214424caac2ffdb0063d4d79cb681f9aa0aa569", size = 14450980, upload-time = "2025-09-09T15:56:05.926Z" },
+    { url = "https://files.pythonhosted.org/packages/93/fb/9af1082bec870188c42a1c239839915b74a5099c392389ff04215dcee812/numpy-2.3.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:cd4260f64bc794c3390a63bf0728220dd1a68170c169088a1e0dfa2fde1be12f", size = 5379709, upload-time = "2025-09-09T15:56:07.95Z" },
+    { url = "https://files.pythonhosted.org/packages/75/0f/bfd7abca52bcbf9a4a65abc83fe18ef01ccdeb37bfb28bbd6ad613447c79/numpy-2.3.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:f0ddb4b96a87b6728df9362135e764eac3cfa674499943ebc44ce96c478ab125", size = 6913923, upload-time = "2025-09-09T15:56:09.443Z" },
+    { url = "https://files.pythonhosted.org/packages/79/55/d69adad255e87ab7afda1caf93ca997859092afeb697703e2f010f7c2e55/numpy-2.3.3-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:afd07d377f478344ec6ca2b8d4ca08ae8bd44706763d1efb56397de606393f48", size = 14589591, upload-time = "2025-09-09T15:56:11.234Z" },
+    { url = "https://files.pythonhosted.org/packages/10/a2/010b0e27ddeacab7839957d7a8f00e91206e0c2c47abbb5f35a2630e5387/numpy-2.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc92a5dedcc53857249ca51ef29f5e5f2f8c513e22cfb90faeb20343b8c6f7a6", size = 16938714, upload-time = "2025-09-09T15:56:14.637Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/6b/12ce8ede632c7126eb2762b9e15e18e204b81725b81f35176eac14dc5b82/numpy-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7af05ed4dc19f308e1d9fc759f36f21921eb7bbfc82843eeec6b2a2863a0aefa", size = 16370592, upload-time = "2025-09-09T15:56:17.285Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/35/aba8568b2593067bb6a8fe4c52babb23b4c3b9c80e1b49dff03a09925e4a/numpy-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:433bf137e338677cebdd5beac0199ac84712ad9d630b74eceeb759eaa45ddf30", size = 18884474, upload-time = "2025-09-09T15:56:20.943Z" },
+    { url = "https://files.pythonhosted.org/packages/45/fa/7f43ba10c77575e8be7b0138d107e4f44ca4a1ef322cd16980ea3e8b8222/numpy-2.3.3-cp311-cp311-win32.whl", hash = "sha256:eb63d443d7b4ffd1e873f8155260d7f58e7e4b095961b01c91062935c2491e57", size = 6599794, upload-time = "2025-09-09T15:56:23.258Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/a2/a4f78cb2241fe5664a22a10332f2be886dcdea8784c9f6a01c272da9b426/numpy-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:ec9d249840f6a565f58d8f913bccac2444235025bbb13e9a4681783572ee3caa", size = 13088104, upload-time = "2025-09-09T15:56:25.476Z" },
+    { url = "https://files.pythonhosted.org/packages/79/64/e424e975adbd38282ebcd4891661965b78783de893b381cbc4832fb9beb2/numpy-2.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:74c2a948d02f88c11a3c075d9733f1ae67d97c6bdb97f2bb542f980458b257e7", size = 10460772, upload-time = "2025-09-09T15:56:27.679Z" },
+    { url = "https://files.pythonhosted.org/packages/51/5d/bb7fc075b762c96329147799e1bcc9176ab07ca6375ea976c475482ad5b3/numpy-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cfdd09f9c84a1a934cde1eec2267f0a43a7cd44b2cca4ff95b7c0d14d144b0bf", size = 20957014, upload-time = "2025-09-09T15:56:29.966Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/0e/c6211bb92af26517acd52125a237a92afe9c3124c6a68d3b9f81b62a0568/numpy-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb32e3cf0f762aee47ad1ddc6672988f7f27045b0783c887190545baba73aa25", size = 14185220, upload-time = "2025-09-09T15:56:32.175Z" },
+    { url = "https://files.pythonhosted.org/packages/22/f2/07bb754eb2ede9073f4054f7c0286b0d9d2e23982e090a80d478b26d35ca/numpy-2.3.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:396b254daeb0a57b1fe0ecb5e3cff6fa79a380fa97c8f7781a6d08cd429418fe", size = 5113918, upload-time = "2025-09-09T15:56:34.175Z" },
+    { url = "https://files.pythonhosted.org/packages/81/0a/afa51697e9fb74642f231ea36aca80fa17c8fb89f7a82abd5174023c3960/numpy-2.3.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:067e3d7159a5d8f8a0b46ee11148fc35ca9b21f61e3c49fbd0a027450e65a33b", size = 6647922, upload-time = "2025-09-09T15:56:36.149Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/f5/122d9cdb3f51c520d150fef6e87df9279e33d19a9611a87c0d2cf78a89f4/numpy-2.3.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c02d0629d25d426585fb2e45a66154081b9fa677bc92a881ff1d216bc9919a8", size = 14281991, upload-time = "2025-09-09T15:56:40.548Z" },
+    { url = "https://files.pythonhosted.org/packages/51/64/7de3c91e821a2debf77c92962ea3fe6ac2bc45d0778c1cbe15d4fce2fd94/numpy-2.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9192da52b9745f7f0766531dcfa978b7763916f158bb63bdb8a1eca0068ab20", size = 16641643, upload-time = "2025-09-09T15:56:43.343Z" },
+    { url = "https://files.pythonhosted.org/packages/30/e4/961a5fa681502cd0d68907818b69f67542695b74e3ceaa513918103b7e80/numpy-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cd7de500a5b66319db419dc3c345244404a164beae0d0937283b907d8152e6ea", size = 16056787, upload-time = "2025-09-09T15:56:46.141Z" },
+    { url = "https://files.pythonhosted.org/packages/99/26/92c912b966e47fbbdf2ad556cb17e3a3088e2e1292b9833be1dfa5361a1a/numpy-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:93d4962d8f82af58f0b2eb85daaf1b3ca23fe0a85d0be8f1f2b7bb46034e56d7", size = 18579598, upload-time = "2025-09-09T15:56:49.844Z" },
+    { url = "https://files.pythonhosted.org/packages/17/b6/fc8f82cb3520768718834f310c37d96380d9dc61bfdaf05fe5c0b7653e01/numpy-2.3.3-cp312-cp312-win32.whl", hash = "sha256:5534ed6b92f9b7dca6c0a19d6df12d41c68b991cef051d108f6dbff3babc4ebf", size = 6320800, upload-time = "2025-09-09T15:56:52.499Z" },
+    { url = "https://files.pythonhosted.org/packages/32/ee/de999f2625b80d043d6d2d628c07d0d5555a677a3cf78fdf868d409b8766/numpy-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:497d7cad08e7092dba36e3d296fe4c97708c93daf26643a1ae4b03f6294d30eb", size = 12786615, upload-time = "2025-09-09T15:56:54.422Z" },
+    { url = "https://files.pythonhosted.org/packages/49/6e/b479032f8a43559c383acb20816644f5f91c88f633d9271ee84f3b3a996c/numpy-2.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:ca0309a18d4dfea6fc6262a66d06c26cfe4640c3926ceec90e57791a82b6eee5", size = 10195936, upload-time = "2025-09-09T15:56:56.541Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/b9/984c2b1ee61a8b803bf63582b4ac4242cf76e2dbd663efeafcb620cc0ccb/numpy-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f5415fb78995644253370985342cd03572ef8620b934da27d77377a2285955bf", size = 20949588, upload-time = "2025-09-09T15:56:59.087Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/e4/07970e3bed0b1384d22af1e9912527ecbeb47d3b26e9b6a3bced068b3bea/numpy-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d00de139a3324e26ed5b95870ce63be7ec7352171bc69a4cf1f157a48e3eb6b7", size = 14177802, upload-time = "2025-09-09T15:57:01.73Z" },
+    { url = "https://files.pythonhosted.org/packages/35/c7/477a83887f9de61f1203bad89cf208b7c19cc9fef0cebef65d5a1a0619f2/numpy-2.3.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:9dc13c6a5829610cc07422bc74d3ac083bd8323f14e2827d992f9e52e22cd6a6", size = 5106537, upload-time = "2025-09-09T15:57:03.765Z" },
+    { url = "https://files.pythonhosted.org/packages/52/47/93b953bd5866a6f6986344d045a207d3f1cfbad99db29f534ea9cee5108c/numpy-2.3.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:d79715d95f1894771eb4e60fb23f065663b2298f7d22945d66877aadf33d00c7", size = 6640743, upload-time = "2025-09-09T15:57:07.921Z" },
+    { url = "https://files.pythonhosted.org/packages/23/83/377f84aaeb800b64c0ef4de58b08769e782edcefa4fea712910b6f0afd3c/numpy-2.3.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:952cfd0748514ea7c3afc729a0fc639e61655ce4c55ab9acfab14bda4f402b4c", size = 14278881, upload-time = "2025-09-09T15:57:11.349Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/a5/bf3db6e66c4b160d6ea10b534c381a1955dfab34cb1017ea93aa33c70ed3/numpy-2.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b83648633d46f77039c29078751f80da65aa64d5622a3cd62aaef9d835b6c93", size = 16636301, upload-time = "2025-09-09T15:57:14.245Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/59/1287924242eb4fa3f9b3a2c30400f2e17eb2707020d1c5e3086fe7330717/numpy-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b001bae8cea1c7dfdb2ae2b017ed0a6f2102d7a70059df1e338e307a4c78a8ae", size = 16053645, upload-time = "2025-09-09T15:57:16.534Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/93/b3d47ed882027c35e94ac2320c37e452a549f582a5e801f2d34b56973c97/numpy-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8e9aced64054739037d42fb84c54dd38b81ee238816c948c8f3ed134665dcd86", size = 18578179, upload-time = "2025-09-09T15:57:18.883Z" },
+    { url = "https://files.pythonhosted.org/packages/20/d9/487a2bccbf7cc9d4bfc5f0f197761a5ef27ba870f1e3bbb9afc4bbe3fcc2/numpy-2.3.3-cp313-cp313-win32.whl", hash = "sha256:9591e1221db3f37751e6442850429b3aabf7026d3b05542d102944ca7f00c8a8", size = 6312250, upload-time = "2025-09-09T15:57:21.296Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/b5/263ebbbbcede85028f30047eab3d58028d7ebe389d6493fc95ae66c636ab/numpy-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f0dadeb302887f07431910f67a14d57209ed91130be0adea2f9793f1a4f817cf", size = 12783269, upload-time = "2025-09-09T15:57:23.034Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/75/67b8ca554bbeaaeb3fac2e8bce46967a5a06544c9108ec0cf5cece559b6c/numpy-2.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:3c7cf302ac6e0b76a64c4aecf1a09e51abd9b01fc7feee80f6c43e3ab1b1dbc5", size = 10195314, upload-time = "2025-09-09T15:57:25.045Z" },
+    { url = "https://files.pythonhosted.org/packages/11/d0/0d1ddec56b162042ddfafeeb293bac672de9b0cfd688383590090963720a/numpy-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:eda59e44957d272846bb407aad19f89dc6f58fecf3504bd144f4c5cf81a7eacc", size = 21048025, upload-time = "2025-09-09T15:57:27.257Z" },
+    { url = "https://files.pythonhosted.org/packages/36/9e/1996ca6b6d00415b6acbdd3c42f7f03ea256e2c3f158f80bd7436a8a19f3/numpy-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:823d04112bc85ef5c4fda73ba24e6096c8f869931405a80aa8b0e604510a26bc", size = 14301053, upload-time = "2025-09-09T15:57:30.077Z" },
+    { url = "https://files.pythonhosted.org/packages/05/24/43da09aa764c68694b76e84b3d3f0c44cb7c18cdc1ba80e48b0ac1d2cd39/numpy-2.3.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:40051003e03db4041aa325da2a0971ba41cf65714e65d296397cc0e32de6018b", size = 5229444, upload-time = "2025-09-09T15:57:32.733Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/14/50ffb0f22f7218ef8af28dd089f79f68289a7a05a208db9a2c5dcbe123c1/numpy-2.3.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:6ee9086235dd6ab7ae75aba5662f582a81ced49f0f1c6de4260a78d8f2d91a19", size = 6738039, upload-time = "2025-09-09T15:57:34.328Z" },
+    { url = "https://files.pythonhosted.org/packages/55/52/af46ac0795e09657d45a7f4db961917314377edecf66db0e39fa7ab5c3d3/numpy-2.3.3-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94fcaa68757c3e2e668ddadeaa86ab05499a70725811e582b6a9858dd472fb30", size = 14352314, upload-time = "2025-09-09T15:57:36.255Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/b1/dc226b4c90eb9f07a3fff95c2f0db3268e2e54e5cce97c4ac91518aee71b/numpy-2.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da1a74b90e7483d6ce5244053399a614b1d6b7bc30a60d2f570e5071f8959d3e", size = 16701722, upload-time = "2025-09-09T15:57:38.622Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/9d/9d8d358f2eb5eced14dba99f110d83b5cd9a4460895230f3b396ad19a323/numpy-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2990adf06d1ecee3b3dcbb4977dfab6e9f09807598d647f04d385d29e7a3c3d3", size = 16132755, upload-time = "2025-09-09T15:57:41.16Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/27/b3922660c45513f9377b3fb42240bec63f203c71416093476ec9aa0719dc/numpy-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ed635ff692483b8e3f0fcaa8e7eb8a75ee71aa6d975388224f70821421800cea", size = 18651560, upload-time = "2025-09-09T15:57:43.459Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/8e/3ab61a730bdbbc201bb245a71102aa609f0008b9ed15255500a99cd7f780/numpy-2.3.3-cp313-cp313t-win32.whl", hash = "sha256:a333b4ed33d8dc2b373cc955ca57babc00cd6f9009991d9edc5ddbc1bac36bcd", size = 6442776, upload-time = "2025-09-09T15:57:45.793Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/3a/e22b766b11f6030dc2decdeff5c2fb1610768055603f9f3be88b6d192fb2/numpy-2.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:4384a169c4d8f97195980815d6fcad04933a7e1ab3b530921c3fef7a1c63426d", size = 12927281, upload-time = "2025-09-09T15:57:47.492Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/42/c2e2bc48c5e9b2a83423f99733950fbefd86f165b468a3d85d52b30bf782/numpy-2.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:75370986cc0bc66f4ce5110ad35aae6d182cc4ce6433c40ad151f53690130bf1", size = 10265275, upload-time = "2025-09-09T15:57:49.647Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/01/342ad585ad82419b99bcf7cebe99e61da6bedb89e213c5fd71acc467faee/numpy-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cd052f1fa6a78dee696b58a914b7229ecfa41f0a6d96dc663c1220a55e137593", size = 20951527, upload-time = "2025-09-09T15:57:52.006Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/d8/204e0d73fc1b7a9ee80ab1fe1983dd33a4d64a4e30a05364b0208e9a241a/numpy-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:414a97499480067d305fcac9716c29cf4d0d76db6ebf0bf3cbce666677f12652", size = 14186159, upload-time = "2025-09-09T15:57:54.407Z" },
+    { url = "https://files.pythonhosted.org/packages/22/af/f11c916d08f3a18fb8ba81ab72b5b74a6e42ead4c2846d270eb19845bf74/numpy-2.3.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:50a5fe69f135f88a2be9b6ca0481a68a136f6febe1916e4920e12f1a34e708a7", size = 5114624, upload-time = "2025-09-09T15:57:56.5Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/11/0ed919c8381ac9d2ffacd63fd1f0c34d27e99cab650f0eb6f110e6ae4858/numpy-2.3.3-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:b912f2ed2b67a129e6a601e9d93d4fa37bef67e54cac442a2f588a54afe5c67a", size = 6642627, upload-time = "2025-09-09T15:57:58.206Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/83/deb5f77cb0f7ba6cb52b91ed388b47f8f3c2e9930d4665c600408d9b90b9/numpy-2.3.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e318ee0596d76d4cb3d78535dc005fa60e5ea348cd131a51e99d0bdbe0b54fe", size = 14296926, upload-time = "2025-09-09T15:58:00.035Z" },
+    { url = "https://files.pythonhosted.org/packages/77/cc/70e59dcb84f2b005d4f306310ff0a892518cc0c8000a33d0e6faf7ca8d80/numpy-2.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce020080e4a52426202bdb6f7691c65bb55e49f261f31a8f506c9f6bc7450421", size = 16638958, upload-time = "2025-09-09T15:58:02.738Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/5a/b2ab6c18b4257e099587d5b7f903317bd7115333ad8d4ec4874278eafa61/numpy-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e6687dc183aa55dae4a705b35f9c0f8cb178bcaa2f029b241ac5356221d5c021", size = 16071920, upload-time = "2025-09-09T15:58:05.029Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/f1/8b3fdc44324a259298520dd82147ff648979bed085feeacc1250ef1656c0/numpy-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d8f3b1080782469fdc1718c4ed1d22549b5fb12af0d57d35e992158a772a37cf", size = 18577076, upload-time = "2025-09-09T15:58:07.745Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/a1/b87a284fb15a42e9274e7fcea0dad259d12ddbf07c1595b26883151ca3b4/numpy-2.3.3-cp314-cp314-win32.whl", hash = "sha256:cb248499b0bc3be66ebd6578b83e5acacf1d6cb2a77f2248ce0e40fbec5a76d0", size = 6366952, upload-time = "2025-09-09T15:58:10.096Z" },
+    { url = "https://files.pythonhosted.org/packages/70/5f/1816f4d08f3b8f66576d8433a66f8fa35a5acfb3bbd0bf6c31183b003f3d/numpy-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:691808c2b26b0f002a032c73255d0bd89751425f379f7bcd22d140db593a96e8", size = 12919322, upload-time = "2025-09-09T15:58:12.138Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/de/072420342e46a8ea41c324a555fa90fcc11637583fb8df722936aed1736d/numpy-2.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:9ad12e976ca7b10f1774b03615a2a4bab8addce37ecc77394d8e986927dc0dfe", size = 10478630, upload-time = "2025-09-09T15:58:14.64Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/df/ee2f1c0a9de7347f14da5dd3cd3c3b034d1b8607ccb6883d7dd5c035d631/numpy-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9cc48e09feb11e1db00b320e9d30a4151f7369afb96bd0e48d942d09da3a0d00", size = 21047987, upload-time = "2025-09-09T15:58:16.889Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/92/9453bdc5a4e9e69cf4358463f25e8260e2ffc126d52e10038b9077815989/numpy-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:901bf6123879b7f251d3631967fd574690734236075082078e0571977c6a8e6a", size = 14301076, upload-time = "2025-09-09T15:58:20.343Z" },
+    { url = "https://files.pythonhosted.org/packages/13/77/1447b9eb500f028bb44253105bd67534af60499588a5149a94f18f2ca917/numpy-2.3.3-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:7f025652034199c301049296b59fa7d52c7e625017cae4c75d8662e377bf487d", size = 5229491, upload-time = "2025-09-09T15:58:22.481Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/f9/d72221b6ca205f9736cb4b2ce3b002f6e45cd67cd6a6d1c8af11a2f0b649/numpy-2.3.3-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:533ca5f6d325c80b6007d4d7fb1984c303553534191024ec6a524a4c92a5935a", size = 6737913, upload-time = "2025-09-09T15:58:24.569Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/5f/d12834711962ad9c46af72f79bb31e73e416ee49d17f4c797f72c96b6ca5/numpy-2.3.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0edd58682a399824633b66885d699d7de982800053acf20be1eaa46d92009c54", size = 14352811, upload-time = "2025-09-09T15:58:26.416Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/0d/fdbec6629d97fd1bebed56cd742884e4eead593611bbe1abc3eb40d304b2/numpy-2.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:367ad5d8fbec5d9296d18478804a530f1191e24ab4d75ab408346ae88045d25e", size = 16702689, upload-time = "2025-09-09T15:58:28.831Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/09/0a35196dc5575adde1eb97ddfbc3e1687a814f905377621d18ca9bc2b7dd/numpy-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8f6ac61a217437946a1fa48d24c47c91a0c4f725237871117dea264982128097", size = 16133855, upload-time = "2025-09-09T15:58:31.349Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/ca/c9de3ea397d576f1b6753eaa906d4cdef1bf97589a6d9825a349b4729cc2/numpy-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:179a42101b845a816d464b6fe9a845dfaf308fdfc7925387195570789bb2c970", size = 18652520, upload-time = "2025-09-09T15:58:33.762Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/c2/e5ed830e08cd0196351db55db82f65bc0ab05da6ef2b72a836dcf1936d2f/numpy-2.3.3-cp314-cp314t-win32.whl", hash = "sha256:1250c5d3d2562ec4174bce2e3a1523041595f9b651065e4a4473f5f48a6bc8a5", size = 6515371, upload-time = "2025-09-09T15:58:36.04Z" },
+    { url = "https://files.pythonhosted.org/packages/47/c7/b0f6b5b67f6788a0725f744496badbb604d226bf233ba716683ebb47b570/numpy-2.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b37a0b2e5935409daebe82c1e42274d30d9dd355852529eab91dab8dcca7419f", size = 13112576, upload-time = "2025-09-09T15:58:37.927Z" },
+    { url = "https://files.pythonhosted.org/packages/06/b9/33bba5ff6fb679aa0b1f8a07e853f002a6b04b9394db3069a1270a7784ca/numpy-2.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:78c9f6560dc7e6b3990e32df7ea1a50bbd0e2a111e05209963f5ddcab7073b0b", size = 10545953, upload-time = "2025-09-09T15:58:40.576Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/f2/7e0a37cfced2644c9563c529f29fa28acbd0960dde32ece683aafa6f4949/numpy-2.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1e02c7159791cd481e1e6d5ddd766b62a4d5acf8df4d4d1afe35ee9c5c33a41e", size = 21131019, upload-time = "2025-09-09T15:58:42.838Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/7e/3291f505297ed63831135a6cc0f474da0c868a1f31b0dd9a9f03a7a0d2ed/numpy-2.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:dca2d0fc80b3893ae72197b39f69d55a3cd8b17ea1b50aa4c62de82419936150", size = 14376288, upload-time = "2025-09-09T15:58:45.425Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/4b/ae02e985bdeee73d7b5abdefeb98aef1207e96d4c0621ee0cf228ddfac3c/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:99683cbe0658f8271b333a1b1b4bb3173750ad59c0c61f5bbdc5b318918fffe3", size = 5305425, upload-time = "2025-09-09T15:58:48.6Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/eb/9df215d6d7250db32007941500dc51c48190be25f2401d5b2b564e467247/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:d9d537a39cc9de668e5cd0e25affb17aec17b577c6b3ae8a3d866b479fbe88d0", size = 6819053, upload-time = "2025-09-09T15:58:50.401Z" },
+    { url = "https://files.pythonhosted.org/packages/57/62/208293d7d6b2a8998a4a1f23ac758648c3c32182d4ce4346062018362e29/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8596ba2f8af5f93b01d97563832686d20206d303024777f6dfc2e7c7c3f1850e", size = 14420354, upload-time = "2025-09-09T15:58:52.704Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/0c/8e86e0ff7072e14a71b4c6af63175e40d1e7e933ce9b9e9f765a95b4e0c3/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1ec5615b05369925bd1125f27df33f3b6c8bc10d788d5999ecd8769a1fa04db", size = 16760413, upload-time = "2025-09-09T15:58:55.027Z" },
+    { url = "https://files.pythonhosted.org/packages/af/11/0cc63f9f321ccf63886ac203336777140011fb669e739da36d8db3c53b98/numpy-2.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2e267c7da5bf7309670523896df97f93f6e469fb931161f483cd6882b3b1a5dc", size = 12971844, upload-time = "2025-09-09T15:58:57.359Z" },
 ]
 
 [[package]]
@@ -1381,7 +1715,7 @@ name = "nvidia-cublas-cu12"
 version = "12.8.4.1"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921 },
+    { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" },
 ]
 
 [[package]]
@@ -1389,7 +1723,7 @@ name = "nvidia-cuda-cupti-cu12"
 version = "12.8.90"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621 },
+    { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" },
 ]
 
 [[package]]
@@ -1397,7 +1731,7 @@ name = "nvidia-cuda-nvrtc-cu12"
 version = "12.8.93"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029 },
+    { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" },
 ]
 
 [[package]]
@@ -1405,7 +1739,7 @@ name = "nvidia-cuda-runtime-cu12"
 version = "12.8.90"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765 },
+    { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" },
 ]
 
 [[package]]
@@ -1416,7 +1750,7 @@ dependencies = [
     { name = "nvidia-cublas-cu12" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467 },
+    { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" },
 ]
 
 [[package]]
@@ -1427,7 +1761,7 @@ dependencies = [
     { name = "nvidia-nvjitlink-cu12" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695 },
+    { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" },
 ]
 
 [[package]]
@@ -1435,7 +1769,7 @@ name = "nvidia-cufile-cu12"
 version = "1.13.1.3"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834 },
+    { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" },
 ]
 
 [[package]]
@@ -1443,7 +1777,7 @@ name = "nvidia-curand-cu12"
 version = "10.3.9.90"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976 },
+    { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" },
 ]
 
 [[package]]
@@ -1456,7 +1790,7 @@ dependencies = [
     { name = "nvidia-nvjitlink-cu12" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905 },
+    { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" },
 ]
 
 [[package]]
@@ -1467,7 +1801,7 @@ dependencies = [
     { name = "nvidia-nvjitlink-cu12" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466 },
+    { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" },
 ]
 
 [[package]]
@@ -1475,7 +1809,7 @@ name = "nvidia-cusparselt-cu12"
 version = "0.7.1"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691 },
+    { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" },
 ]
 
 [[package]]
@@ -1483,7 +1817,7 @@ name = "nvidia-nccl-cu12"
 version = "2.27.3"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/5c/5b/4e4fff7bad39adf89f735f2bc87248c81db71205b62bcc0d5ca5b606b3c3/nvidia_nccl_cu12-2.27.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adf27ccf4238253e0b826bce3ff5fa532d65fc42322c8bfdfaf28024c0fbe039", size = 322364134 },
+    { url = "https://files.pythonhosted.org/packages/5c/5b/4e4fff7bad39adf89f735f2bc87248c81db71205b62bcc0d5ca5b606b3c3/nvidia_nccl_cu12-2.27.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adf27ccf4238253e0b826bce3ff5fa532d65fc42322c8bfdfaf28024c0fbe039", size = 322364134, upload-time = "2025-06-03T21:58:04.013Z" },
 ]
 
 [[package]]
@@ -1491,7 +1825,7 @@ name = "nvidia-nvjitlink-cu12"
 version = "12.8.93"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836 },
+    { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" },
 ]
 
 [[package]]
@@ -1499,16 +1833,16 @@ name = "nvidia-nvtx-cu12"
 version = "12.8.90"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954 },
+    { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" },
 ]
 
 [[package]]
 name = "opt-einsum"
 version = "3.4.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/8c/b9/2ac072041e899a52f20cf9510850ff58295003aa75525e58343591b0cbfb/opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac", size = 63004 }
+sdist = { url = "https://files.pythonhosted.org/packages/8c/b9/2ac072041e899a52f20cf9510850ff58295003aa75525e58343591b0cbfb/opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac", size = 63004, upload-time = "2024-09-26T14:33:24.483Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/23/cd/066e86230ae37ed0be70aae89aabf03ca8d9f39c8aea0dec8029455b5540/opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd", size = 71932 },
+    { url = "https://files.pythonhosted.org/packages/23/cd/066e86230ae37ed0be70aae89aabf03ca8d9f39c8aea0dec8029455b5540/opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd", size = 71932, upload-time = "2024-09-26T14:33:23.039Z" },
 ]
 
 [[package]]
@@ -1516,58 +1850,65 @@ name = "optree"
 version = "0.17.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "typing-extensions" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/56/c7/0853e0c59b135dff770615d2713b547b6b3b5cde7c10995b4a5825244612/optree-0.17.0.tar.gz", hash = "sha256:5335a5ec44479920620d72324c66563bd705ab2a698605dd4b6ee67dbcad7ecd", size = 163111 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/91/f9/6ca076fd4c6f16be031afdc711a2676c1ff15bd1717ee2e699179b1a29bc/optree-0.17.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98990201f352dba253af1a995c1453818db5f08de4cae7355d85aa6023676a52", size = 350398 },
-    { url = "https://files.pythonhosted.org/packages/95/4c/81344cbdcf8ea8525a21c9d65892d7529010ee2146c53423b2e9a84441ba/optree-0.17.0-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:e1a40adf6bb78a6a4b4f480879de2cb6b57d46d680a4d9834aa824f41e69c0d9", size = 404834 },
-    { url = "https://files.pythonhosted.org/packages/e5/c4/ac1880372a89f5c21514a7965dfa23b1afb2ad683fb9804d366727de9ecf/optree-0.17.0-cp310-cp310-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:78a113436a0a440f900b2799584f3cc2b2eea1b245d81c3583af42ac003e333c", size = 402116 },
-    { url = "https://files.pythonhosted.org/packages/ff/72/ad6be4d6a03805cf3921b492494cb3371ca28060d5ad19d5a36e10c4d67d/optree-0.17.0-cp310-cp310-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e45c16018f4283f028cf839b707b7ac734e8056a31b7198a1577161fcbe146d", size = 398491 },
-    { url = "https://files.pythonhosted.org/packages/d9/c1/6827fb504351f9a3935699b0eb31c8a6af59d775ee78289a25e0ba54f732/optree-0.17.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b698613d821d80cc216a2444ebc3145c8bf671b55a2223058a6574c1483a65f6", size = 387957 },
-    { url = "https://files.pythonhosted.org/packages/73/5c/13a2a864b0c0b39c3c193be534a195a3ab2463c7d0443d4a76e749e3ff83/optree-0.17.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3080c564c9760711aa72d1b4d700ce1417f99ad087136f415c4eb8221169e2a3", size = 362797 },
-    { url = "https://files.pythonhosted.org/packages/da/f5/ff7dcb5a0108ee89c2be09aed2ebd26a7e1333d8122031aa9d9322b24ee6/optree-0.17.0-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:834a8fb358b608240b3a38706a09b43974675624485fad64c8ee641dae2eb57d", size = 419450 },
-    { url = "https://files.pythonhosted.org/packages/1b/e6/48a97aefd18770b55e5ed456d8183891f325cdb6d90592e5f072ed6951f8/optree-0.17.0-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1a2bd263e6b5621d000d0f94de1f245414fd5dbce365a24b7b89b1ed0ef56cf9", size = 417557 },
-    { url = "https://files.pythonhosted.org/packages/c4/b1/4e280edab8a86be47ec1f9bd9ed4b685d2e15f0950ae62b613b26d12a1da/optree-0.17.0-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9b37daca4ad89339b1f5320cc61ac600dcf976adbb060769d36d5542d6ebfedf", size = 414174 },
-    { url = "https://files.pythonhosted.org/packages/db/3b/49a9a1986215dd342525974deeb17c260a83fee8fad147276fd710ac8718/optree-0.17.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a146a6917f3e28cfdc268ff1770aa696c346482dd3da681c3ff92153d94450ea", size = 402000 },
-    { url = "https://files.pythonhosted.org/packages/41/90/e12dea2cb5d8a5e17bbe3011ed4e972b89c027272a816db4897589751cad/optree-0.17.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e13ae51a63d69db445f269a3a4fd1d6edb064a705188d007ea47c9f034788fc5", size = 365869 },
-    { url = "https://files.pythonhosted.org/packages/76/ee/21af214663960a479863cd6c03d7a0abc8123ea22a6ea34689c2eed88ccd/optree-0.17.0-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:5958f58423cc7870cb011c8c8f92687397380886e8c9d33adac752147e7bbc3f", size = 424465 },
-    { url = "https://files.pythonhosted.org/packages/54/a3/64b184a79373753f4f46a5cd301ea581f71d6dc1a5c103bd2394f0925d40/optree-0.17.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:970ae4e47727b4c5526fc583b87d29190e576f6a2b6c19e8671589b73d256250", size = 420686 },
-    { url = "https://files.pythonhosted.org/packages/6c/6d/b6051b0b1ef9a49df96a66e9e62fc02620d2115d1ba659888c94e67fcfc9/optree-0.17.0-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54177fd3e6e05c08b66329e26d7d44b85f24125f25c6b74c921499a1b31b8f70", size = 421225 },
-    { url = "https://files.pythonhosted.org/packages/f6/f1/940bc959aaef9eede8bb1b1127833b0929c6ffa9268ec0f6cb19877e2027/optree-0.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1959cfbc38c228c8195354967cda64887b96219924b7b3759e5ee355582c1ec", size = 408819 },
-    { url = "https://files.pythonhosted.org/packages/dd/12/24d4a417fd325ec06cfbce52716ac4f816ef696653b868960ac2ccb28436/optree-0.17.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfeea4aa0fd354d27922aba63ff9d86e4e126c6bf89cfb02849e68515519f1a5", size = 368513 },
-    { url = "https://files.pythonhosted.org/packages/30/e2/34e392209933e2c582c67594a7a6b4851bca4015c83b51c7508384b616b4/optree-0.17.0-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6b2ff8999a9b84d00f23a032b6b3f13678894432a335d024e0670b9880f238ca", size = 430378 },
-    { url = "https://files.pythonhosted.org/packages/5f/16/0a0d6139022e9a53ecb1212fb6fbc5b60eff824371071ef5f5fa481d8167/optree-0.17.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ea8bef525432b38a84e7448348da1a2dc308375bce79c77675cc50a501305851", size = 423294 },
-    { url = "https://files.pythonhosted.org/packages/ef/60/2e083dabb6aff6d939d8aab16ba3dbe6eee9429597a13f3fca57b33cdcde/optree-0.17.0-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f95b81aa67538d38316b184a6ff39a3725ee5c8555fba21dcb692f8d7c39302e", size = 424633 },
-    { url = "https://files.pythonhosted.org/packages/af/fd/0e4229b5fa3fd9d3c779a606c0f358ffbdfee717f49b3477facd04de2cec/optree-0.17.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e808a1125169ae90de623456ef2423eb84a8578a74f03fe48b06b8561c2cc31d", size = 414866 },
-    { url = "https://files.pythonhosted.org/packages/39/df/b8882f5519c85af146de3a79a08066a56fe634b23052c593fcedc70bfcd7/optree-0.17.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e45a13b35873712e095fe0f7fd6e9c4f98f3bd5af6f5dc33c17b80357bc97fc", size = 386945 },
-    { url = "https://files.pythonhosted.org/packages/ca/d7/91f4efb509bda601a1591465c4a5bd55320e4bafe06b294bf80754127b0e/optree-0.17.0-cp313-cp313t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:bfaf04d833dc53e5cfccff3b564e934a49086158472e31d84df31fce6d4f7b1c", size = 444177 },
-    { url = "https://files.pythonhosted.org/packages/84/17/a4833006e925c6ed5c45ceb02e65c9e9a260e70da6523858fcf628481847/optree-0.17.0-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b4c1d030ac1c881803f5c8e23d241159ae403fd00cdf57625328f282fc671ebd", size = 439198 },
-    { url = "https://files.pythonhosted.org/packages/ef/d1/c08fc60f6dfcb1b86ca1fdc0add08a98412a1596cd45830acbdc309f2cdb/optree-0.17.0-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd7738709970acab5d963896192b63b2718be93bb6c0bcea91895ea157fa2b13", size = 439391 },
-    { url = "https://files.pythonhosted.org/packages/05/8f/461e10201003e6ad6bff3c594a29a7e044454aba68c5f795f4c8386ce47c/optree-0.17.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1644bc24b6e93cafccfdeee44157c3d4ae9bb0af3e861300602d716699865b1a", size = 426555 },
-    { url = "https://files.pythonhosted.org/packages/3c/21/6480d23b52b2e23b976fe254b9fbdc4b514e90a349b1ee73565b185c69f1/optree-0.17.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd21e0a89806cc3b86aaa578a73897d56085038fe432043534a23b2e559d7691", size = 369929 },
-    { url = "https://files.pythonhosted.org/packages/b3/29/69bb26473ff862a1792f5568c977e7a2580e08afe0fdcd7a7b3e1e4d6933/optree-0.17.0-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:9211c61285b8b3e42fd0e803cebd6e2b0987d8b2edffe45b42923debca09a9df", size = 430381 },
-    { url = "https://files.pythonhosted.org/packages/c8/8b/2c0a38c0d0c2396d698b97216cd6814d6754d11997b6ac66c57d87d71bae/optree-0.17.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:87938255749a45979c4e331627cb33d81aa08b0a09d024368b3e25ff67f0e9f2", size = 424461 },
-    { url = "https://files.pythonhosted.org/packages/a7/77/08fda3f97621190d50762225ee8bad87463a8b3a55fba451a999971ff130/optree-0.17.0-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3432858145fd1955a3be12207507466ac40a6911f428bf5d2d6c7f67486530a2", size = 427234 },
-    { url = "https://files.pythonhosted.org/packages/ea/b5/b4f19952c36d6448c85a6ef6be5f916dd13548de2b684ab123f04b450850/optree-0.17.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5afe3e9e2f6da0a0a5c0892f32f675eb88965036b061aa555b74e6c412a05e17", size = 413863 },
-    { url = "https://files.pythonhosted.org/packages/88/42/6003f13e66cfbe7f0011bf8509da2479aba93068cdb9d79bf46010255089/optree-0.17.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5739c03a3362be42cb7649e82457c90aa818aa3e82af9681d3100c3346f4a90f", size = 386975 },
-    { url = "https://files.pythonhosted.org/packages/d0/53/621642abd76eda5a941b47adc98be81f0052683160be776499d11b4af83d/optree-0.17.0-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:ee07b59a08bd45aedd5252241a98841f1a5082a7b9b73df2dae6a433aa2a91d8", size = 444173 },
-    { url = "https://files.pythonhosted.org/packages/5b/d3/8819a2d5105a240d6793d11a61d597db91756ce84da5cee08808c6b8f61f/optree-0.17.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:875c017890a4b5d566af5593cab67fe3c4845544942af57e6bb9dea17e060297", size = 439080 },
-    { url = "https://files.pythonhosted.org/packages/c6/ef/9dbd34dfd1ad89feb239ca9925897a14ac94f190379a3bd991afdfd94186/optree-0.17.0-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ffa5686191139f763e13445a169765c83517164bc28e60dbedb19bed2b2655f1", size = 439422 },
-    { url = "https://files.pythonhosted.org/packages/86/ca/a7a7549af2951925a692df508902ed2a6a94a51bc846806d2281b1029ef9/optree-0.17.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:575cf48cc2190acb565bd2b26b6f9b15c4e3b60183e86031215badc9d5441345", size = 426579 },
-    { url = "https://files.pythonhosted.org/packages/ed/d7/3036d15c028c447b1bd65dcf8f66cfd775bfa4e52daa74b82fb1d3c88faf/optree-0.17.0-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adde1427e0982cfc5f56939c26b4ebbd833091a176734c79fb95c78bdf833dff", size = 350952 },
-    { url = "https://files.pythonhosted.org/packages/71/45/e710024ef77324e745de48efd64f6270d8c209f14107a48ffef4049ac57a/optree-0.17.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a80b7e5de5dd09b9c8b62d501e29a3850b047565c336c9d004b07ee1c01f4ae1", size = 389568 },
-    { url = "https://files.pythonhosted.org/packages/69/c4/94a187ed3ca71194b9da6a276790e1703c7544c8f695ac915214ae8ce934/optree-0.17.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f87f6f39015fc82d7adeee19900d246b89911319726e93cb2dbd4d1a809899bd", size = 363728 },
-    { url = "https://files.pythonhosted.org/packages/cd/99/23b7a484da8dfb814107b20ef2c93ef27c04f36aeb83bd976964a5b69e06/optree-0.17.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58b0a83a967d2ef0f343db7182f0ad074eb1166bcaea909ae33909462013f151", size = 404649 },
+    { name = "typing-extensions", marker = "python_full_version >= '3.10'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/56/c7/0853e0c59b135dff770615d2713b547b6b3b5cde7c10995b4a5825244612/optree-0.17.0.tar.gz", hash = "sha256:5335a5ec44479920620d72324c66563bd705ab2a698605dd4b6ee67dbcad7ecd", size = 163111, upload-time = "2025-07-25T11:26:11.586Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/91/f9/6ca076fd4c6f16be031afdc711a2676c1ff15bd1717ee2e699179b1a29bc/optree-0.17.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98990201f352dba253af1a995c1453818db5f08de4cae7355d85aa6023676a52", size = 350398, upload-time = "2025-07-25T11:24:26.672Z" },
+    { url = "https://files.pythonhosted.org/packages/95/4c/81344cbdcf8ea8525a21c9d65892d7529010ee2146c53423b2e9a84441ba/optree-0.17.0-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:e1a40adf6bb78a6a4b4f480879de2cb6b57d46d680a4d9834aa824f41e69c0d9", size = 404834, upload-time = "2025-07-25T11:24:28.988Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/c4/ac1880372a89f5c21514a7965dfa23b1afb2ad683fb9804d366727de9ecf/optree-0.17.0-cp310-cp310-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:78a113436a0a440f900b2799584f3cc2b2eea1b245d81c3583af42ac003e333c", size = 402116, upload-time = "2025-07-25T11:24:30.396Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/72/ad6be4d6a03805cf3921b492494cb3371ca28060d5ad19d5a36e10c4d67d/optree-0.17.0-cp310-cp310-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e45c16018f4283f028cf839b707b7ac734e8056a31b7198a1577161fcbe146d", size = 398491, upload-time = "2025-07-25T11:24:31.725Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/c1/6827fb504351f9a3935699b0eb31c8a6af59d775ee78289a25e0ba54f732/optree-0.17.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b698613d821d80cc216a2444ebc3145c8bf671b55a2223058a6574c1483a65f6", size = 387957, upload-time = "2025-07-25T11:24:32.759Z" },
+    { url = "https://files.pythonhosted.org/packages/73/5c/13a2a864b0c0b39c3c193be534a195a3ab2463c7d0443d4a76e749e3ff83/optree-0.17.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3080c564c9760711aa72d1b4d700ce1417f99ad087136f415c4eb8221169e2a3", size = 362797, upload-time = "2025-07-25T11:24:39.509Z" },
+    { url = "https://files.pythonhosted.org/packages/da/f5/ff7dcb5a0108ee89c2be09aed2ebd26a7e1333d8122031aa9d9322b24ee6/optree-0.17.0-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:834a8fb358b608240b3a38706a09b43974675624485fad64c8ee641dae2eb57d", size = 419450, upload-time = "2025-07-25T11:24:40.555Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/e6/48a97aefd18770b55e5ed456d8183891f325cdb6d90592e5f072ed6951f8/optree-0.17.0-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1a2bd263e6b5621d000d0f94de1f245414fd5dbce365a24b7b89b1ed0ef56cf9", size = 417557, upload-time = "2025-07-25T11:24:42.396Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/b1/4e280edab8a86be47ec1f9bd9ed4b685d2e15f0950ae62b613b26d12a1da/optree-0.17.0-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9b37daca4ad89339b1f5320cc61ac600dcf976adbb060769d36d5542d6ebfedf", size = 414174, upload-time = "2025-07-25T11:24:43.51Z" },
+    { url = "https://files.pythonhosted.org/packages/db/3b/49a9a1986215dd342525974deeb17c260a83fee8fad147276fd710ac8718/optree-0.17.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a146a6917f3e28cfdc268ff1770aa696c346482dd3da681c3ff92153d94450ea", size = 402000, upload-time = "2025-07-25T11:24:44.819Z" },
+    { url = "https://files.pythonhosted.org/packages/41/90/e12dea2cb5d8a5e17bbe3011ed4e972b89c027272a816db4897589751cad/optree-0.17.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e13ae51a63d69db445f269a3a4fd1d6edb064a705188d007ea47c9f034788fc5", size = 365869, upload-time = "2025-07-25T11:24:51.807Z" },
+    { url = "https://files.pythonhosted.org/packages/76/ee/21af214663960a479863cd6c03d7a0abc8123ea22a6ea34689c2eed88ccd/optree-0.17.0-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:5958f58423cc7870cb011c8c8f92687397380886e8c9d33adac752147e7bbc3f", size = 424465, upload-time = "2025-07-25T11:24:53.124Z" },
+    { url = "https://files.pythonhosted.org/packages/54/a3/64b184a79373753f4f46a5cd301ea581f71d6dc1a5c103bd2394f0925d40/optree-0.17.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:970ae4e47727b4c5526fc583b87d29190e576f6a2b6c19e8671589b73d256250", size = 420686, upload-time = "2025-07-25T11:24:54.212Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/6d/b6051b0b1ef9a49df96a66e9e62fc02620d2115d1ba659888c94e67fcfc9/optree-0.17.0-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54177fd3e6e05c08b66329e26d7d44b85f24125f25c6b74c921499a1b31b8f70", size = 421225, upload-time = "2025-07-25T11:24:55.213Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/f1/940bc959aaef9eede8bb1b1127833b0929c6ffa9268ec0f6cb19877e2027/optree-0.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1959cfbc38c228c8195354967cda64887b96219924b7b3759e5ee355582c1ec", size = 408819, upload-time = "2025-07-25T11:24:56.315Z" },
+    { url = "https://files.pythonhosted.org/packages/21/04/9706d11b880186e9e9d66d7c21ce249b2ce0212645137cc13fdd18247c26/optree-0.17.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5995a3efce4b00a14049268a81ab0379656a41ddf3c3761e3b88937fca44d48", size = 348177, upload-time = "2025-07-25T11:25:00.999Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/4b/0415c18816818ac871c9f3d5c7c5f4ceb83baff03ed511c9c94591ace4bc/optree-0.17.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d06e8143d16fe6c0708f3cc2807b5b65f815d60ee2b52f3d79e4022c95563482", size = 354389, upload-time = "2025-07-25T11:25:02.337Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/12/24d4a417fd325ec06cfbce52716ac4f816ef696653b868960ac2ccb28436/optree-0.17.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfeea4aa0fd354d27922aba63ff9d86e4e126c6bf89cfb02849e68515519f1a5", size = 368513, upload-time = "2025-07-25T11:25:05.548Z" },
+    { url = "https://files.pythonhosted.org/packages/30/e2/34e392209933e2c582c67594a7a6b4851bca4015c83b51c7508384b616b4/optree-0.17.0-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6b2ff8999a9b84d00f23a032b6b3f13678894432a335d024e0670b9880f238ca", size = 430378, upload-time = "2025-07-25T11:25:06.918Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/16/0a0d6139022e9a53ecb1212fb6fbc5b60eff824371071ef5f5fa481d8167/optree-0.17.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ea8bef525432b38a84e7448348da1a2dc308375bce79c77675cc50a501305851", size = 423294, upload-time = "2025-07-25T11:25:08.043Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/60/2e083dabb6aff6d939d8aab16ba3dbe6eee9429597a13f3fca57b33cdcde/optree-0.17.0-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f95b81aa67538d38316b184a6ff39a3725ee5c8555fba21dcb692f8d7c39302e", size = 424633, upload-time = "2025-07-25T11:25:09.141Z" },
+    { url = "https://files.pythonhosted.org/packages/af/fd/0e4229b5fa3fd9d3c779a606c0f358ffbdfee717f49b3477facd04de2cec/optree-0.17.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e808a1125169ae90de623456ef2423eb84a8578a74f03fe48b06b8561c2cc31d", size = 414866, upload-time = "2025-07-25T11:25:10.214Z" },
+    { url = "https://files.pythonhosted.org/packages/39/df/b8882f5519c85af146de3a79a08066a56fe634b23052c593fcedc70bfcd7/optree-0.17.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e45a13b35873712e095fe0f7fd6e9c4f98f3bd5af6f5dc33c17b80357bc97fc", size = 386945, upload-time = "2025-07-25T11:25:17.728Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/d7/91f4efb509bda601a1591465c4a5bd55320e4bafe06b294bf80754127b0e/optree-0.17.0-cp313-cp313t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:bfaf04d833dc53e5cfccff3b564e934a49086158472e31d84df31fce6d4f7b1c", size = 444177, upload-time = "2025-07-25T11:25:18.749Z" },
+    { url = "https://files.pythonhosted.org/packages/84/17/a4833006e925c6ed5c45ceb02e65c9e9a260e70da6523858fcf628481847/optree-0.17.0-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b4c1d030ac1c881803f5c8e23d241159ae403fd00cdf57625328f282fc671ebd", size = 439198, upload-time = "2025-07-25T11:25:19.865Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/d1/c08fc60f6dfcb1b86ca1fdc0add08a98412a1596cd45830acbdc309f2cdb/optree-0.17.0-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd7738709970acab5d963896192b63b2718be93bb6c0bcea91895ea157fa2b13", size = 439391, upload-time = "2025-07-25T11:25:20.942Z" },
+    { url = "https://files.pythonhosted.org/packages/05/8f/461e10201003e6ad6bff3c594a29a7e044454aba68c5f795f4c8386ce47c/optree-0.17.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1644bc24b6e93cafccfdeee44157c3d4ae9bb0af3e861300602d716699865b1a", size = 426555, upload-time = "2025-07-25T11:25:21.968Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/21/6480d23b52b2e23b976fe254b9fbdc4b514e90a349b1ee73565b185c69f1/optree-0.17.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd21e0a89806cc3b86aaa578a73897d56085038fe432043534a23b2e559d7691", size = 369929, upload-time = "2025-07-25T11:25:28.897Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/29/69bb26473ff862a1792f5568c977e7a2580e08afe0fdcd7a7b3e1e4d6933/optree-0.17.0-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:9211c61285b8b3e42fd0e803cebd6e2b0987d8b2edffe45b42923debca09a9df", size = 430381, upload-time = "2025-07-25T11:25:29.984Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/8b/2c0a38c0d0c2396d698b97216cd6814d6754d11997b6ac66c57d87d71bae/optree-0.17.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:87938255749a45979c4e331627cb33d81aa08b0a09d024368b3e25ff67f0e9f2", size = 424461, upload-time = "2025-07-25T11:25:31.116Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/77/08fda3f97621190d50762225ee8bad87463a8b3a55fba451a999971ff130/optree-0.17.0-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3432858145fd1955a3be12207507466ac40a6911f428bf5d2d6c7f67486530a2", size = 427234, upload-time = "2025-07-25T11:25:32.289Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/b5/b4f19952c36d6448c85a6ef6be5f916dd13548de2b684ab123f04b450850/optree-0.17.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5afe3e9e2f6da0a0a5c0892f32f675eb88965036b061aa555b74e6c412a05e17", size = 413863, upload-time = "2025-07-25T11:25:33.379Z" },
+    { url = "https://files.pythonhosted.org/packages/88/42/6003f13e66cfbe7f0011bf8509da2479aba93068cdb9d79bf46010255089/optree-0.17.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5739c03a3362be42cb7649e82457c90aa818aa3e82af9681d3100c3346f4a90f", size = 386975, upload-time = "2025-07-25T11:25:40.376Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/53/621642abd76eda5a941b47adc98be81f0052683160be776499d11b4af83d/optree-0.17.0-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:ee07b59a08bd45aedd5252241a98841f1a5082a7b9b73df2dae6a433aa2a91d8", size = 444173, upload-time = "2025-07-25T11:25:41.474Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/d3/8819a2d5105a240d6793d11a61d597db91756ce84da5cee08808c6b8f61f/optree-0.17.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:875c017890a4b5d566af5593cab67fe3c4845544942af57e6bb9dea17e060297", size = 439080, upload-time = "2025-07-25T11:25:42.605Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/ef/9dbd34dfd1ad89feb239ca9925897a14ac94f190379a3bd991afdfd94186/optree-0.17.0-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ffa5686191139f763e13445a169765c83517164bc28e60dbedb19bed2b2655f1", size = 439422, upload-time = "2025-07-25T11:25:43.672Z" },
+    { url = "https://files.pythonhosted.org/packages/86/ca/a7a7549af2951925a692df508902ed2a6a94a51bc846806d2281b1029ef9/optree-0.17.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:575cf48cc2190acb565bd2b26b6f9b15c4e3b60183e86031215badc9d5441345", size = 426579, upload-time = "2025-07-25T11:25:44.765Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/29/3bb53de2de3b36a51e46b6d9ada7ee1a3a312ac461cd54292a023adc807c/optree-0.17.0-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:537498cf7bf7a4fe71f7ffd815e72b8672aea0fac82e1513f6b6e35e8569f5aa", size = 350302, upload-time = "2025-07-25T11:25:52.016Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/3b/d17a31447ed7ef6f10bd0caf40742b016fcdeaa3abb7568307b04a0f50cf/optree-0.17.0-cp39-cp39-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:3b3bb2326b550ddb048e3454fad40183b7fed74dda4351b016d20362809180af", size = 405358, upload-time = "2025-07-25T11:25:53.085Z" },
+    { url = "https://files.pythonhosted.org/packages/db/f3/b9f0a8c98fd0c7f53fa9d9a46d75bb1182aeecd7ecde6f353d3e69ec9618/optree-0.17.0-cp39-cp39-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c0d3d702044e5acbec2cf8349789f6b096057bd00dc8e1e1c97b990347279fda", size = 402694, upload-time = "2025-07-25T11:25:54.537Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/dd/0d9d7426fd6b5d90ad40e4d93717a955d4257d06574dfe7a1da0d24cb06c/optree-0.17.0-cp39-cp39-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9155e82717be1dda1f3c1244e9cb5b3733d5dd3ba47702730c7816be083a5cb", size = 398857, upload-time = "2025-07-25T11:25:55.921Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/57/dacec3f8c70f4685bb07fce19cf3361037fde2b596f6f7228e1a4b39677b/optree-0.17.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8e825501f55360e8381718623b094579dedc485e57010e01593d72a43b43e68", size = 387849, upload-time = "2025-07-25T11:25:57.046Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/d7/3036d15c028c447b1bd65dcf8f66cfd775bfa4e52daa74b82fb1d3c88faf/optree-0.17.0-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adde1427e0982cfc5f56939c26b4ebbd833091a176734c79fb95c78bdf833dff", size = 350952, upload-time = "2025-07-25T11:26:02.692Z" },
+    { url = "https://files.pythonhosted.org/packages/71/45/e710024ef77324e745de48efd64f6270d8c209f14107a48ffef4049ac57a/optree-0.17.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a80b7e5de5dd09b9c8b62d501e29a3850b047565c336c9d004b07ee1c01f4ae1", size = 389568, upload-time = "2025-07-25T11:26:04.094Z" },
+    { url = "https://files.pythonhosted.org/packages/69/c4/94a187ed3ca71194b9da6a276790e1703c7544c8f695ac915214ae8ce934/optree-0.17.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f87f6f39015fc82d7adeee19900d246b89911319726e93cb2dbd4d1a809899bd", size = 363728, upload-time = "2025-07-25T11:26:07.959Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/99/23b7a484da8dfb814107b20ef2c93ef27c04f36aeb83bd976964a5b69e06/optree-0.17.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58b0a83a967d2ef0f343db7182f0ad074eb1166bcaea909ae33909462013f151", size = 404649, upload-time = "2025-07-25T11:26:09.463Z" },
 ]
 
 [[package]]
 name = "packaging"
 version = "25.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727 }
+sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469 },
+    { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" },
 ]
 
 [[package]]
@@ -1575,172 +1916,191 @@ name = "pandas"
 version = "2.3.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "python-dateutil" },
     { name = "pytz" },
     { name = "tzdata" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/3d/f7/f425a00df4fcc22b292c6895c6831c0c8ae1d9fac1e024d16f98a9ce8749/pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c", size = 11555763 },
-    { url = "https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a", size = 10801217 },
-    { url = "https://files.pythonhosted.org/packages/1d/03/3fc4a529a7710f890a239cc496fc6d50ad4a0995657dccc1d64695adb9f4/pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1", size = 12148791 },
-    { url = "https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838", size = 12769373 },
-    { url = "https://files.pythonhosted.org/packages/df/91/82cc5169b6b25440a7fc0ef3a694582418d875c8e3ebf796a6d6470aa578/pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250", size = 13200444 },
-    { url = "https://files.pythonhosted.org/packages/10/ae/89b3283800ab58f7af2952704078555fa60c807fff764395bb57ea0b0dbd/pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4", size = 13858459 },
-    { url = "https://files.pythonhosted.org/packages/85/72/530900610650f54a35a19476eca5104f38555afccda1aa11a92ee14cb21d/pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826", size = 11346086 },
-    { url = "https://files.pythonhosted.org/packages/c1/fa/7ac648108144a095b4fb6aa3de1954689f7af60a14cf25583f4960ecb878/pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523", size = 11578790 },
-    { url = "https://files.pythonhosted.org/packages/9b/35/74442388c6cf008882d4d4bdfc4109be87e9b8b7ccd097ad1e7f006e2e95/pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45", size = 10833831 },
-    { url = "https://files.pythonhosted.org/packages/fe/e4/de154cbfeee13383ad58d23017da99390b91d73f8c11856f2095e813201b/pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66", size = 12199267 },
-    { url = "https://files.pythonhosted.org/packages/bf/c9/63f8d545568d9ab91476b1818b4741f521646cbdd151c6efebf40d6de6f7/pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b", size = 12789281 },
-    { url = "https://files.pythonhosted.org/packages/f2/00/a5ac8c7a0e67fd1a6059e40aa08fa1c52cc00709077d2300e210c3ce0322/pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791", size = 13240453 },
-    { url = "https://files.pythonhosted.org/packages/27/4d/5c23a5bc7bd209231618dd9e606ce076272c9bc4f12023a70e03a86b4067/pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151", size = 13890361 },
-    { url = "https://files.pythonhosted.org/packages/8e/59/712db1d7040520de7a4965df15b774348980e6df45c129b8c64d0dbe74ef/pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c", size = 11348702 },
-    { url = "https://files.pythonhosted.org/packages/9c/fb/231d89e8637c808b997d172b18e9d4a4bc7bf31296196c260526055d1ea0/pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53", size = 11597846 },
-    { url = "https://files.pythonhosted.org/packages/5c/bd/bf8064d9cfa214294356c2d6702b716d3cf3bb24be59287a6a21e24cae6b/pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35", size = 10729618 },
-    { url = "https://files.pythonhosted.org/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908", size = 11737212 },
-    { url = "https://files.pythonhosted.org/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89", size = 12362693 },
-    { url = "https://files.pythonhosted.org/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98", size = 12771002 },
-    { url = "https://files.pythonhosted.org/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084", size = 13450971 },
-    { url = "https://files.pythonhosted.org/packages/86/41/585a168330ff063014880a80d744219dbf1dd7a1c706e75ab3425a987384/pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b", size = 10992722 },
-    { url = "https://files.pythonhosted.org/packages/cd/4b/18b035ee18f97c1040d94debd8f2e737000ad70ccc8f5513f4eefad75f4b/pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713", size = 11544671 },
-    { url = "https://files.pythonhosted.org/packages/31/94/72fac03573102779920099bcac1c3b05975c2cb5f01eac609faf34bed1ca/pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8", size = 10680807 },
-    { url = "https://files.pythonhosted.org/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d", size = 11709872 },
-    { url = "https://files.pythonhosted.org/packages/15/07/284f757f63f8a8d69ed4472bfd85122bd086e637bf4ed09de572d575a693/pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac", size = 12306371 },
-    { url = "https://files.pythonhosted.org/packages/33/81/a3afc88fca4aa925804a27d2676d22dcd2031c2ebe08aabd0ae55b9ff282/pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c", size = 12765333 },
-    { url = "https://files.pythonhosted.org/packages/8d/0f/b4d4ae743a83742f1153464cf1a8ecfafc3ac59722a0b5c8602310cb7158/pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493", size = 13418120 },
-    { url = "https://files.pythonhosted.org/packages/4f/c7/e54682c96a895d0c808453269e0b5928a07a127a15704fedb643e9b0a4c8/pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee", size = 10993991 },
-    { url = "https://files.pythonhosted.org/packages/f9/ca/3f8d4f49740799189e1395812f3bf23b5e8fc7c190827d55a610da72ce55/pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5", size = 12048227 },
-    { url = "https://files.pythonhosted.org/packages/0e/5a/f43efec3e8c0cc92c4663ccad372dbdff72b60bdb56b2749f04aa1d07d7e/pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21", size = 11411056 },
-    { url = "https://files.pythonhosted.org/packages/46/b1/85331edfc591208c9d1a63a06baa67b21d332e63b7a591a5ba42a10bb507/pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78", size = 11645189 },
-    { url = "https://files.pythonhosted.org/packages/44/23/78d645adc35d94d1ac4f2a3c4112ab6f5b8999f4898b8cdf01252f8df4a9/pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110", size = 12121912 },
-    { url = "https://files.pythonhosted.org/packages/53/da/d10013df5e6aaef6b425aa0c32e1fc1f3e431e4bcabd420517dceadce354/pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86", size = 12712160 },
-    { url = "https://files.pythonhosted.org/packages/bd/17/e756653095a083d8a37cbd816cb87148debcfcd920129b25f99dd8d04271/pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc", size = 13199233 },
-    { url = "https://files.pythonhosted.org/packages/04/fd/74903979833db8390b73b3a8a7d30d146d710bd32703724dd9083950386f/pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0", size = 11540635 },
-    { url = "https://files.pythonhosted.org/packages/21/00/266d6b357ad5e6d3ad55093a7e8efc7dd245f5a842b584db9f30b0f0a287/pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593", size = 10759079 },
-    { url = "https://files.pythonhosted.org/packages/ca/05/d01ef80a7a3a12b2f8bbf16daba1e17c98a2f039cbc8e2f77a2c5a63d382/pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c", size = 11814049 },
-    { url = "https://files.pythonhosted.org/packages/15/b2/0e62f78c0c5ba7e3d2c5945a82456f4fac76c480940f805e0b97fcbc2f65/pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b", size = 12332638 },
-    { url = "https://files.pythonhosted.org/packages/c5/33/dd70400631b62b9b29c3c93d2feee1d0964dc2bae2e5ad7a6c73a7f25325/pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6", size = 12886834 },
-    { url = "https://files.pythonhosted.org/packages/d3/18/b5d48f55821228d0d2692b34fd5034bb185e854bdb592e9c640f6290e012/pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3", size = 13409925 },
-    { url = "https://files.pythonhosted.org/packages/a6/3d/124ac75fcd0ecc09b8fdccb0246ef65e35b012030defb0e0eba2cbbbe948/pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5", size = 11109071 },
-    { url = "https://files.pythonhosted.org/packages/89/9c/0e21c895c38a157e0faa1fb64587a9226d6dd46452cac4532d80c3c4a244/pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec", size = 12048504 },
-    { url = "https://files.pythonhosted.org/packages/d7/82/b69a1c95df796858777b68fbe6a81d37443a33319761d7c652ce77797475/pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7", size = 11410702 },
-    { url = "https://files.pythonhosted.org/packages/f9/88/702bde3ba0a94b8c73a0181e05144b10f13f29ebfc2150c3a79062a8195d/pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450", size = 11634535 },
-    { url = "https://files.pythonhosted.org/packages/a4/1e/1bac1a839d12e6a82ec6cb40cda2edde64a2013a66963293696bbf31fbbb/pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5", size = 12121582 },
-    { url = "https://files.pythonhosted.org/packages/44/91/483de934193e12a3b1d6ae7c8645d083ff88dec75f46e827562f1e4b4da6/pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788", size = 12699963 },
-    { url = "https://files.pythonhosted.org/packages/70/44/5191d2e4026f86a2a109053e194d3ba7a31a2d10a9c2348368c63ed4e85a/pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87", size = 13202175 },
+sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3d/f7/f425a00df4fcc22b292c6895c6831c0c8ae1d9fac1e024d16f98a9ce8749/pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c", size = 11555763, upload-time = "2025-09-29T23:16:53.287Z" },
+    { url = "https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a", size = 10801217, upload-time = "2025-09-29T23:17:04.522Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/03/3fc4a529a7710f890a239cc496fc6d50ad4a0995657dccc1d64695adb9f4/pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1", size = 12148791, upload-time = "2025-09-29T23:17:18.444Z" },
+    { url = "https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838", size = 12769373, upload-time = "2025-09-29T23:17:35.846Z" },
+    { url = "https://files.pythonhosted.org/packages/df/91/82cc5169b6b25440a7fc0ef3a694582418d875c8e3ebf796a6d6470aa578/pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250", size = 13200444, upload-time = "2025-09-29T23:17:49.341Z" },
+    { url = "https://files.pythonhosted.org/packages/10/ae/89b3283800ab58f7af2952704078555fa60c807fff764395bb57ea0b0dbd/pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4", size = 13858459, upload-time = "2025-09-29T23:18:03.722Z" },
+    { url = "https://files.pythonhosted.org/packages/85/72/530900610650f54a35a19476eca5104f38555afccda1aa11a92ee14cb21d/pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826", size = 11346086, upload-time = "2025-09-29T23:18:18.505Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/fa/7ac648108144a095b4fb6aa3de1954689f7af60a14cf25583f4960ecb878/pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523", size = 11578790, upload-time = "2025-09-29T23:18:30.065Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/35/74442388c6cf008882d4d4bdfc4109be87e9b8b7ccd097ad1e7f006e2e95/pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45", size = 10833831, upload-time = "2025-09-29T23:38:56.071Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/e4/de154cbfeee13383ad58d23017da99390b91d73f8c11856f2095e813201b/pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66", size = 12199267, upload-time = "2025-09-29T23:18:41.627Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/c9/63f8d545568d9ab91476b1818b4741f521646cbdd151c6efebf40d6de6f7/pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b", size = 12789281, upload-time = "2025-09-29T23:18:56.834Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/00/a5ac8c7a0e67fd1a6059e40aa08fa1c52cc00709077d2300e210c3ce0322/pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791", size = 13240453, upload-time = "2025-09-29T23:19:09.247Z" },
+    { url = "https://files.pythonhosted.org/packages/27/4d/5c23a5bc7bd209231618dd9e606ce076272c9bc4f12023a70e03a86b4067/pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151", size = 13890361, upload-time = "2025-09-29T23:19:25.342Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/59/712db1d7040520de7a4965df15b774348980e6df45c129b8c64d0dbe74ef/pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c", size = 11348702, upload-time = "2025-09-29T23:19:38.296Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/fb/231d89e8637c808b997d172b18e9d4a4bc7bf31296196c260526055d1ea0/pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53", size = 11597846, upload-time = "2025-09-29T23:19:48.856Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/bd/bf8064d9cfa214294356c2d6702b716d3cf3bb24be59287a6a21e24cae6b/pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35", size = 10729618, upload-time = "2025-09-29T23:39:08.659Z" },
+    { url = "https://files.pythonhosted.org/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908", size = 11737212, upload-time = "2025-09-29T23:19:59.765Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89", size = 12362693, upload-time = "2025-09-29T23:20:14.098Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98", size = 12771002, upload-time = "2025-09-29T23:20:26.76Z" },
+    { url = "https://files.pythonhosted.org/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084", size = 13450971, upload-time = "2025-09-29T23:20:41.344Z" },
+    { url = "https://files.pythonhosted.org/packages/86/41/585a168330ff063014880a80d744219dbf1dd7a1c706e75ab3425a987384/pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b", size = 10992722, upload-time = "2025-09-29T23:20:54.139Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/4b/18b035ee18f97c1040d94debd8f2e737000ad70ccc8f5513f4eefad75f4b/pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713", size = 11544671, upload-time = "2025-09-29T23:21:05.024Z" },
+    { url = "https://files.pythonhosted.org/packages/31/94/72fac03573102779920099bcac1c3b05975c2cb5f01eac609faf34bed1ca/pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8", size = 10680807, upload-time = "2025-09-29T23:21:15.979Z" },
+    { url = "https://files.pythonhosted.org/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d", size = 11709872, upload-time = "2025-09-29T23:21:27.165Z" },
+    { url = "https://files.pythonhosted.org/packages/15/07/284f757f63f8a8d69ed4472bfd85122bd086e637bf4ed09de572d575a693/pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac", size = 12306371, upload-time = "2025-09-29T23:21:40.532Z" },
+    { url = "https://files.pythonhosted.org/packages/33/81/a3afc88fca4aa925804a27d2676d22dcd2031c2ebe08aabd0ae55b9ff282/pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c", size = 12765333, upload-time = "2025-09-29T23:21:55.77Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/0f/b4d4ae743a83742f1153464cf1a8ecfafc3ac59722a0b5c8602310cb7158/pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493", size = 13418120, upload-time = "2025-09-29T23:22:10.109Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/c7/e54682c96a895d0c808453269e0b5928a07a127a15704fedb643e9b0a4c8/pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee", size = 10993991, upload-time = "2025-09-29T23:25:04.889Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/ca/3f8d4f49740799189e1395812f3bf23b5e8fc7c190827d55a610da72ce55/pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5", size = 12048227, upload-time = "2025-09-29T23:22:24.343Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/5a/f43efec3e8c0cc92c4663ccad372dbdff72b60bdb56b2749f04aa1d07d7e/pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21", size = 11411056, upload-time = "2025-09-29T23:22:37.762Z" },
+    { url = "https://files.pythonhosted.org/packages/46/b1/85331edfc591208c9d1a63a06baa67b21d332e63b7a591a5ba42a10bb507/pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78", size = 11645189, upload-time = "2025-09-29T23:22:51.688Z" },
+    { url = "https://files.pythonhosted.org/packages/44/23/78d645adc35d94d1ac4f2a3c4112ab6f5b8999f4898b8cdf01252f8df4a9/pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110", size = 12121912, upload-time = "2025-09-29T23:23:05.042Z" },
+    { url = "https://files.pythonhosted.org/packages/53/da/d10013df5e6aaef6b425aa0c32e1fc1f3e431e4bcabd420517dceadce354/pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86", size = 12712160, upload-time = "2025-09-29T23:23:28.57Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/17/e756653095a083d8a37cbd816cb87148debcfcd920129b25f99dd8d04271/pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc", size = 13199233, upload-time = "2025-09-29T23:24:24.876Z" },
+    { url = "https://files.pythonhosted.org/packages/04/fd/74903979833db8390b73b3a8a7d30d146d710bd32703724dd9083950386f/pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0", size = 11540635, upload-time = "2025-09-29T23:25:52.486Z" },
+    { url = "https://files.pythonhosted.org/packages/21/00/266d6b357ad5e6d3ad55093a7e8efc7dd245f5a842b584db9f30b0f0a287/pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593", size = 10759079, upload-time = "2025-09-29T23:26:33.204Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/05/d01ef80a7a3a12b2f8bbf16daba1e17c98a2f039cbc8e2f77a2c5a63d382/pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c", size = 11814049, upload-time = "2025-09-29T23:27:15.384Z" },
+    { url = "https://files.pythonhosted.org/packages/15/b2/0e62f78c0c5ba7e3d2c5945a82456f4fac76c480940f805e0b97fcbc2f65/pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b", size = 12332638, upload-time = "2025-09-29T23:27:51.625Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/33/dd70400631b62b9b29c3c93d2feee1d0964dc2bae2e5ad7a6c73a7f25325/pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6", size = 12886834, upload-time = "2025-09-29T23:28:21.289Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/18/b5d48f55821228d0d2692b34fd5034bb185e854bdb592e9c640f6290e012/pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3", size = 13409925, upload-time = "2025-09-29T23:28:58.261Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/3d/124ac75fcd0ecc09b8fdccb0246ef65e35b012030defb0e0eba2cbbbe948/pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5", size = 11109071, upload-time = "2025-09-29T23:32:27.484Z" },
+    { url = "https://files.pythonhosted.org/packages/89/9c/0e21c895c38a157e0faa1fb64587a9226d6dd46452cac4532d80c3c4a244/pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec", size = 12048504, upload-time = "2025-09-29T23:29:31.47Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/82/b69a1c95df796858777b68fbe6a81d37443a33319761d7c652ce77797475/pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7", size = 11410702, upload-time = "2025-09-29T23:29:54.591Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/88/702bde3ba0a94b8c73a0181e05144b10f13f29ebfc2150c3a79062a8195d/pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450", size = 11634535, upload-time = "2025-09-29T23:30:21.003Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/1e/1bac1a839d12e6a82ec6cb40cda2edde64a2013a66963293696bbf31fbbb/pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5", size = 12121582, upload-time = "2025-09-29T23:30:43.391Z" },
+    { url = "https://files.pythonhosted.org/packages/44/91/483de934193e12a3b1d6ae7c8645d083ff88dec75f46e827562f1e4b4da6/pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788", size = 12699963, upload-time = "2025-09-29T23:31:10.009Z" },
+    { url = "https://files.pythonhosted.org/packages/70/44/5191d2e4026f86a2a109053e194d3ba7a31a2d10a9c2348368c63ed4e85a/pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87", size = 13202175, upload-time = "2025-09-29T23:31:59.173Z" },
+    { url = "https://files.pythonhosted.org/packages/56/b4/52eeb530a99e2a4c55ffcd352772b599ed4473a0f892d127f4147cf0f88e/pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2", size = 11567720, upload-time = "2025-09-29T23:33:06.209Z" },
+    { url = "https://files.pythonhosted.org/packages/48/4a/2d8b67632a021bced649ba940455ed441ca854e57d6e7658a6024587b083/pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8", size = 10810302, upload-time = "2025-09-29T23:33:35.846Z" },
+    { url = "https://files.pythonhosted.org/packages/13/e6/d2465010ee0569a245c975dc6967b801887068bc893e908239b1f4b6c1ac/pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff", size = 12154874, upload-time = "2025-09-29T23:33:49.939Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/18/aae8c0aa69a386a3255940e9317f793808ea79d0a525a97a903366bb2569/pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29", size = 12790141, upload-time = "2025-09-29T23:34:05.655Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/26/617f98de789de00c2a444fbe6301bb19e66556ac78cff933d2c98f62f2b4/pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73", size = 13208697, upload-time = "2025-09-29T23:34:21.835Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/fb/25709afa4552042bd0e15717c75e9b4a2294c3dc4f7e6ea50f03c5136600/pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9", size = 13879233, upload-time = "2025-09-29T23:34:35.079Z" },
+    { url = "https://files.pythonhosted.org/packages/98/af/7be05277859a7bc399da8ba68b88c96b27b48740b6cf49688899c6eb4176/pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa", size = 11359119, upload-time = "2025-09-29T23:34:46.339Z" },
 ]
 
 [[package]]
 name = "pillow"
 version = "11.3.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/d0d6dea55cd152ce3d6767bb38a8fc10e33796ba4ba210cbab9354b6d238/pillow-11.3.0.tar.gz", hash = "sha256:3828ee7586cd0b2091b6209e5ad53e20d0649bbe87164a459d0676e035e8f523", size = 47113069 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/4c/5d/45a3553a253ac8763f3561371432a90bdbe6000fbdcf1397ffe502aa206c/pillow-11.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1b9c17fd4ace828b3003dfd1e30bff24863e0eb59b535e8f80194d9cc7ecf860", size = 5316554 },
-    { url = "https://files.pythonhosted.org/packages/7c/c8/67c12ab069ef586a25a4a79ced553586748fad100c77c0ce59bb4983ac98/pillow-11.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:65dc69160114cdd0ca0f35cb434633c75e8e7fad4cf855177a05bf38678f73ad", size = 4686548 },
-    { url = "https://files.pythonhosted.org/packages/2f/bd/6741ebd56263390b382ae4c5de02979af7f8bd9807346d068700dd6d5cf9/pillow-11.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7107195ddc914f656c7fc8e4a5e1c25f32e9236ea3ea860f257b0436011fddd0", size = 5859742 },
-    { url = "https://files.pythonhosted.org/packages/ca/0b/c412a9e27e1e6a829e6ab6c2dca52dd563efbedf4c9c6aa453d9a9b77359/pillow-11.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc3e831b563b3114baac7ec2ee86819eb03caa1a2cef0b481a5675b59c4fe23b", size = 7633087 },
-    { url = "https://files.pythonhosted.org/packages/59/9d/9b7076aaf30f5dd17e5e5589b2d2f5a5d7e30ff67a171eb686e4eecc2adf/pillow-11.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1f182ebd2303acf8c380a54f615ec883322593320a9b00438eb842c1f37ae50", size = 5963350 },
-    { url = "https://files.pythonhosted.org/packages/f0/16/1a6bf01fb622fb9cf5c91683823f073f053005c849b1f52ed613afcf8dae/pillow-11.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4445fa62e15936a028672fd48c4c11a66d641d2c05726c7ec1f8ba6a572036ae", size = 6631840 },
-    { url = "https://files.pythonhosted.org/packages/7b/e6/6ff7077077eb47fde78739e7d570bdcd7c10495666b6afcd23ab56b19a43/pillow-11.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:71f511f6b3b91dd543282477be45a033e4845a40278fa8dcdbfdb07109bf18f9", size = 6074005 },
-    { url = "https://files.pythonhosted.org/packages/c3/3a/b13f36832ea6d279a697231658199e0a03cd87ef12048016bdcc84131601/pillow-11.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:040a5b691b0713e1f6cbe222e0f4f74cd233421e105850ae3b3c0ceda520f42e", size = 6708372 },
-    { url = "https://files.pythonhosted.org/packages/6c/e4/61b2e1a7528740efbc70b3d581f33937e38e98ef3d50b05007267a55bcb2/pillow-11.3.0-cp310-cp310-win32.whl", hash = "sha256:89bd777bc6624fe4115e9fac3352c79ed60f3bb18651420635f26e643e3dd1f6", size = 6277090 },
-    { url = "https://files.pythonhosted.org/packages/a9/d3/60c781c83a785d6afbd6a326ed4d759d141de43aa7365725cbcd65ce5e54/pillow-11.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:19d2ff547c75b8e3ff46f4d9ef969a06c30ab2d4263a9e287733aa8b2429ce8f", size = 6985988 },
-    { url = "https://files.pythonhosted.org/packages/9f/28/4f4a0203165eefb3763939c6789ba31013a2e90adffb456610f30f613850/pillow-11.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:819931d25e57b513242859ce1876c58c59dc31587847bf74cfe06b2e0cb22d2f", size = 2422899 },
-    { url = "https://files.pythonhosted.org/packages/db/26/77f8ed17ca4ffd60e1dcd220a6ec6d71210ba398cfa33a13a1cd614c5613/pillow-11.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1cd110edf822773368b396281a2293aeb91c90a2db00d78ea43e7e861631b722", size = 5316531 },
-    { url = "https://files.pythonhosted.org/packages/cb/39/ee475903197ce709322a17a866892efb560f57900d9af2e55f86db51b0a5/pillow-11.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c412fddd1b77a75aa904615ebaa6001f169b26fd467b4be93aded278266b288", size = 4686560 },
-    { url = "https://files.pythonhosted.org/packages/d5/90/442068a160fd179938ba55ec8c97050a612426fae5ec0a764e345839f76d/pillow-11.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1aa4de119a0ecac0a34a9c8bde33f34022e2e8f99104e47a3ca392fd60e37d", size = 5870978 },
-    { url = "https://files.pythonhosted.org/packages/13/92/dcdd147ab02daf405387f0218dcf792dc6dd5b14d2573d40b4caeef01059/pillow-11.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:91da1d88226663594e3f6b4b8c3c8d85bd504117d043740a8e0ec449087cc494", size = 7641168 },
-    { url = "https://files.pythonhosted.org/packages/6e/db/839d6ba7fd38b51af641aa904e2960e7a5644d60ec754c046b7d2aee00e5/pillow-11.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:643f189248837533073c405ec2f0bb250ba54598cf80e8c1e043381a60632f58", size = 5973053 },
-    { url = "https://files.pythonhosted.org/packages/f2/2f/d7675ecae6c43e9f12aa8d58b6012683b20b6edfbdac7abcb4e6af7a3784/pillow-11.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:106064daa23a745510dabce1d84f29137a37224831d88eb4ce94bb187b1d7e5f", size = 6640273 },
-    { url = "https://files.pythonhosted.org/packages/45/ad/931694675ede172e15b2ff03c8144a0ddaea1d87adb72bb07655eaffb654/pillow-11.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd8ff254faf15591e724dc7c4ddb6bf4793efcbe13802a4ae3e863cd300b493e", size = 6082043 },
-    { url = "https://files.pythonhosted.org/packages/3a/04/ba8f2b11fc80d2dd462d7abec16351b45ec99cbbaea4387648a44190351a/pillow-11.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:932c754c2d51ad2b2271fd01c3d121daaa35e27efae2a616f77bf164bc0b3e94", size = 6715516 },
-    { url = "https://files.pythonhosted.org/packages/48/59/8cd06d7f3944cc7d892e8533c56b0acb68399f640786313275faec1e3b6f/pillow-11.3.0-cp311-cp311-win32.whl", hash = "sha256:b4b8f3efc8d530a1544e5962bd6b403d5f7fe8b9e08227c6b255f98ad82b4ba0", size = 6274768 },
-    { url = "https://files.pythonhosted.org/packages/f1/cc/29c0f5d64ab8eae20f3232da8f8571660aa0ab4b8f1331da5c2f5f9a938e/pillow-11.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:1a992e86b0dd7aeb1f053cd506508c0999d710a8f07b4c791c63843fc6a807ac", size = 6986055 },
-    { url = "https://files.pythonhosted.org/packages/c6/df/90bd886fabd544c25addd63e5ca6932c86f2b701d5da6c7839387a076b4a/pillow-11.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:30807c931ff7c095620fe04448e2c2fc673fcbb1ffe2a7da3fb39613489b1ddd", size = 2423079 },
-    { url = "https://files.pythonhosted.org/packages/40/fe/1bc9b3ee13f68487a99ac9529968035cca2f0a51ec36892060edcc51d06a/pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4", size = 5278800 },
-    { url = "https://files.pythonhosted.org/packages/2c/32/7e2ac19b5713657384cec55f89065fb306b06af008cfd87e572035b27119/pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69", size = 4686296 },
-    { url = "https://files.pythonhosted.org/packages/8e/1e/b9e12bbe6e4c2220effebc09ea0923a07a6da1e1f1bfbc8d7d29a01ce32b/pillow-11.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb76541cba2f958032d79d143b98a3a6b3ea87f0959bbe256c0b5e416599fd5d", size = 5871726 },
-    { url = "https://files.pythonhosted.org/packages/8d/33/e9200d2bd7ba00dc3ddb78df1198a6e80d7669cce6c2bdbeb2530a74ec58/pillow-11.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67172f2944ebba3d4a7b54f2e95c786a3a50c21b88456329314caaa28cda70f6", size = 7644652 },
-    { url = "https://files.pythonhosted.org/packages/41/f1/6f2427a26fc683e00d985bc391bdd76d8dd4e92fac33d841127eb8fb2313/pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7", size = 5977787 },
-    { url = "https://files.pythonhosted.org/packages/e4/c9/06dd4a38974e24f932ff5f98ea3c546ce3f8c995d3f0985f8e5ba48bba19/pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024", size = 6645236 },
-    { url = "https://files.pythonhosted.org/packages/40/e7/848f69fb79843b3d91241bad658e9c14f39a32f71a301bcd1d139416d1be/pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809", size = 6086950 },
-    { url = "https://files.pythonhosted.org/packages/0b/1a/7cff92e695a2a29ac1958c2a0fe4c0b2393b60aac13b04a4fe2735cad52d/pillow-11.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6be31e3fc9a621e071bc17bb7de63b85cbe0bfae91bb0363c893cbe67247780d", size = 6723358 },
-    { url = "https://files.pythonhosted.org/packages/26/7d/73699ad77895f69edff76b0f332acc3d497f22f5d75e5360f78cbcaff248/pillow-11.3.0-cp312-cp312-win32.whl", hash = "sha256:7b161756381f0918e05e7cb8a371fff367e807770f8fe92ecb20d905d0e1c149", size = 6275079 },
-    { url = "https://files.pythonhosted.org/packages/8c/ce/e7dfc873bdd9828f3b6e5c2bbb74e47a98ec23cc5c74fc4e54462f0d9204/pillow-11.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a6444696fce635783440b7f7a9fc24b3ad10a9ea3f0ab66c5905be1c19ccf17d", size = 6986324 },
-    { url = "https://files.pythonhosted.org/packages/16/8f/b13447d1bf0b1f7467ce7d86f6e6edf66c0ad7cf44cf5c87a37f9bed9936/pillow-11.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:2aceea54f957dd4448264f9bf40875da0415c83eb85f55069d89c0ed436e3542", size = 2423067 },
-    { url = "https://files.pythonhosted.org/packages/1e/93/0952f2ed8db3a5a4c7a11f91965d6184ebc8cd7cbb7941a260d5f018cd2d/pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1c627742b539bba4309df89171356fcb3cc5a9178355b2727d1b74a6cf155fbd", size = 2128328 },
-    { url = "https://files.pythonhosted.org/packages/4b/e8/100c3d114b1a0bf4042f27e0f87d2f25e857e838034e98ca98fe7b8c0a9c/pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:30b7c02f3899d10f13d7a48163c8969e4e653f8b43416d23d13d1bbfdc93b9f8", size = 2170652 },
-    { url = "https://files.pythonhosted.org/packages/aa/86/3f758a28a6e381758545f7cdb4942e1cb79abd271bea932998fc0db93cb6/pillow-11.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7859a4cc7c9295f5838015d8cc0a9c215b77e43d07a25e460f35cf516df8626f", size = 2227443 },
-    { url = "https://files.pythonhosted.org/packages/01/f4/91d5b3ffa718df2f53b0dc109877993e511f4fd055d7e9508682e8aba092/pillow-11.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec1ee50470b0d050984394423d96325b744d55c701a439d2bd66089bff963d3c", size = 5278474 },
-    { url = "https://files.pythonhosted.org/packages/f9/0e/37d7d3eca6c879fbd9dba21268427dffda1ab00d4eb05b32923d4fbe3b12/pillow-11.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7db51d222548ccfd274e4572fdbf3e810a5e66b00608862f947b163e613b67dd", size = 4686038 },
-    { url = "https://files.pythonhosted.org/packages/ff/b0/3426e5c7f6565e752d81221af9d3676fdbb4f352317ceafd42899aaf5d8a/pillow-11.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2d6fcc902a24ac74495df63faad1884282239265c6839a0a6416d33faedfae7e", size = 5864407 },
-    { url = "https://files.pythonhosted.org/packages/fc/c1/c6c423134229f2a221ee53f838d4be9d82bab86f7e2f8e75e47b6bf6cd77/pillow-11.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0f5d8f4a08090c6d6d578351a2b91acf519a54986c055af27e7a93feae6d3f1", size = 7639094 },
-    { url = "https://files.pythonhosted.org/packages/ba/c9/09e6746630fe6372c67c648ff9deae52a2bc20897d51fa293571977ceb5d/pillow-11.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c37d8ba9411d6003bba9e518db0db0c58a680ab9fe5179f040b0463644bc9805", size = 5973503 },
-    { url = "https://files.pythonhosted.org/packages/d5/1c/a2a29649c0b1983d3ef57ee87a66487fdeb45132df66ab30dd37f7dbe162/pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13f87d581e71d9189ab21fe0efb5a23e9f28552d5be6979e84001d3b8505abe8", size = 6642574 },
-    { url = "https://files.pythonhosted.org/packages/36/de/d5cc31cc4b055b6c6fd990e3e7f0f8aaf36229a2698501bcb0cdf67c7146/pillow-11.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:023f6d2d11784a465f09fd09a34b150ea4672e85fb3d05931d89f373ab14abb2", size = 6084060 },
-    { url = "https://files.pythonhosted.org/packages/d5/ea/502d938cbaeec836ac28a9b730193716f0114c41325db428e6b280513f09/pillow-11.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:45dfc51ac5975b938e9809451c51734124e73b04d0f0ac621649821a63852e7b", size = 6721407 },
-    { url = "https://files.pythonhosted.org/packages/45/9c/9c5e2a73f125f6cbc59cc7087c8f2d649a7ae453f83bd0362ff7c9e2aee2/pillow-11.3.0-cp313-cp313-win32.whl", hash = "sha256:a4d336baed65d50d37b88ca5b60c0fa9d81e3a87d4a7930d3880d1624d5b31f3", size = 6273841 },
-    { url = "https://files.pythonhosted.org/packages/23/85/397c73524e0cd212067e0c969aa245b01d50183439550d24d9f55781b776/pillow-11.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0bce5c4fd0921f99d2e858dc4d4d64193407e1b99478bc5cacecba2311abde51", size = 6978450 },
-    { url = "https://files.pythonhosted.org/packages/17/d2/622f4547f69cd173955194b78e4d19ca4935a1b0f03a302d655c9f6aae65/pillow-11.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:1904e1264881f682f02b7f8167935cce37bc97db457f8e7849dc3a6a52b99580", size = 2423055 },
-    { url = "https://files.pythonhosted.org/packages/dd/80/a8a2ac21dda2e82480852978416cfacd439a4b490a501a288ecf4fe2532d/pillow-11.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4c834a3921375c48ee6b9624061076bc0a32a60b5532b322cc0ea64e639dd50e", size = 5281110 },
-    { url = "https://files.pythonhosted.org/packages/44/d6/b79754ca790f315918732e18f82a8146d33bcd7f4494380457ea89eb883d/pillow-11.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5e05688ccef30ea69b9317a9ead994b93975104a677a36a8ed8106be9260aa6d", size = 4689547 },
-    { url = "https://files.pythonhosted.org/packages/49/20/716b8717d331150cb00f7fdd78169c01e8e0c219732a78b0e59b6bdb2fd6/pillow-11.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1019b04af07fc0163e2810167918cb5add8d74674b6267616021ab558dc98ced", size = 5901554 },
-    { url = "https://files.pythonhosted.org/packages/74/cf/a9f3a2514a65bb071075063a96f0a5cf949c2f2fce683c15ccc83b1c1cab/pillow-11.3.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f944255db153ebb2b19c51fe85dd99ef0ce494123f21b9db4877ffdfc5590c7c", size = 7669132 },
-    { url = "https://files.pythonhosted.org/packages/98/3c/da78805cbdbee9cb43efe8261dd7cc0b4b93f2ac79b676c03159e9db2187/pillow-11.3.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f85acb69adf2aaee8b7da124efebbdb959a104db34d3a2cb0f3793dbae422a8", size = 6005001 },
-    { url = "https://files.pythonhosted.org/packages/6c/fa/ce044b91faecf30e635321351bba32bab5a7e034c60187fe9698191aef4f/pillow-11.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f6ecbeff5005399bb48d198f098a9b4b6bdf27b8487c7f38ca16eeb070cd59", size = 6668814 },
-    { url = "https://files.pythonhosted.org/packages/7b/51/90f9291406d09bf93686434f9183aba27b831c10c87746ff49f127ee80cb/pillow-11.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a7bc6e6fd0395bc052f16b1a8670859964dbd7003bd0af2ff08342eb6e442cfe", size = 6113124 },
-    { url = "https://files.pythonhosted.org/packages/cd/5a/6fec59b1dfb619234f7636d4157d11fb4e196caeee220232a8d2ec48488d/pillow-11.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:83e1b0161c9d148125083a35c1c5a89db5b7054834fd4387499e06552035236c", size = 6747186 },
-    { url = "https://files.pythonhosted.org/packages/49/6b/00187a044f98255225f172de653941e61da37104a9ea60e4f6887717e2b5/pillow-11.3.0-cp313-cp313t-win32.whl", hash = "sha256:2a3117c06b8fb646639dce83694f2f9eac405472713fcb1ae887469c0d4f6788", size = 6277546 },
-    { url = "https://files.pythonhosted.org/packages/e8/5c/6caaba7e261c0d75bab23be79f1d06b5ad2a2ae49f028ccec801b0e853d6/pillow-11.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:857844335c95bea93fb39e0fa2726b4d9d758850b34075a7e3ff4f4fa3aa3b31", size = 6985102 },
-    { url = "https://files.pythonhosted.org/packages/f3/7e/b623008460c09a0cb38263c93b828c666493caee2eb34ff67f778b87e58c/pillow-11.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:8797edc41f3e8536ae4b10897ee2f637235c94f27404cac7297f7b607dd0716e", size = 2424803 },
-    { url = "https://files.pythonhosted.org/packages/73/f4/04905af42837292ed86cb1b1dabe03dce1edc008ef14c473c5c7e1443c5d/pillow-11.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d9da3df5f9ea2a89b81bb6087177fb1f4d1c7146d583a3fe5c672c0d94e55e12", size = 5278520 },
-    { url = "https://files.pythonhosted.org/packages/41/b0/33d79e377a336247df6348a54e6d2a2b85d644ca202555e3faa0cf811ecc/pillow-11.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b275ff9b04df7b640c59ec5a3cb113eefd3795a8df80bac69646ef699c6981a", size = 4686116 },
-    { url = "https://files.pythonhosted.org/packages/49/2d/ed8bc0ab219ae8768f529597d9509d184fe8a6c4741a6864fea334d25f3f/pillow-11.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0743841cabd3dba6a83f38a92672cccbd69af56e3e91777b0ee7f4dba4385632", size = 5864597 },
-    { url = "https://files.pythonhosted.org/packages/b5/3d/b932bb4225c80b58dfadaca9d42d08d0b7064d2d1791b6a237f87f661834/pillow-11.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2465a69cf967b8b49ee1b96d76718cd98c4e925414ead59fdf75cf0fd07df673", size = 7638246 },
-    { url = "https://files.pythonhosted.org/packages/09/b5/0487044b7c096f1b48f0d7ad416472c02e0e4bf6919541b111efd3cae690/pillow-11.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41742638139424703b4d01665b807c6468e23e699e8e90cffefe291c5832b027", size = 5973336 },
-    { url = "https://files.pythonhosted.org/packages/a8/2d/524f9318f6cbfcc79fbc004801ea6b607ec3f843977652fdee4857a7568b/pillow-11.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93efb0b4de7e340d99057415c749175e24c8864302369e05914682ba642e5d77", size = 6642699 },
-    { url = "https://files.pythonhosted.org/packages/6f/d2/a9a4f280c6aefedce1e8f615baaa5474e0701d86dd6f1dede66726462bbd/pillow-11.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7966e38dcd0fa11ca390aed7c6f20454443581d758242023cf36fcb319b1a874", size = 6083789 },
-    { url = "https://files.pythonhosted.org/packages/fe/54/86b0cd9dbb683a9d5e960b66c7379e821a19be4ac5810e2e5a715c09a0c0/pillow-11.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:98a9afa7b9007c67ed84c57c9e0ad86a6000da96eaa638e4f8abe5b65ff83f0a", size = 6720386 },
-    { url = "https://files.pythonhosted.org/packages/e7/95/88efcaf384c3588e24259c4203b909cbe3e3c2d887af9e938c2022c9dd48/pillow-11.3.0-cp314-cp314-win32.whl", hash = "sha256:02a723e6bf909e7cea0dac1b0e0310be9d7650cd66222a5f1c571455c0a45214", size = 6370911 },
-    { url = "https://files.pythonhosted.org/packages/2e/cc/934e5820850ec5eb107e7b1a72dd278140731c669f396110ebc326f2a503/pillow-11.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:a418486160228f64dd9e9efcd132679b7a02a5f22c982c78b6fc7dab3fefb635", size = 7117383 },
-    { url = "https://files.pythonhosted.org/packages/d6/e9/9c0a616a71da2a5d163aa37405e8aced9a906d574b4a214bede134e731bc/pillow-11.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:155658efb5e044669c08896c0c44231c5e9abcaadbc5cd3648df2f7c0b96b9a6", size = 2511385 },
-    { url = "https://files.pythonhosted.org/packages/1a/33/c88376898aff369658b225262cd4f2659b13e8178e7534df9e6e1fa289f6/pillow-11.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:59a03cdf019efbfeeed910bf79c7c93255c3d54bc45898ac2a4140071b02b4ae", size = 5281129 },
-    { url = "https://files.pythonhosted.org/packages/1f/70/d376247fb36f1844b42910911c83a02d5544ebd2a8bad9efcc0f707ea774/pillow-11.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f8a5827f84d973d8636e9dc5764af4f0cf2318d26744b3d902931701b0d46653", size = 4689580 },
-    { url = "https://files.pythonhosted.org/packages/eb/1c/537e930496149fbac69efd2fc4329035bbe2e5475b4165439e3be9cb183b/pillow-11.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ee92f2fd10f4adc4b43d07ec5e779932b4eb3dbfbc34790ada5a6669bc095aa6", size = 5902860 },
-    { url = "https://files.pythonhosted.org/packages/bd/57/80f53264954dcefeebcf9dae6e3eb1daea1b488f0be8b8fef12f79a3eb10/pillow-11.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c96d333dcf42d01f47b37e0979b6bd73ec91eae18614864622d9b87bbd5bbf36", size = 7670694 },
-    { url = "https://files.pythonhosted.org/packages/70/ff/4727d3b71a8578b4587d9c276e90efad2d6fe0335fd76742a6da08132e8c/pillow-11.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c96f993ab8c98460cd0c001447bff6194403e8b1d7e149ade5f00594918128b", size = 6005888 },
-    { url = "https://files.pythonhosted.org/packages/05/ae/716592277934f85d3be51d7256f3636672d7b1abfafdc42cf3f8cbd4b4c8/pillow-11.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41342b64afeba938edb034d122b2dda5db2139b9a4af999729ba8818e0056477", size = 6670330 },
-    { url = "https://files.pythonhosted.org/packages/e7/bb/7fe6cddcc8827b01b1a9766f5fdeb7418680744f9082035bdbabecf1d57f/pillow-11.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:068d9c39a2d1b358eb9f245ce7ab1b5c3246c7c8c7d9ba58cfa5b43146c06e50", size = 6114089 },
-    { url = "https://files.pythonhosted.org/packages/8b/f5/06bfaa444c8e80f1a8e4bff98da9c83b37b5be3b1deaa43d27a0db37ef84/pillow-11.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a1bc6ba083b145187f648b667e05a2534ecc4b9f2784c2cbe3089e44868f2b9b", size = 6748206 },
-    { url = "https://files.pythonhosted.org/packages/f0/77/bc6f92a3e8e6e46c0ca78abfffec0037845800ea38c73483760362804c41/pillow-11.3.0-cp314-cp314t-win32.whl", hash = "sha256:118ca10c0d60b06d006be10a501fd6bbdfef559251ed31b794668ed569c87e12", size = 6377370 },
-    { url = "https://files.pythonhosted.org/packages/4a/82/3a721f7d69dca802befb8af08b7c79ebcab461007ce1c18bd91a5d5896f9/pillow-11.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8924748b688aa210d79883357d102cd64690e56b923a186f35a82cbc10f997db", size = 7121500 },
-    { url = "https://files.pythonhosted.org/packages/89/c7/5572fa4a3f45740eaab6ae86fcdf7195b55beac1371ac8c619d880cfe948/pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa", size = 2512835 },
-    { url = "https://files.pythonhosted.org/packages/6f/8b/209bd6b62ce8367f47e68a218bffac88888fdf2c9fcf1ecadc6c3ec1ebc7/pillow-11.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3cee80663f29e3843b68199b9d6f4f54bd1d4a6b59bdd91bceefc51238bcb967", size = 5270556 },
-    { url = "https://files.pythonhosted.org/packages/2e/e6/231a0b76070c2cfd9e260a7a5b504fb72da0a95279410fa7afd99d9751d6/pillow-11.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b5f56c3f344f2ccaf0dd875d3e180f631dc60a51b314295a3e681fe8cf851fbe", size = 4654625 },
-    { url = "https://files.pythonhosted.org/packages/13/f4/10cf94fda33cb12765f2397fc285fa6d8eb9c29de7f3185165b702fc7386/pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e67d793d180c9df62f1f40aee3accca4829d3794c95098887edc18af4b8b780c", size = 4874207 },
-    { url = "https://files.pythonhosted.org/packages/72/c9/583821097dc691880c92892e8e2d41fe0a5a3d6021f4963371d2f6d57250/pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d000f46e2917c705e9fb93a3606ee4a819d1e3aa7a9b442f6444f07e77cf5e25", size = 6583939 },
-    { url = "https://files.pythonhosted.org/packages/3b/8e/5c9d410f9217b12320efc7c413e72693f48468979a013ad17fd690397b9a/pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:527b37216b6ac3a12d7838dc3bd75208ec57c1c6d11ef01902266a5a0c14fc27", size = 4957166 },
-    { url = "https://files.pythonhosted.org/packages/62/bb/78347dbe13219991877ffb3a91bf09da8317fbfcd4b5f9140aeae020ad71/pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be5463ac478b623b9dd3937afd7fb7ab3d79dd290a28e2b6df292dc75063eb8a", size = 5581482 },
-    { url = "https://files.pythonhosted.org/packages/d9/28/1000353d5e61498aaeaaf7f1e4b49ddb05f2c6575f9d4f9f914a3538b6e1/pillow-11.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8dc70ca24c110503e16918a658b869019126ecfe03109b754c402daff12b3d9f", size = 6984596 },
-    { url = "https://files.pythonhosted.org/packages/9e/e3/6fa84033758276fb31da12e5fb66ad747ae83b93c67af17f8c6ff4cc8f34/pillow-11.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7c8ec7a017ad1bd562f93dbd8505763e688d388cde6e4a010ae1486916e713e6", size = 5270566 },
-    { url = "https://files.pythonhosted.org/packages/5b/ee/e8d2e1ab4892970b561e1ba96cbd59c0d28cf66737fc44abb2aec3795a4e/pillow-11.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9ab6ae226de48019caa8074894544af5b53a117ccb9d3b3dcb2871464c829438", size = 4654618 },
-    { url = "https://files.pythonhosted.org/packages/f2/6d/17f80f4e1f0761f02160fc433abd4109fa1548dcfdca46cfdadaf9efa565/pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe27fb049cdcca11f11a7bfda64043c37b30e6b91f10cb5bab275806c32f6ab3", size = 4874248 },
-    { url = "https://files.pythonhosted.org/packages/de/5f/c22340acd61cef960130585bbe2120e2fd8434c214802f07e8c03596b17e/pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:465b9e8844e3c3519a983d58b80be3f668e2a7a5db97f2784e7079fbc9f9822c", size = 6583963 },
-    { url = "https://files.pythonhosted.org/packages/31/5e/03966aedfbfcbb4d5f8aa042452d3361f325b963ebbadddac05b122e47dd/pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5418b53c0d59b3824d05e029669efa023bbef0f3e92e75ec8428f3799487f361", size = 4957170 },
-    { url = "https://files.pythonhosted.org/packages/cc/2d/e082982aacc927fc2cab48e1e731bdb1643a1406acace8bed0900a61464e/pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:504b6f59505f08ae014f724b6207ff6222662aab5cc9542577fb084ed0676ac7", size = 5581505 },
-    { url = "https://files.pythonhosted.org/packages/34/e7/ae39f538fd6844e982063c3a5e4598b8ced43b9633baa3a85ef33af8c05c/pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8", size = 6984598 },
+sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/d0d6dea55cd152ce3d6767bb38a8fc10e33796ba4ba210cbab9354b6d238/pillow-11.3.0.tar.gz", hash = "sha256:3828ee7586cd0b2091b6209e5ad53e20d0649bbe87164a459d0676e035e8f523", size = 47113069, upload-time = "2025-07-01T09:16:30.666Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/4c/5d/45a3553a253ac8763f3561371432a90bdbe6000fbdcf1397ffe502aa206c/pillow-11.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1b9c17fd4ace828b3003dfd1e30bff24863e0eb59b535e8f80194d9cc7ecf860", size = 5316554, upload-time = "2025-07-01T09:13:39.342Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/c8/67c12ab069ef586a25a4a79ced553586748fad100c77c0ce59bb4983ac98/pillow-11.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:65dc69160114cdd0ca0f35cb434633c75e8e7fad4cf855177a05bf38678f73ad", size = 4686548, upload-time = "2025-07-01T09:13:41.835Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/bd/6741ebd56263390b382ae4c5de02979af7f8bd9807346d068700dd6d5cf9/pillow-11.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7107195ddc914f656c7fc8e4a5e1c25f32e9236ea3ea860f257b0436011fddd0", size = 5859742, upload-time = "2025-07-03T13:09:47.439Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/0b/c412a9e27e1e6a829e6ab6c2dca52dd563efbedf4c9c6aa453d9a9b77359/pillow-11.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc3e831b563b3114baac7ec2ee86819eb03caa1a2cef0b481a5675b59c4fe23b", size = 7633087, upload-time = "2025-07-03T13:09:51.796Z" },
+    { url = "https://files.pythonhosted.org/packages/59/9d/9b7076aaf30f5dd17e5e5589b2d2f5a5d7e30ff67a171eb686e4eecc2adf/pillow-11.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1f182ebd2303acf8c380a54f615ec883322593320a9b00438eb842c1f37ae50", size = 5963350, upload-time = "2025-07-01T09:13:43.865Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/16/1a6bf01fb622fb9cf5c91683823f073f053005c849b1f52ed613afcf8dae/pillow-11.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4445fa62e15936a028672fd48c4c11a66d641d2c05726c7ec1f8ba6a572036ae", size = 6631840, upload-time = "2025-07-01T09:13:46.161Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/e6/6ff7077077eb47fde78739e7d570bdcd7c10495666b6afcd23ab56b19a43/pillow-11.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:71f511f6b3b91dd543282477be45a033e4845a40278fa8dcdbfdb07109bf18f9", size = 6074005, upload-time = "2025-07-01T09:13:47.829Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/3a/b13f36832ea6d279a697231658199e0a03cd87ef12048016bdcc84131601/pillow-11.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:040a5b691b0713e1f6cbe222e0f4f74cd233421e105850ae3b3c0ceda520f42e", size = 6708372, upload-time = "2025-07-01T09:13:52.145Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/e4/61b2e1a7528740efbc70b3d581f33937e38e98ef3d50b05007267a55bcb2/pillow-11.3.0-cp310-cp310-win32.whl", hash = "sha256:89bd777bc6624fe4115e9fac3352c79ed60f3bb18651420635f26e643e3dd1f6", size = 6277090, upload-time = "2025-07-01T09:13:53.915Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/d3/60c781c83a785d6afbd6a326ed4d759d141de43aa7365725cbcd65ce5e54/pillow-11.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:19d2ff547c75b8e3ff46f4d9ef969a06c30ab2d4263a9e287733aa8b2429ce8f", size = 6985988, upload-time = "2025-07-01T09:13:55.699Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/28/4f4a0203165eefb3763939c6789ba31013a2e90adffb456610f30f613850/pillow-11.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:819931d25e57b513242859ce1876c58c59dc31587847bf74cfe06b2e0cb22d2f", size = 2422899, upload-time = "2025-07-01T09:13:57.497Z" },
+    { url = "https://files.pythonhosted.org/packages/db/26/77f8ed17ca4ffd60e1dcd220a6ec6d71210ba398cfa33a13a1cd614c5613/pillow-11.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1cd110edf822773368b396281a2293aeb91c90a2db00d78ea43e7e861631b722", size = 5316531, upload-time = "2025-07-01T09:13:59.203Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/39/ee475903197ce709322a17a866892efb560f57900d9af2e55f86db51b0a5/pillow-11.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c412fddd1b77a75aa904615ebaa6001f169b26fd467b4be93aded278266b288", size = 4686560, upload-time = "2025-07-01T09:14:01.101Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/90/442068a160fd179938ba55ec8c97050a612426fae5ec0a764e345839f76d/pillow-11.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1aa4de119a0ecac0a34a9c8bde33f34022e2e8f99104e47a3ca392fd60e37d", size = 5870978, upload-time = "2025-07-03T13:09:55.638Z" },
+    { url = "https://files.pythonhosted.org/packages/13/92/dcdd147ab02daf405387f0218dcf792dc6dd5b14d2573d40b4caeef01059/pillow-11.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:91da1d88226663594e3f6b4b8c3c8d85bd504117d043740a8e0ec449087cc494", size = 7641168, upload-time = "2025-07-03T13:10:00.37Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/db/839d6ba7fd38b51af641aa904e2960e7a5644d60ec754c046b7d2aee00e5/pillow-11.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:643f189248837533073c405ec2f0bb250ba54598cf80e8c1e043381a60632f58", size = 5973053, upload-time = "2025-07-01T09:14:04.491Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/2f/d7675ecae6c43e9f12aa8d58b6012683b20b6edfbdac7abcb4e6af7a3784/pillow-11.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:106064daa23a745510dabce1d84f29137a37224831d88eb4ce94bb187b1d7e5f", size = 6640273, upload-time = "2025-07-01T09:14:06.235Z" },
+    { url = "https://files.pythonhosted.org/packages/45/ad/931694675ede172e15b2ff03c8144a0ddaea1d87adb72bb07655eaffb654/pillow-11.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd8ff254faf15591e724dc7c4ddb6bf4793efcbe13802a4ae3e863cd300b493e", size = 6082043, upload-time = "2025-07-01T09:14:07.978Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/04/ba8f2b11fc80d2dd462d7abec16351b45ec99cbbaea4387648a44190351a/pillow-11.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:932c754c2d51ad2b2271fd01c3d121daaa35e27efae2a616f77bf164bc0b3e94", size = 6715516, upload-time = "2025-07-01T09:14:10.233Z" },
+    { url = "https://files.pythonhosted.org/packages/48/59/8cd06d7f3944cc7d892e8533c56b0acb68399f640786313275faec1e3b6f/pillow-11.3.0-cp311-cp311-win32.whl", hash = "sha256:b4b8f3efc8d530a1544e5962bd6b403d5f7fe8b9e08227c6b255f98ad82b4ba0", size = 6274768, upload-time = "2025-07-01T09:14:11.921Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/cc/29c0f5d64ab8eae20f3232da8f8571660aa0ab4b8f1331da5c2f5f9a938e/pillow-11.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:1a992e86b0dd7aeb1f053cd506508c0999d710a8f07b4c791c63843fc6a807ac", size = 6986055, upload-time = "2025-07-01T09:14:13.623Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/df/90bd886fabd544c25addd63e5ca6932c86f2b701d5da6c7839387a076b4a/pillow-11.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:30807c931ff7c095620fe04448e2c2fc673fcbb1ffe2a7da3fb39613489b1ddd", size = 2423079, upload-time = "2025-07-01T09:14:15.268Z" },
+    { url = "https://files.pythonhosted.org/packages/40/fe/1bc9b3ee13f68487a99ac9529968035cca2f0a51ec36892060edcc51d06a/pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4", size = 5278800, upload-time = "2025-07-01T09:14:17.648Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/32/7e2ac19b5713657384cec55f89065fb306b06af008cfd87e572035b27119/pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69", size = 4686296, upload-time = "2025-07-01T09:14:19.828Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/1e/b9e12bbe6e4c2220effebc09ea0923a07a6da1e1f1bfbc8d7d29a01ce32b/pillow-11.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb76541cba2f958032d79d143b98a3a6b3ea87f0959bbe256c0b5e416599fd5d", size = 5871726, upload-time = "2025-07-03T13:10:04.448Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/33/e9200d2bd7ba00dc3ddb78df1198a6e80d7669cce6c2bdbeb2530a74ec58/pillow-11.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67172f2944ebba3d4a7b54f2e95c786a3a50c21b88456329314caaa28cda70f6", size = 7644652, upload-time = "2025-07-03T13:10:10.391Z" },
+    { url = "https://files.pythonhosted.org/packages/41/f1/6f2427a26fc683e00d985bc391bdd76d8dd4e92fac33d841127eb8fb2313/pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7", size = 5977787, upload-time = "2025-07-01T09:14:21.63Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/c9/06dd4a38974e24f932ff5f98ea3c546ce3f8c995d3f0985f8e5ba48bba19/pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024", size = 6645236, upload-time = "2025-07-01T09:14:23.321Z" },
+    { url = "https://files.pythonhosted.org/packages/40/e7/848f69fb79843b3d91241bad658e9c14f39a32f71a301bcd1d139416d1be/pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809", size = 6086950, upload-time = "2025-07-01T09:14:25.237Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/1a/7cff92e695a2a29ac1958c2a0fe4c0b2393b60aac13b04a4fe2735cad52d/pillow-11.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6be31e3fc9a621e071bc17bb7de63b85cbe0bfae91bb0363c893cbe67247780d", size = 6723358, upload-time = "2025-07-01T09:14:27.053Z" },
+    { url = "https://files.pythonhosted.org/packages/26/7d/73699ad77895f69edff76b0f332acc3d497f22f5d75e5360f78cbcaff248/pillow-11.3.0-cp312-cp312-win32.whl", hash = "sha256:7b161756381f0918e05e7cb8a371fff367e807770f8fe92ecb20d905d0e1c149", size = 6275079, upload-time = "2025-07-01T09:14:30.104Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/ce/e7dfc873bdd9828f3b6e5c2bbb74e47a98ec23cc5c74fc4e54462f0d9204/pillow-11.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a6444696fce635783440b7f7a9fc24b3ad10a9ea3f0ab66c5905be1c19ccf17d", size = 6986324, upload-time = "2025-07-01T09:14:31.899Z" },
+    { url = "https://files.pythonhosted.org/packages/16/8f/b13447d1bf0b1f7467ce7d86f6e6edf66c0ad7cf44cf5c87a37f9bed9936/pillow-11.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:2aceea54f957dd4448264f9bf40875da0415c83eb85f55069d89c0ed436e3542", size = 2423067, upload-time = "2025-07-01T09:14:33.709Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/93/0952f2ed8db3a5a4c7a11f91965d6184ebc8cd7cbb7941a260d5f018cd2d/pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1c627742b539bba4309df89171356fcb3cc5a9178355b2727d1b74a6cf155fbd", size = 2128328, upload-time = "2025-07-01T09:14:35.276Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/e8/100c3d114b1a0bf4042f27e0f87d2f25e857e838034e98ca98fe7b8c0a9c/pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:30b7c02f3899d10f13d7a48163c8969e4e653f8b43416d23d13d1bbfdc93b9f8", size = 2170652, upload-time = "2025-07-01T09:14:37.203Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/86/3f758a28a6e381758545f7cdb4942e1cb79abd271bea932998fc0db93cb6/pillow-11.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7859a4cc7c9295f5838015d8cc0a9c215b77e43d07a25e460f35cf516df8626f", size = 2227443, upload-time = "2025-07-01T09:14:39.344Z" },
+    { url = "https://files.pythonhosted.org/packages/01/f4/91d5b3ffa718df2f53b0dc109877993e511f4fd055d7e9508682e8aba092/pillow-11.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec1ee50470b0d050984394423d96325b744d55c701a439d2bd66089bff963d3c", size = 5278474, upload-time = "2025-07-01T09:14:41.843Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/0e/37d7d3eca6c879fbd9dba21268427dffda1ab00d4eb05b32923d4fbe3b12/pillow-11.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7db51d222548ccfd274e4572fdbf3e810a5e66b00608862f947b163e613b67dd", size = 4686038, upload-time = "2025-07-01T09:14:44.008Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/b0/3426e5c7f6565e752d81221af9d3676fdbb4f352317ceafd42899aaf5d8a/pillow-11.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2d6fcc902a24ac74495df63faad1884282239265c6839a0a6416d33faedfae7e", size = 5864407, upload-time = "2025-07-03T13:10:15.628Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/c1/c6c423134229f2a221ee53f838d4be9d82bab86f7e2f8e75e47b6bf6cd77/pillow-11.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0f5d8f4a08090c6d6d578351a2b91acf519a54986c055af27e7a93feae6d3f1", size = 7639094, upload-time = "2025-07-03T13:10:21.857Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/c9/09e6746630fe6372c67c648ff9deae52a2bc20897d51fa293571977ceb5d/pillow-11.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c37d8ba9411d6003bba9e518db0db0c58a680ab9fe5179f040b0463644bc9805", size = 5973503, upload-time = "2025-07-01T09:14:45.698Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/1c/a2a29649c0b1983d3ef57ee87a66487fdeb45132df66ab30dd37f7dbe162/pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13f87d581e71d9189ab21fe0efb5a23e9f28552d5be6979e84001d3b8505abe8", size = 6642574, upload-time = "2025-07-01T09:14:47.415Z" },
+    { url = "https://files.pythonhosted.org/packages/36/de/d5cc31cc4b055b6c6fd990e3e7f0f8aaf36229a2698501bcb0cdf67c7146/pillow-11.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:023f6d2d11784a465f09fd09a34b150ea4672e85fb3d05931d89f373ab14abb2", size = 6084060, upload-time = "2025-07-01T09:14:49.636Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/ea/502d938cbaeec836ac28a9b730193716f0114c41325db428e6b280513f09/pillow-11.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:45dfc51ac5975b938e9809451c51734124e73b04d0f0ac621649821a63852e7b", size = 6721407, upload-time = "2025-07-01T09:14:51.962Z" },
+    { url = "https://files.pythonhosted.org/packages/45/9c/9c5e2a73f125f6cbc59cc7087c8f2d649a7ae453f83bd0362ff7c9e2aee2/pillow-11.3.0-cp313-cp313-win32.whl", hash = "sha256:a4d336baed65d50d37b88ca5b60c0fa9d81e3a87d4a7930d3880d1624d5b31f3", size = 6273841, upload-time = "2025-07-01T09:14:54.142Z" },
+    { url = "https://files.pythonhosted.org/packages/23/85/397c73524e0cd212067e0c969aa245b01d50183439550d24d9f55781b776/pillow-11.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0bce5c4fd0921f99d2e858dc4d4d64193407e1b99478bc5cacecba2311abde51", size = 6978450, upload-time = "2025-07-01T09:14:56.436Z" },
+    { url = "https://files.pythonhosted.org/packages/17/d2/622f4547f69cd173955194b78e4d19ca4935a1b0f03a302d655c9f6aae65/pillow-11.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:1904e1264881f682f02b7f8167935cce37bc97db457f8e7849dc3a6a52b99580", size = 2423055, upload-time = "2025-07-01T09:14:58.072Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/80/a8a2ac21dda2e82480852978416cfacd439a4b490a501a288ecf4fe2532d/pillow-11.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4c834a3921375c48ee6b9624061076bc0a32a60b5532b322cc0ea64e639dd50e", size = 5281110, upload-time = "2025-07-01T09:14:59.79Z" },
+    { url = "https://files.pythonhosted.org/packages/44/d6/b79754ca790f315918732e18f82a8146d33bcd7f4494380457ea89eb883d/pillow-11.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5e05688ccef30ea69b9317a9ead994b93975104a677a36a8ed8106be9260aa6d", size = 4689547, upload-time = "2025-07-01T09:15:01.648Z" },
+    { url = "https://files.pythonhosted.org/packages/49/20/716b8717d331150cb00f7fdd78169c01e8e0c219732a78b0e59b6bdb2fd6/pillow-11.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1019b04af07fc0163e2810167918cb5add8d74674b6267616021ab558dc98ced", size = 5901554, upload-time = "2025-07-03T13:10:27.018Z" },
+    { url = "https://files.pythonhosted.org/packages/74/cf/a9f3a2514a65bb071075063a96f0a5cf949c2f2fce683c15ccc83b1c1cab/pillow-11.3.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f944255db153ebb2b19c51fe85dd99ef0ce494123f21b9db4877ffdfc5590c7c", size = 7669132, upload-time = "2025-07-03T13:10:33.01Z" },
+    { url = "https://files.pythonhosted.org/packages/98/3c/da78805cbdbee9cb43efe8261dd7cc0b4b93f2ac79b676c03159e9db2187/pillow-11.3.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f85acb69adf2aaee8b7da124efebbdb959a104db34d3a2cb0f3793dbae422a8", size = 6005001, upload-time = "2025-07-01T09:15:03.365Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/fa/ce044b91faecf30e635321351bba32bab5a7e034c60187fe9698191aef4f/pillow-11.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f6ecbeff5005399bb48d198f098a9b4b6bdf27b8487c7f38ca16eeb070cd59", size = 6668814, upload-time = "2025-07-01T09:15:05.655Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/51/90f9291406d09bf93686434f9183aba27b831c10c87746ff49f127ee80cb/pillow-11.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a7bc6e6fd0395bc052f16b1a8670859964dbd7003bd0af2ff08342eb6e442cfe", size = 6113124, upload-time = "2025-07-01T09:15:07.358Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/5a/6fec59b1dfb619234f7636d4157d11fb4e196caeee220232a8d2ec48488d/pillow-11.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:83e1b0161c9d148125083a35c1c5a89db5b7054834fd4387499e06552035236c", size = 6747186, upload-time = "2025-07-01T09:15:09.317Z" },
+    { url = "https://files.pythonhosted.org/packages/49/6b/00187a044f98255225f172de653941e61da37104a9ea60e4f6887717e2b5/pillow-11.3.0-cp313-cp313t-win32.whl", hash = "sha256:2a3117c06b8fb646639dce83694f2f9eac405472713fcb1ae887469c0d4f6788", size = 6277546, upload-time = "2025-07-01T09:15:11.311Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/5c/6caaba7e261c0d75bab23be79f1d06b5ad2a2ae49f028ccec801b0e853d6/pillow-11.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:857844335c95bea93fb39e0fa2726b4d9d758850b34075a7e3ff4f4fa3aa3b31", size = 6985102, upload-time = "2025-07-01T09:15:13.164Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/7e/b623008460c09a0cb38263c93b828c666493caee2eb34ff67f778b87e58c/pillow-11.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:8797edc41f3e8536ae4b10897ee2f637235c94f27404cac7297f7b607dd0716e", size = 2424803, upload-time = "2025-07-01T09:15:15.695Z" },
+    { url = "https://files.pythonhosted.org/packages/73/f4/04905af42837292ed86cb1b1dabe03dce1edc008ef14c473c5c7e1443c5d/pillow-11.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d9da3df5f9ea2a89b81bb6087177fb1f4d1c7146d583a3fe5c672c0d94e55e12", size = 5278520, upload-time = "2025-07-01T09:15:17.429Z" },
+    { url = "https://files.pythonhosted.org/packages/41/b0/33d79e377a336247df6348a54e6d2a2b85d644ca202555e3faa0cf811ecc/pillow-11.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b275ff9b04df7b640c59ec5a3cb113eefd3795a8df80bac69646ef699c6981a", size = 4686116, upload-time = "2025-07-01T09:15:19.423Z" },
+    { url = "https://files.pythonhosted.org/packages/49/2d/ed8bc0ab219ae8768f529597d9509d184fe8a6c4741a6864fea334d25f3f/pillow-11.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0743841cabd3dba6a83f38a92672cccbd69af56e3e91777b0ee7f4dba4385632", size = 5864597, upload-time = "2025-07-03T13:10:38.404Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/3d/b932bb4225c80b58dfadaca9d42d08d0b7064d2d1791b6a237f87f661834/pillow-11.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2465a69cf967b8b49ee1b96d76718cd98c4e925414ead59fdf75cf0fd07df673", size = 7638246, upload-time = "2025-07-03T13:10:44.987Z" },
+    { url = "https://files.pythonhosted.org/packages/09/b5/0487044b7c096f1b48f0d7ad416472c02e0e4bf6919541b111efd3cae690/pillow-11.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41742638139424703b4d01665b807c6468e23e699e8e90cffefe291c5832b027", size = 5973336, upload-time = "2025-07-01T09:15:21.237Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/2d/524f9318f6cbfcc79fbc004801ea6b607ec3f843977652fdee4857a7568b/pillow-11.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93efb0b4de7e340d99057415c749175e24c8864302369e05914682ba642e5d77", size = 6642699, upload-time = "2025-07-01T09:15:23.186Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/d2/a9a4f280c6aefedce1e8f615baaa5474e0701d86dd6f1dede66726462bbd/pillow-11.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7966e38dcd0fa11ca390aed7c6f20454443581d758242023cf36fcb319b1a874", size = 6083789, upload-time = "2025-07-01T09:15:25.1Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/54/86b0cd9dbb683a9d5e960b66c7379e821a19be4ac5810e2e5a715c09a0c0/pillow-11.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:98a9afa7b9007c67ed84c57c9e0ad86a6000da96eaa638e4f8abe5b65ff83f0a", size = 6720386, upload-time = "2025-07-01T09:15:27.378Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/95/88efcaf384c3588e24259c4203b909cbe3e3c2d887af9e938c2022c9dd48/pillow-11.3.0-cp314-cp314-win32.whl", hash = "sha256:02a723e6bf909e7cea0dac1b0e0310be9d7650cd66222a5f1c571455c0a45214", size = 6370911, upload-time = "2025-07-01T09:15:29.294Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/cc/934e5820850ec5eb107e7b1a72dd278140731c669f396110ebc326f2a503/pillow-11.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:a418486160228f64dd9e9efcd132679b7a02a5f22c982c78b6fc7dab3fefb635", size = 7117383, upload-time = "2025-07-01T09:15:31.128Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/e9/9c0a616a71da2a5d163aa37405e8aced9a906d574b4a214bede134e731bc/pillow-11.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:155658efb5e044669c08896c0c44231c5e9abcaadbc5cd3648df2f7c0b96b9a6", size = 2511385, upload-time = "2025-07-01T09:15:33.328Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/33/c88376898aff369658b225262cd4f2659b13e8178e7534df9e6e1fa289f6/pillow-11.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:59a03cdf019efbfeeed910bf79c7c93255c3d54bc45898ac2a4140071b02b4ae", size = 5281129, upload-time = "2025-07-01T09:15:35.194Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/70/d376247fb36f1844b42910911c83a02d5544ebd2a8bad9efcc0f707ea774/pillow-11.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f8a5827f84d973d8636e9dc5764af4f0cf2318d26744b3d902931701b0d46653", size = 4689580, upload-time = "2025-07-01T09:15:37.114Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/1c/537e930496149fbac69efd2fc4329035bbe2e5475b4165439e3be9cb183b/pillow-11.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ee92f2fd10f4adc4b43d07ec5e779932b4eb3dbfbc34790ada5a6669bc095aa6", size = 5902860, upload-time = "2025-07-03T13:10:50.248Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/57/80f53264954dcefeebcf9dae6e3eb1daea1b488f0be8b8fef12f79a3eb10/pillow-11.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c96d333dcf42d01f47b37e0979b6bd73ec91eae18614864622d9b87bbd5bbf36", size = 7670694, upload-time = "2025-07-03T13:10:56.432Z" },
+    { url = "https://files.pythonhosted.org/packages/70/ff/4727d3b71a8578b4587d9c276e90efad2d6fe0335fd76742a6da08132e8c/pillow-11.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c96f993ab8c98460cd0c001447bff6194403e8b1d7e149ade5f00594918128b", size = 6005888, upload-time = "2025-07-01T09:15:39.436Z" },
+    { url = "https://files.pythonhosted.org/packages/05/ae/716592277934f85d3be51d7256f3636672d7b1abfafdc42cf3f8cbd4b4c8/pillow-11.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41342b64afeba938edb034d122b2dda5db2139b9a4af999729ba8818e0056477", size = 6670330, upload-time = "2025-07-01T09:15:41.269Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/bb/7fe6cddcc8827b01b1a9766f5fdeb7418680744f9082035bdbabecf1d57f/pillow-11.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:068d9c39a2d1b358eb9f245ce7ab1b5c3246c7c8c7d9ba58cfa5b43146c06e50", size = 6114089, upload-time = "2025-07-01T09:15:43.13Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/f5/06bfaa444c8e80f1a8e4bff98da9c83b37b5be3b1deaa43d27a0db37ef84/pillow-11.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a1bc6ba083b145187f648b667e05a2534ecc4b9f2784c2cbe3089e44868f2b9b", size = 6748206, upload-time = "2025-07-01T09:15:44.937Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/77/bc6f92a3e8e6e46c0ca78abfffec0037845800ea38c73483760362804c41/pillow-11.3.0-cp314-cp314t-win32.whl", hash = "sha256:118ca10c0d60b06d006be10a501fd6bbdfef559251ed31b794668ed569c87e12", size = 6377370, upload-time = "2025-07-01T09:15:46.673Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/82/3a721f7d69dca802befb8af08b7c79ebcab461007ce1c18bd91a5d5896f9/pillow-11.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8924748b688aa210d79883357d102cd64690e56b923a186f35a82cbc10f997db", size = 7121500, upload-time = "2025-07-01T09:15:48.512Z" },
+    { url = "https://files.pythonhosted.org/packages/89/c7/5572fa4a3f45740eaab6ae86fcdf7195b55beac1371ac8c619d880cfe948/pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa", size = 2512835, upload-time = "2025-07-01T09:15:50.399Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/8e/9c089f01677d1264ab8648352dcb7773f37da6ad002542760c80107da816/pillow-11.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:48d254f8a4c776de343051023eb61ffe818299eeac478da55227d96e241de53f", size = 5316478, upload-time = "2025-07-01T09:15:52.209Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/a9/5749930caf674695867eb56a581e78eb5f524b7583ff10b01b6e5048acb3/pillow-11.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7aee118e30a4cf54fdd873bd3a29de51e29105ab11f9aad8c32123f58c8f8081", size = 4686522, upload-time = "2025-07-01T09:15:54.162Z" },
+    { url = "https://files.pythonhosted.org/packages/43/46/0b85b763eb292b691030795f9f6bb6fcaf8948c39413c81696a01c3577f7/pillow-11.3.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:23cff760a9049c502721bdb743a7cb3e03365fafcdfc2ef9784610714166e5a4", size = 5853376, upload-time = "2025-07-03T13:11:01.066Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/c6/1a230ec0067243cbd60bc2dad5dc3ab46a8a41e21c15f5c9b52b26873069/pillow-11.3.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6359a3bc43f57d5b375d1ad54a0074318a0844d11b76abccf478c37c986d3cfc", size = 7626020, upload-time = "2025-07-03T13:11:06.479Z" },
+    { url = "https://files.pythonhosted.org/packages/63/dd/f296c27ffba447bfad76c6a0c44c1ea97a90cb9472b9304c94a732e8dbfb/pillow-11.3.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:092c80c76635f5ecb10f3f83d76716165c96f5229addbd1ec2bdbbda7d496e06", size = 5956732, upload-time = "2025-07-01T09:15:56.111Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/a0/98a3630f0b57f77bae67716562513d3032ae70414fcaf02750279c389a9e/pillow-11.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cadc9e0ea0a2431124cde7e1697106471fc4c1da01530e679b2391c37d3fbb3a", size = 6624404, upload-time = "2025-07-01T09:15:58.245Z" },
+    { url = "https://files.pythonhosted.org/packages/de/e6/83dfba5646a290edd9a21964da07674409e410579c341fc5b8f7abd81620/pillow-11.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6a418691000f2a418c9135a7cf0d797c1bb7d9a485e61fe8e7722845b95ef978", size = 6067760, upload-time = "2025-07-01T09:16:00.003Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/41/15ab268fe6ee9a2bc7391e2bbb20a98d3974304ab1a406a992dcb297a370/pillow-11.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:97afb3a00b65cc0804d1c7abddbf090a81eaac02768af58cbdcaaa0a931e0b6d", size = 6700534, upload-time = "2025-07-01T09:16:02.29Z" },
+    { url = "https://files.pythonhosted.org/packages/64/79/6d4f638b288300bed727ff29f2a3cb63db054b33518a95f27724915e3fbc/pillow-11.3.0-cp39-cp39-win32.whl", hash = "sha256:ea944117a7974ae78059fcc1800e5d3295172bb97035c0c1d9345fca1419da71", size = 6277091, upload-time = "2025-07-01T09:16:04.4Z" },
+    { url = "https://files.pythonhosted.org/packages/46/05/4106422f45a05716fd34ed21763f8ec182e8ea00af6e9cb05b93a247361a/pillow-11.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:e5c5858ad8ec655450a7c7df532e9842cf8df7cc349df7225c60d5d348c8aada", size = 6986091, upload-time = "2025-07-01T09:16:06.342Z" },
+    { url = "https://files.pythonhosted.org/packages/63/c6/287fd55c2c12761d0591549d48885187579b7c257bef0c6660755b0b59ae/pillow-11.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:6abdbfd3aea42be05702a8dd98832329c167ee84400a1d1f61ab11437f1717eb", size = 2422632, upload-time = "2025-07-01T09:16:08.142Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/8b/209bd6b62ce8367f47e68a218bffac88888fdf2c9fcf1ecadc6c3ec1ebc7/pillow-11.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3cee80663f29e3843b68199b9d6f4f54bd1d4a6b59bdd91bceefc51238bcb967", size = 5270556, upload-time = "2025-07-01T09:16:09.961Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/e6/231a0b76070c2cfd9e260a7a5b504fb72da0a95279410fa7afd99d9751d6/pillow-11.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b5f56c3f344f2ccaf0dd875d3e180f631dc60a51b314295a3e681fe8cf851fbe", size = 4654625, upload-time = "2025-07-01T09:16:11.913Z" },
+    { url = "https://files.pythonhosted.org/packages/13/f4/10cf94fda33cb12765f2397fc285fa6d8eb9c29de7f3185165b702fc7386/pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e67d793d180c9df62f1f40aee3accca4829d3794c95098887edc18af4b8b780c", size = 4874207, upload-time = "2025-07-03T13:11:10.201Z" },
+    { url = "https://files.pythonhosted.org/packages/72/c9/583821097dc691880c92892e8e2d41fe0a5a3d6021f4963371d2f6d57250/pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d000f46e2917c705e9fb93a3606ee4a819d1e3aa7a9b442f6444f07e77cf5e25", size = 6583939, upload-time = "2025-07-03T13:11:15.68Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/8e/5c9d410f9217b12320efc7c413e72693f48468979a013ad17fd690397b9a/pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:527b37216b6ac3a12d7838dc3bd75208ec57c1c6d11ef01902266a5a0c14fc27", size = 4957166, upload-time = "2025-07-01T09:16:13.74Z" },
+    { url = "https://files.pythonhosted.org/packages/62/bb/78347dbe13219991877ffb3a91bf09da8317fbfcd4b5f9140aeae020ad71/pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be5463ac478b623b9dd3937afd7fb7ab3d79dd290a28e2b6df292dc75063eb8a", size = 5581482, upload-time = "2025-07-01T09:16:16.107Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/28/1000353d5e61498aaeaaf7f1e4b49ddb05f2c6575f9d4f9f914a3538b6e1/pillow-11.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8dc70ca24c110503e16918a658b869019126ecfe03109b754c402daff12b3d9f", size = 6984596, upload-time = "2025-07-01T09:16:18.07Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/e3/6fa84033758276fb31da12e5fb66ad747ae83b93c67af17f8c6ff4cc8f34/pillow-11.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7c8ec7a017ad1bd562f93dbd8505763e688d388cde6e4a010ae1486916e713e6", size = 5270566, upload-time = "2025-07-01T09:16:19.801Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/ee/e8d2e1ab4892970b561e1ba96cbd59c0d28cf66737fc44abb2aec3795a4e/pillow-11.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9ab6ae226de48019caa8074894544af5b53a117ccb9d3b3dcb2871464c829438", size = 4654618, upload-time = "2025-07-01T09:16:21.818Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/6d/17f80f4e1f0761f02160fc433abd4109fa1548dcfdca46cfdadaf9efa565/pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe27fb049cdcca11f11a7bfda64043c37b30e6b91f10cb5bab275806c32f6ab3", size = 4874248, upload-time = "2025-07-03T13:11:20.738Z" },
+    { url = "https://files.pythonhosted.org/packages/de/5f/c22340acd61cef960130585bbe2120e2fd8434c214802f07e8c03596b17e/pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:465b9e8844e3c3519a983d58b80be3f668e2a7a5db97f2784e7079fbc9f9822c", size = 6583963, upload-time = "2025-07-03T13:11:26.283Z" },
+    { url = "https://files.pythonhosted.org/packages/31/5e/03966aedfbfcbb4d5f8aa042452d3361f325b963ebbadddac05b122e47dd/pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5418b53c0d59b3824d05e029669efa023bbef0f3e92e75ec8428f3799487f361", size = 4957170, upload-time = "2025-07-01T09:16:23.762Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/2d/e082982aacc927fc2cab48e1e731bdb1643a1406acace8bed0900a61464e/pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:504b6f59505f08ae014f724b6207ff6222662aab5cc9542577fb084ed0676ac7", size = 5581505, upload-time = "2025-07-01T09:16:25.593Z" },
+    { url = "https://files.pythonhosted.org/packages/34/e7/ae39f538fd6844e982063c3a5e4598b8ced43b9633baa3a85ef33af8c05c/pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8", size = 6984598, upload-time = "2025-07-01T09:16:27.732Z" },
 ]
 
 [[package]]
 name = "pluggy"
 version = "1.6.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412 }
+sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538 },
+    { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
 ]
 
 [[package]]
@@ -1750,159 +2110,198 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "polars-runtime-32" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/a1/3e/35fcf5bf51404371bb172b289a5065778dc97adca4416e199c294125eb05/polars-1.34.0.tar.gz", hash = "sha256:5de5f871027db4b11bcf39215a2d6b13b4a80baf8a55c5862d4ebedfd5cd4013", size = 684309 }
+sdist = { url = "https://files.pythonhosted.org/packages/a1/3e/35fcf5bf51404371bb172b289a5065778dc97adca4416e199c294125eb05/polars-1.34.0.tar.gz", hash = "sha256:5de5f871027db4b11bcf39215a2d6b13b4a80baf8a55c5862d4ebedfd5cd4013", size = 684309, upload-time = "2025-10-02T18:31:04.396Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/6b/80/1791ac226bb989bef30fe8fde752b2021b6ec5dfd6e880262596aedf4c05/polars-1.34.0-py3-none-any.whl", hash = "sha256:40d2f357b4d9e447ad28bd2c9923e4318791a7c18eb68f31f1fbf11180f41391", size = 772686 },
+    { url = "https://files.pythonhosted.org/packages/6b/80/1791ac226bb989bef30fe8fde752b2021b6ec5dfd6e880262596aedf4c05/polars-1.34.0-py3-none-any.whl", hash = "sha256:40d2f357b4d9e447ad28bd2c9923e4318791a7c18eb68f31f1fbf11180f41391", size = 772686, upload-time = "2025-10-02T18:29:59.492Z" },
 ]
 
 [package.optional-dependencies]
 pandas = [
     { name = "pandas" },
-    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" },
-    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" },
+    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
 ]
 pyarrow = [
-    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" },
-    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" },
+    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
 ]
 
 [[package]]
 name = "polars-runtime-32"
 version = "1.34.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/02/10/1189afb14cc47ed215ccf7fbd00ed21c48edfd89e51c16f8628a33ae4b1b/polars_runtime_32-1.34.0.tar.gz", hash = "sha256:ebe6f865128a0d833f53a3f6828360761ad86d1698bceb22bef9fd999500dc1c", size = 2634491 }
+sdist = { url = "https://files.pythonhosted.org/packages/02/10/1189afb14cc47ed215ccf7fbd00ed21c48edfd89e51c16f8628a33ae4b1b/polars_runtime_32-1.34.0.tar.gz", hash = "sha256:ebe6f865128a0d833f53a3f6828360761ad86d1698bceb22bef9fd999500dc1c", size = 2634491, upload-time = "2025-10-02T18:31:05.502Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/97/35/bc4f1a9dcef61845e8e4e5d2318470b002b93a3564026f0643f562761ecb/polars_runtime_32-1.34.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2878f9951e91121afe60c25433ef270b9a221e6ebf3de5f6642346b38cab3f03", size = 39655423 },
-    { url = "https://files.pythonhosted.org/packages/a6/bb/d655a103e75b7c81c47a3c2d276be0200c0c15cfb6fd47f17932ddcf7519/polars_runtime_32-1.34.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:fbc329c7d34a924228cc5dcdbbd4696d94411a3a5b15ad8bb868634c204e1951", size = 35986049 },
-    { url = "https://files.pythonhosted.org/packages/9e/ce/11ca850b7862cb43605e5d86cdf655614376e0a059871cf8305af5406554/polars_runtime_32-1.34.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93fa51d88a2d12ea996a5747aad5647d22a86cce73c80f208e61f487b10bc448", size = 40261269 },
-    { url = "https://files.pythonhosted.org/packages/d8/25/77d12018c35489e19f7650b40679714a834effafc25d61e8dcee7c4fafce/polars_runtime_32-1.34.0-cp39-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:79e4d696392c6d8d51f4347f0b167c52eef303c9d87093c0c68e8651198735b7", size = 37049077 },
-    { url = "https://files.pythonhosted.org/packages/e2/75/c30049d45ea1365151f86f650ed5354124ff3209f0abe588664c8eb13a31/polars_runtime_32-1.34.0-cp39-abi3-win_amd64.whl", hash = "sha256:2501d6b29d9001ea5ea2fd9b598787e10ddf45d8c4a87c2bead75159e8a15711", size = 40105782 },
-    { url = "https://files.pythonhosted.org/packages/a3/31/84efa27aa3478c8670bac1a720c8b1aee5c58c9c657c980e5e5c47fde883/polars_runtime_32-1.34.0-cp39-abi3-win_arm64.whl", hash = "sha256:f9ed1765378dfe0bcd1ac5ec570dd9eab27ea728bbc980cc9a76eebc55586559", size = 35873216 },
+    { url = "https://files.pythonhosted.org/packages/97/35/bc4f1a9dcef61845e8e4e5d2318470b002b93a3564026f0643f562761ecb/polars_runtime_32-1.34.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2878f9951e91121afe60c25433ef270b9a221e6ebf3de5f6642346b38cab3f03", size = 39655423, upload-time = "2025-10-02T18:30:02.846Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/bb/d655a103e75b7c81c47a3c2d276be0200c0c15cfb6fd47f17932ddcf7519/polars_runtime_32-1.34.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:fbc329c7d34a924228cc5dcdbbd4696d94411a3a5b15ad8bb868634c204e1951", size = 35986049, upload-time = "2025-10-02T18:30:05.848Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/ce/11ca850b7862cb43605e5d86cdf655614376e0a059871cf8305af5406554/polars_runtime_32-1.34.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93fa51d88a2d12ea996a5747aad5647d22a86cce73c80f208e61f487b10bc448", size = 40261269, upload-time = "2025-10-02T18:30:08.48Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/25/77d12018c35489e19f7650b40679714a834effafc25d61e8dcee7c4fafce/polars_runtime_32-1.34.0-cp39-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:79e4d696392c6d8d51f4347f0b167c52eef303c9d87093c0c68e8651198735b7", size = 37049077, upload-time = "2025-10-02T18:30:11.162Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/75/c30049d45ea1365151f86f650ed5354124ff3209f0abe588664c8eb13a31/polars_runtime_32-1.34.0-cp39-abi3-win_amd64.whl", hash = "sha256:2501d6b29d9001ea5ea2fd9b598787e10ddf45d8c4a87c2bead75159e8a15711", size = 40105782, upload-time = "2025-10-02T18:30:14.597Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/31/84efa27aa3478c8670bac1a720c8b1aee5c58c9c657c980e5e5c47fde883/polars_runtime_32-1.34.0-cp39-abi3-win_arm64.whl", hash = "sha256:f9ed1765378dfe0bcd1ac5ec570dd9eab27ea728bbc980cc9a76eebc55586559", size = 35873216, upload-time = "2025-10-02T18:30:17.439Z" },
 ]
 
 [[package]]
 name = "propcache"
 version = "0.3.2"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a6/16/43264e4a779dd8588c21a70f0709665ee8f611211bdd2c87d952cfa7c776/propcache-0.3.2.tar.gz", hash = "sha256:20d7d62e4e7ef05f221e0db2856b979540686342e7dd9973b815599c7057e168", size = 44139 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ab/14/510deed325e262afeb8b360043c5d7c960da7d3ecd6d6f9496c9c56dc7f4/propcache-0.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:22d9962a358aedbb7a2e36187ff273adeaab9743373a272976d2e348d08c7770", size = 73178 },
-    { url = "https://files.pythonhosted.org/packages/cd/4e/ad52a7925ff01c1325653a730c7ec3175a23f948f08626a534133427dcff/propcache-0.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0d0fda578d1dc3f77b6b5a5dce3b9ad69a8250a891760a548df850a5e8da87f3", size = 43133 },
-    { url = "https://files.pythonhosted.org/packages/63/7c/e9399ba5da7780871db4eac178e9c2e204c23dd3e7d32df202092a1ed400/propcache-0.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3def3da3ac3ce41562d85db655d18ebac740cb3fa4367f11a52b3da9d03a5cc3", size = 43039 },
-    { url = "https://files.pythonhosted.org/packages/22/e1/58da211eb8fdc6fc854002387d38f415a6ca5f5c67c1315b204a5d3e9d7a/propcache-0.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9bec58347a5a6cebf239daba9bda37dffec5b8d2ce004d9fe4edef3d2815137e", size = 201903 },
-    { url = "https://files.pythonhosted.org/packages/c4/0a/550ea0f52aac455cb90111c8bab995208443e46d925e51e2f6ebdf869525/propcache-0.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55ffda449a507e9fbd4aca1a7d9aa6753b07d6166140e5a18d2ac9bc49eac220", size = 213362 },
-    { url = "https://files.pythonhosted.org/packages/5a/af/9893b7d878deda9bb69fcf54600b247fba7317761b7db11fede6e0f28bd0/propcache-0.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64a67fb39229a8a8491dd42f864e5e263155e729c2e7ff723d6e25f596b1e8cb", size = 210525 },
-    { url = "https://files.pythonhosted.org/packages/7c/bb/38fd08b278ca85cde36d848091ad2b45954bc5f15cce494bb300b9285831/propcache-0.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9da1cf97b92b51253d5b68cf5a2b9e0dafca095e36b7f2da335e27dc6172a614", size = 198283 },
-    { url = "https://files.pythonhosted.org/packages/78/8c/9fe55bd01d362bafb413dfe508c48753111a1e269737fa143ba85693592c/propcache-0.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f559e127134b07425134b4065be45b166183fdcb433cb6c24c8e4149056ad50", size = 191872 },
-    { url = "https://files.pythonhosted.org/packages/54/14/4701c33852937a22584e08abb531d654c8bcf7948a8f87ad0a4822394147/propcache-0.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:aff2e4e06435d61f11a428360a932138d0ec288b0a31dd9bd78d200bd4a2b339", size = 199452 },
-    { url = "https://files.pythonhosted.org/packages/16/44/447f2253d859602095356007657ee535e0093215ea0b3d1d6a41d16e5201/propcache-0.3.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4927842833830942a5d0a56e6f4839bc484785b8e1ce8d287359794818633ba0", size = 191567 },
-    { url = "https://files.pythonhosted.org/packages/f2/b3/e4756258749bb2d3b46defcff606a2f47410bab82be5824a67e84015b267/propcache-0.3.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6107ddd08b02654a30fb8ad7a132021759d750a82578b94cd55ee2772b6ebea2", size = 193015 },
-    { url = "https://files.pythonhosted.org/packages/1e/df/e6d3c7574233164b6330b9fd697beeac402afd367280e6dc377bb99b43d9/propcache-0.3.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:70bd8b9cd6b519e12859c99f3fc9a93f375ebd22a50296c3a295028bea73b9e7", size = 204660 },
-    { url = "https://files.pythonhosted.org/packages/b2/53/e4d31dd5170b4a0e2e6b730f2385a96410633b4833dc25fe5dffd1f73294/propcache-0.3.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2183111651d710d3097338dd1893fcf09c9f54e27ff1a8795495a16a469cc90b", size = 206105 },
-    { url = "https://files.pythonhosted.org/packages/7f/fe/74d54cf9fbe2a20ff786e5f7afcfde446588f0cf15fb2daacfbc267b866c/propcache-0.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fb075ad271405dcad8e2a7ffc9a750a3bf70e533bd86e89f0603e607b93aa64c", size = 196980 },
-    { url = "https://files.pythonhosted.org/packages/22/ec/c469c9d59dada8a7679625e0440b544fe72e99311a4679c279562051f6fc/propcache-0.3.2-cp310-cp310-win32.whl", hash = "sha256:404d70768080d3d3bdb41d0771037da19d8340d50b08e104ca0e7f9ce55fce70", size = 37679 },
-    { url = "https://files.pythonhosted.org/packages/38/35/07a471371ac89d418f8d0b699c75ea6dca2041fbda360823de21f6a9ce0a/propcache-0.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:7435d766f978b4ede777002e6b3b6641dd229cd1da8d3d3106a45770365f9ad9", size = 41459 },
-    { url = "https://files.pythonhosted.org/packages/80/8d/e8b436717ab9c2cfc23b116d2c297305aa4cd8339172a456d61ebf5669b8/propcache-0.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0b8d2f607bd8f80ddc04088bc2a037fdd17884a6fcadc47a96e334d72f3717be", size = 74207 },
-    { url = "https://files.pythonhosted.org/packages/d6/29/1e34000e9766d112171764b9fa3226fa0153ab565d0c242c70e9945318a7/propcache-0.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06766d8f34733416e2e34f46fea488ad5d60726bb9481d3cddf89a6fa2d9603f", size = 43648 },
-    { url = "https://files.pythonhosted.org/packages/46/92/1ad5af0df781e76988897da39b5f086c2bf0f028b7f9bd1f409bb05b6874/propcache-0.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2dc1f4a1df4fecf4e6f68013575ff4af84ef6f478fe5344317a65d38a8e6dc9", size = 43496 },
-    { url = "https://files.pythonhosted.org/packages/b3/ce/e96392460f9fb68461fabab3e095cb00c8ddf901205be4eae5ce246e5b7e/propcache-0.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be29c4f4810c5789cf10ddf6af80b041c724e629fa51e308a7a0fb19ed1ef7bf", size = 217288 },
-    { url = "https://files.pythonhosted.org/packages/c5/2a/866726ea345299f7ceefc861a5e782b045545ae6940851930a6adaf1fca6/propcache-0.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59d61f6970ecbd8ff2e9360304d5c8876a6abd4530cb752c06586849ac8a9dc9", size = 227456 },
-    { url = "https://files.pythonhosted.org/packages/de/03/07d992ccb6d930398689187e1b3c718339a1c06b8b145a8d9650e4726166/propcache-0.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62180e0b8dbb6b004baec00a7983e4cc52f5ada9cd11f48c3528d8cfa7b96a66", size = 225429 },
-    { url = "https://files.pythonhosted.org/packages/5d/e6/116ba39448753b1330f48ab8ba927dcd6cf0baea8a0ccbc512dfb49ba670/propcache-0.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c144ca294a204c470f18cf4c9d78887810d04a3e2fbb30eea903575a779159df", size = 213472 },
-    { url = "https://files.pythonhosted.org/packages/a6/85/f01f5d97e54e428885a5497ccf7f54404cbb4f906688a1690cd51bf597dc/propcache-0.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5c2a784234c28854878d68978265617aa6dc0780e53d44b4d67f3651a17a9a2", size = 204480 },
-    { url = "https://files.pythonhosted.org/packages/e3/79/7bf5ab9033b8b8194cc3f7cf1aaa0e9c3256320726f64a3e1f113a812dce/propcache-0.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5745bc7acdafa978ca1642891b82c19238eadc78ba2aaa293c6863b304e552d7", size = 214530 },
-    { url = "https://files.pythonhosted.org/packages/31/0b/bd3e0c00509b609317df4a18e6b05a450ef2d9a963e1d8bc9c9415d86f30/propcache-0.3.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:c0075bf773d66fa8c9d41f66cc132ecc75e5bb9dd7cce3cfd14adc5ca184cb95", size = 205230 },
-    { url = "https://files.pythonhosted.org/packages/7a/23/fae0ff9b54b0de4e819bbe559508da132d5683c32d84d0dc2ccce3563ed4/propcache-0.3.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5f57aa0847730daceff0497f417c9de353c575d8da3579162cc74ac294c5369e", size = 206754 },
-    { url = "https://files.pythonhosted.org/packages/b7/7f/ad6a3c22630aaa5f618b4dc3c3598974a72abb4c18e45a50b3cdd091eb2f/propcache-0.3.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:eef914c014bf72d18efb55619447e0aecd5fb7c2e3fa7441e2e5d6099bddff7e", size = 218430 },
-    { url = "https://files.pythonhosted.org/packages/5b/2c/ba4f1c0e8a4b4c75910742f0d333759d441f65a1c7f34683b4a74c0ee015/propcache-0.3.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2a4092e8549031e82facf3decdbc0883755d5bbcc62d3aea9d9e185549936dcf", size = 223884 },
-    { url = "https://files.pythonhosted.org/packages/88/e4/ebe30fc399e98572019eee82ad0caf512401661985cbd3da5e3140ffa1b0/propcache-0.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:85871b050f174bc0bfb437efbdb68aaf860611953ed12418e4361bc9c392749e", size = 211480 },
-    { url = "https://files.pythonhosted.org/packages/96/0a/7d5260b914e01d1d0906f7f38af101f8d8ed0dc47426219eeaf05e8ea7c2/propcache-0.3.2-cp311-cp311-win32.whl", hash = "sha256:36c8d9b673ec57900c3554264e630d45980fd302458e4ac801802a7fd2ef7897", size = 37757 },
-    { url = "https://files.pythonhosted.org/packages/e1/2d/89fe4489a884bc0da0c3278c552bd4ffe06a1ace559db5ef02ef24ab446b/propcache-0.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53af8cb6a781b02d2ea079b5b853ba9430fcbe18a8e3ce647d5982a3ff69f39", size = 41500 },
-    { url = "https://files.pythonhosted.org/packages/a8/42/9ca01b0a6f48e81615dca4765a8f1dd2c057e0540f6116a27dc5ee01dfb6/propcache-0.3.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8de106b6c84506b31c27168582cd3cb3000a6412c16df14a8628e5871ff83c10", size = 73674 },
-    { url = "https://files.pythonhosted.org/packages/af/6e/21293133beb550f9c901bbece755d582bfaf2176bee4774000bd4dd41884/propcache-0.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:28710b0d3975117239c76600ea351934ac7b5ff56e60953474342608dbbb6154", size = 43570 },
-    { url = "https://files.pythonhosted.org/packages/0c/c8/0393a0a3a2b8760eb3bde3c147f62b20044f0ddac81e9d6ed7318ec0d852/propcache-0.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce26862344bdf836650ed2487c3d724b00fbfec4233a1013f597b78c1cb73615", size = 43094 },
-    { url = "https://files.pythonhosted.org/packages/37/2c/489afe311a690399d04a3e03b069225670c1d489eb7b044a566511c1c498/propcache-0.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bca54bd347a253af2cf4544bbec232ab982f4868de0dd684246b67a51bc6b1db", size = 226958 },
-    { url = "https://files.pythonhosted.org/packages/9d/ca/63b520d2f3d418c968bf596839ae26cf7f87bead026b6192d4da6a08c467/propcache-0.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55780d5e9a2ddc59711d727226bb1ba83a22dd32f64ee15594b9392b1f544eb1", size = 234894 },
-    { url = "https://files.pythonhosted.org/packages/11/60/1d0ed6fff455a028d678df30cc28dcee7af77fa2b0e6962ce1df95c9a2a9/propcache-0.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:035e631be25d6975ed87ab23153db6a73426a48db688070d925aa27e996fe93c", size = 233672 },
-    { url = "https://files.pythonhosted.org/packages/37/7c/54fd5301ef38505ab235d98827207176a5c9b2aa61939b10a460ca53e123/propcache-0.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee6f22b6eaa39297c751d0e80c0d3a454f112f5c6481214fcf4c092074cecd67", size = 224395 },
-    { url = "https://files.pythonhosted.org/packages/ee/1a/89a40e0846f5de05fdc6779883bf46ba980e6df4d2ff8fb02643de126592/propcache-0.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ca3aee1aa955438c4dba34fc20a9f390e4c79967257d830f137bd5a8a32ed3b", size = 212510 },
-    { url = "https://files.pythonhosted.org/packages/5e/33/ca98368586c9566a6b8d5ef66e30484f8da84c0aac3f2d9aec6d31a11bd5/propcache-0.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7a4f30862869fa2b68380d677cc1c5fcf1e0f2b9ea0cf665812895c75d0ca3b8", size = 222949 },
-    { url = "https://files.pythonhosted.org/packages/ba/11/ace870d0aafe443b33b2f0b7efdb872b7c3abd505bfb4890716ad7865e9d/propcache-0.3.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b77ec3c257d7816d9f3700013639db7491a434644c906a2578a11daf13176251", size = 217258 },
-    { url = "https://files.pythonhosted.org/packages/5b/d2/86fd6f7adffcfc74b42c10a6b7db721d1d9ca1055c45d39a1a8f2a740a21/propcache-0.3.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cab90ac9d3f14b2d5050928483d3d3b8fb6b4018893fc75710e6aa361ecb2474", size = 213036 },
-    { url = "https://files.pythonhosted.org/packages/07/94/2d7d1e328f45ff34a0a284cf5a2847013701e24c2a53117e7c280a4316b3/propcache-0.3.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0b504d29f3c47cf6b9e936c1852246c83d450e8e063d50562115a6be6d3a2535", size = 227684 },
-    { url = "https://files.pythonhosted.org/packages/b7/05/37ae63a0087677e90b1d14710e532ff104d44bc1efa3b3970fff99b891dc/propcache-0.3.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ce2ac2675a6aa41ddb2a0c9cbff53780a617ac3d43e620f8fd77ba1c84dcfc06", size = 234562 },
-    { url = "https://files.pythonhosted.org/packages/a4/7c/3f539fcae630408d0bd8bf3208b9a647ccad10976eda62402a80adf8fc34/propcache-0.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b4239611205294cc433845b914131b2a1f03500ff3c1ed093ed216b82621e1", size = 222142 },
-    { url = "https://files.pythonhosted.org/packages/7c/d2/34b9eac8c35f79f8a962546b3e97e9d4b990c420ee66ac8255d5d9611648/propcache-0.3.2-cp312-cp312-win32.whl", hash = "sha256:df4a81b9b53449ebc90cc4deefb052c1dd934ba85012aa912c7ea7b7e38b60c1", size = 37711 },
-    { url = "https://files.pythonhosted.org/packages/19/61/d582be5d226cf79071681d1b46b848d6cb03d7b70af7063e33a2787eaa03/propcache-0.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7046e79b989d7fe457bb755844019e10f693752d169076138abf17f31380800c", size = 41479 },
-    { url = "https://files.pythonhosted.org/packages/dc/d1/8c747fafa558c603c4ca19d8e20b288aa0c7cda74e9402f50f31eb65267e/propcache-0.3.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ca592ed634a73ca002967458187109265e980422116c0a107cf93d81f95af945", size = 71286 },
-    { url = "https://files.pythonhosted.org/packages/61/99/d606cb7986b60d89c36de8a85d58764323b3a5ff07770a99d8e993b3fa73/propcache-0.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9ecb0aad4020e275652ba3975740f241bd12a61f1a784df044cf7477a02bc252", size = 42425 },
-    { url = "https://files.pythonhosted.org/packages/8c/96/ef98f91bbb42b79e9bb82bdd348b255eb9d65f14dbbe3b1594644c4073f7/propcache-0.3.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7f08f1cc28bd2eade7a8a3d2954ccc673bb02062e3e7da09bc75d843386b342f", size = 41846 },
-    { url = "https://files.pythonhosted.org/packages/5b/ad/3f0f9a705fb630d175146cd7b1d2bf5555c9beaed54e94132b21aac098a6/propcache-0.3.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1a342c834734edb4be5ecb1e9fb48cb64b1e2320fccbd8c54bf8da8f2a84c33", size = 208871 },
-    { url = "https://files.pythonhosted.org/packages/3a/38/2085cda93d2c8b6ec3e92af2c89489a36a5886b712a34ab25de9fbca7992/propcache-0.3.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8a544caaae1ac73f1fecfae70ded3e93728831affebd017d53449e3ac052ac1e", size = 215720 },
-    { url = "https://files.pythonhosted.org/packages/61/c1/d72ea2dc83ac7f2c8e182786ab0fc2c7bd123a1ff9b7975bee671866fe5f/propcache-0.3.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:310d11aa44635298397db47a3ebce7db99a4cc4b9bbdfcf6c98a60c8d5261cf1", size = 215203 },
-    { url = "https://files.pythonhosted.org/packages/af/81/b324c44ae60c56ef12007105f1460d5c304b0626ab0cc6b07c8f2a9aa0b8/propcache-0.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c1396592321ac83157ac03a2023aa6cc4a3cc3cfdecb71090054c09e5a7cce3", size = 206365 },
-    { url = "https://files.pythonhosted.org/packages/09/73/88549128bb89e66d2aff242488f62869014ae092db63ccea53c1cc75a81d/propcache-0.3.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cabf5b5902272565e78197edb682017d21cf3b550ba0460ee473753f28d23c1", size = 196016 },
-    { url = "https://files.pythonhosted.org/packages/b9/3f/3bdd14e737d145114a5eb83cb172903afba7242f67c5877f9909a20d948d/propcache-0.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0a2f2235ac46a7aa25bdeb03a9e7060f6ecbd213b1f9101c43b3090ffb971ef6", size = 205596 },
-    { url = "https://files.pythonhosted.org/packages/0f/ca/2f4aa819c357d3107c3763d7ef42c03980f9ed5c48c82e01e25945d437c1/propcache-0.3.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:92b69e12e34869a6970fd2f3da91669899994b47c98f5d430b781c26f1d9f387", size = 200977 },
-    { url = "https://files.pythonhosted.org/packages/cd/4a/e65276c7477533c59085251ae88505caf6831c0e85ff8b2e31ebcbb949b1/propcache-0.3.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:54e02207c79968ebbdffc169591009f4474dde3b4679e16634d34c9363ff56b4", size = 197220 },
-    { url = "https://files.pythonhosted.org/packages/7c/54/fc7152e517cf5578278b242396ce4d4b36795423988ef39bb8cd5bf274c8/propcache-0.3.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4adfb44cb588001f68c5466579d3f1157ca07f7504fc91ec87862e2b8e556b88", size = 210642 },
-    { url = "https://files.pythonhosted.org/packages/b9/80/abeb4a896d2767bf5f1ea7b92eb7be6a5330645bd7fb844049c0e4045d9d/propcache-0.3.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fd3e6019dc1261cd0291ee8919dd91fbab7b169bb76aeef6c716833a3f65d206", size = 212789 },
-    { url = "https://files.pythonhosted.org/packages/b3/db/ea12a49aa7b2b6d68a5da8293dcf50068d48d088100ac016ad92a6a780e6/propcache-0.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4c181cad81158d71c41a2bce88edce078458e2dd5ffee7eddd6b05da85079f43", size = 205880 },
-    { url = "https://files.pythonhosted.org/packages/d1/e5/9076a0bbbfb65d1198007059c65639dfd56266cf8e477a9707e4b1999ff4/propcache-0.3.2-cp313-cp313-win32.whl", hash = "sha256:8a08154613f2249519e549de2330cf8e2071c2887309a7b07fb56098f5170a02", size = 37220 },
-    { url = "https://files.pythonhosted.org/packages/d3/f5/b369e026b09a26cd77aa88d8fffd69141d2ae00a2abaaf5380d2603f4b7f/propcache-0.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:e41671f1594fc4ab0a6dec1351864713cb3a279910ae8b58f884a88a0a632c05", size = 40678 },
-    { url = "https://files.pythonhosted.org/packages/a4/3a/6ece377b55544941a08d03581c7bc400a3c8cd3c2865900a68d5de79e21f/propcache-0.3.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:9a3cf035bbaf035f109987d9d55dc90e4b0e36e04bbbb95af3055ef17194057b", size = 76560 },
-    { url = "https://files.pythonhosted.org/packages/0c/da/64a2bb16418740fa634b0e9c3d29edff1db07f56d3546ca2d86ddf0305e1/propcache-0.3.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:156c03d07dc1323d8dacaa221fbe028c5c70d16709cdd63502778e6c3ccca1b0", size = 44676 },
-    { url = "https://files.pythonhosted.org/packages/36/7b/f025e06ea51cb72c52fb87e9b395cced02786610b60a3ed51da8af017170/propcache-0.3.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74413c0ba02ba86f55cf60d18daab219f7e531620c15f1e23d95563f505efe7e", size = 44701 },
-    { url = "https://files.pythonhosted.org/packages/a4/00/faa1b1b7c3b74fc277f8642f32a4c72ba1d7b2de36d7cdfb676db7f4303e/propcache-0.3.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f066b437bb3fa39c58ff97ab2ca351db465157d68ed0440abecb21715eb24b28", size = 276934 },
-    { url = "https://files.pythonhosted.org/packages/74/ab/935beb6f1756e0476a4d5938ff44bf0d13a055fed880caf93859b4f1baf4/propcache-0.3.2-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1304b085c83067914721e7e9d9917d41ad87696bf70f0bc7dee450e9c71ad0a", size = 278316 },
-    { url = "https://files.pythonhosted.org/packages/f8/9d/994a5c1ce4389610838d1caec74bdf0e98b306c70314d46dbe4fcf21a3e2/propcache-0.3.2-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab50cef01b372763a13333b4e54021bdcb291fc9a8e2ccb9c2df98be51bcde6c", size = 282619 },
-    { url = "https://files.pythonhosted.org/packages/2b/00/a10afce3d1ed0287cef2e09506d3be9822513f2c1e96457ee369adb9a6cd/propcache-0.3.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fad3b2a085ec259ad2c2842666b2a0a49dea8463579c606426128925af1ed725", size = 265896 },
-    { url = "https://files.pythonhosted.org/packages/2e/a8/2aa6716ffa566ca57c749edb909ad27884680887d68517e4be41b02299f3/propcache-0.3.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:261fa020c1c14deafd54c76b014956e2f86991af198c51139faf41c4d5e83892", size = 252111 },
-    { url = "https://files.pythonhosted.org/packages/36/4f/345ca9183b85ac29c8694b0941f7484bf419c7f0fea2d1e386b4f7893eed/propcache-0.3.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:46d7f8aa79c927e5f987ee3a80205c987717d3659f035c85cf0c3680526bdb44", size = 268334 },
-    { url = "https://files.pythonhosted.org/packages/3e/ca/fcd54f78b59e3f97b3b9715501e3147f5340167733d27db423aa321e7148/propcache-0.3.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:6d8f3f0eebf73e3c0ff0e7853f68be638b4043c65a70517bb575eff54edd8dbe", size = 255026 },
-    { url = "https://files.pythonhosted.org/packages/8b/95/8e6a6bbbd78ac89c30c225210a5c687790e532ba4088afb8c0445b77ef37/propcache-0.3.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:03c89c1b14a5452cf15403e291c0ccd7751d5b9736ecb2c5bab977ad6c5bcd81", size = 250724 },
-    { url = "https://files.pythonhosted.org/packages/ee/b0/0dd03616142baba28e8b2d14ce5df6631b4673850a3d4f9c0f9dd714a404/propcache-0.3.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:0cc17efde71e12bbaad086d679ce575268d70bc123a5a71ea7ad76f70ba30bba", size = 268868 },
-    { url = "https://files.pythonhosted.org/packages/c5/98/2c12407a7e4fbacd94ddd32f3b1e3d5231e77c30ef7162b12a60e2dd5ce3/propcache-0.3.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:acdf05d00696bc0447e278bb53cb04ca72354e562cf88ea6f9107df8e7fd9770", size = 271322 },
-    { url = "https://files.pythonhosted.org/packages/35/91/9cb56efbb428b006bb85db28591e40b7736847b8331d43fe335acf95f6c8/propcache-0.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4445542398bd0b5d32df908031cb1b30d43ac848e20470a878b770ec2dcc6330", size = 265778 },
-    { url = "https://files.pythonhosted.org/packages/9a/4c/b0fe775a2bdd01e176b14b574be679d84fc83958335790f7c9a686c1f468/propcache-0.3.2-cp313-cp313t-win32.whl", hash = "sha256:f86e5d7cd03afb3a1db8e9f9f6eff15794e79e791350ac48a8c924e6f439f394", size = 41175 },
-    { url = "https://files.pythonhosted.org/packages/a4/ff/47f08595e3d9b5e149c150f88d9714574f1a7cbd89fe2817158a952674bf/propcache-0.3.2-cp313-cp313t-win_amd64.whl", hash = "sha256:9704bedf6e7cbe3c65eca4379a9b53ee6a83749f047808cbb5044d40d7d72198", size = 44857 },
-    { url = "https://files.pythonhosted.org/packages/cc/35/cc0aaecf278bb4575b8555f2b137de5ab821595ddae9da9d3cd1da4072c7/propcache-0.3.2-py3-none-any.whl", hash = "sha256:98f1ec44fb675f5052cccc8e609c46ed23a35a1cfd18545ad4e29002d858a43f", size = 12663 },
+sdist = { url = "https://files.pythonhosted.org/packages/a6/16/43264e4a779dd8588c21a70f0709665ee8f611211bdd2c87d952cfa7c776/propcache-0.3.2.tar.gz", hash = "sha256:20d7d62e4e7ef05f221e0db2856b979540686342e7dd9973b815599c7057e168", size = 44139, upload-time = "2025-06-09T22:56:06.081Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ab/14/510deed325e262afeb8b360043c5d7c960da7d3ecd6d6f9496c9c56dc7f4/propcache-0.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:22d9962a358aedbb7a2e36187ff273adeaab9743373a272976d2e348d08c7770", size = 73178, upload-time = "2025-06-09T22:53:40.126Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/4e/ad52a7925ff01c1325653a730c7ec3175a23f948f08626a534133427dcff/propcache-0.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0d0fda578d1dc3f77b6b5a5dce3b9ad69a8250a891760a548df850a5e8da87f3", size = 43133, upload-time = "2025-06-09T22:53:41.965Z" },
+    { url = "https://files.pythonhosted.org/packages/63/7c/e9399ba5da7780871db4eac178e9c2e204c23dd3e7d32df202092a1ed400/propcache-0.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3def3da3ac3ce41562d85db655d18ebac740cb3fa4367f11a52b3da9d03a5cc3", size = 43039, upload-time = "2025-06-09T22:53:43.268Z" },
+    { url = "https://files.pythonhosted.org/packages/22/e1/58da211eb8fdc6fc854002387d38f415a6ca5f5c67c1315b204a5d3e9d7a/propcache-0.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9bec58347a5a6cebf239daba9bda37dffec5b8d2ce004d9fe4edef3d2815137e", size = 201903, upload-time = "2025-06-09T22:53:44.872Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/0a/550ea0f52aac455cb90111c8bab995208443e46d925e51e2f6ebdf869525/propcache-0.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55ffda449a507e9fbd4aca1a7d9aa6753b07d6166140e5a18d2ac9bc49eac220", size = 213362, upload-time = "2025-06-09T22:53:46.707Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/af/9893b7d878deda9bb69fcf54600b247fba7317761b7db11fede6e0f28bd0/propcache-0.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64a67fb39229a8a8491dd42f864e5e263155e729c2e7ff723d6e25f596b1e8cb", size = 210525, upload-time = "2025-06-09T22:53:48.547Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/bb/38fd08b278ca85cde36d848091ad2b45954bc5f15cce494bb300b9285831/propcache-0.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9da1cf97b92b51253d5b68cf5a2b9e0dafca095e36b7f2da335e27dc6172a614", size = 198283, upload-time = "2025-06-09T22:53:50.067Z" },
+    { url = "https://files.pythonhosted.org/packages/78/8c/9fe55bd01d362bafb413dfe508c48753111a1e269737fa143ba85693592c/propcache-0.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f559e127134b07425134b4065be45b166183fdcb433cb6c24c8e4149056ad50", size = 191872, upload-time = "2025-06-09T22:53:51.438Z" },
+    { url = "https://files.pythonhosted.org/packages/54/14/4701c33852937a22584e08abb531d654c8bcf7948a8f87ad0a4822394147/propcache-0.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:aff2e4e06435d61f11a428360a932138d0ec288b0a31dd9bd78d200bd4a2b339", size = 199452, upload-time = "2025-06-09T22:53:53.229Z" },
+    { url = "https://files.pythonhosted.org/packages/16/44/447f2253d859602095356007657ee535e0093215ea0b3d1d6a41d16e5201/propcache-0.3.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4927842833830942a5d0a56e6f4839bc484785b8e1ce8d287359794818633ba0", size = 191567, upload-time = "2025-06-09T22:53:54.541Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/b3/e4756258749bb2d3b46defcff606a2f47410bab82be5824a67e84015b267/propcache-0.3.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6107ddd08b02654a30fb8ad7a132021759d750a82578b94cd55ee2772b6ebea2", size = 193015, upload-time = "2025-06-09T22:53:56.44Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/df/e6d3c7574233164b6330b9fd697beeac402afd367280e6dc377bb99b43d9/propcache-0.3.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:70bd8b9cd6b519e12859c99f3fc9a93f375ebd22a50296c3a295028bea73b9e7", size = 204660, upload-time = "2025-06-09T22:53:57.839Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/53/e4d31dd5170b4a0e2e6b730f2385a96410633b4833dc25fe5dffd1f73294/propcache-0.3.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2183111651d710d3097338dd1893fcf09c9f54e27ff1a8795495a16a469cc90b", size = 206105, upload-time = "2025-06-09T22:53:59.638Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/fe/74d54cf9fbe2a20ff786e5f7afcfde446588f0cf15fb2daacfbc267b866c/propcache-0.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fb075ad271405dcad8e2a7ffc9a750a3bf70e533bd86e89f0603e607b93aa64c", size = 196980, upload-time = "2025-06-09T22:54:01.071Z" },
+    { url = "https://files.pythonhosted.org/packages/22/ec/c469c9d59dada8a7679625e0440b544fe72e99311a4679c279562051f6fc/propcache-0.3.2-cp310-cp310-win32.whl", hash = "sha256:404d70768080d3d3bdb41d0771037da19d8340d50b08e104ca0e7f9ce55fce70", size = 37679, upload-time = "2025-06-09T22:54:03.003Z" },
+    { url = "https://files.pythonhosted.org/packages/38/35/07a471371ac89d418f8d0b699c75ea6dca2041fbda360823de21f6a9ce0a/propcache-0.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:7435d766f978b4ede777002e6b3b6641dd229cd1da8d3d3106a45770365f9ad9", size = 41459, upload-time = "2025-06-09T22:54:04.134Z" },
+    { url = "https://files.pythonhosted.org/packages/80/8d/e8b436717ab9c2cfc23b116d2c297305aa4cd8339172a456d61ebf5669b8/propcache-0.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0b8d2f607bd8f80ddc04088bc2a037fdd17884a6fcadc47a96e334d72f3717be", size = 74207, upload-time = "2025-06-09T22:54:05.399Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/29/1e34000e9766d112171764b9fa3226fa0153ab565d0c242c70e9945318a7/propcache-0.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06766d8f34733416e2e34f46fea488ad5d60726bb9481d3cddf89a6fa2d9603f", size = 43648, upload-time = "2025-06-09T22:54:08.023Z" },
+    { url = "https://files.pythonhosted.org/packages/46/92/1ad5af0df781e76988897da39b5f086c2bf0f028b7f9bd1f409bb05b6874/propcache-0.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2dc1f4a1df4fecf4e6f68013575ff4af84ef6f478fe5344317a65d38a8e6dc9", size = 43496, upload-time = "2025-06-09T22:54:09.228Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/ce/e96392460f9fb68461fabab3e095cb00c8ddf901205be4eae5ce246e5b7e/propcache-0.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be29c4f4810c5789cf10ddf6af80b041c724e629fa51e308a7a0fb19ed1ef7bf", size = 217288, upload-time = "2025-06-09T22:54:10.466Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/2a/866726ea345299f7ceefc861a5e782b045545ae6940851930a6adaf1fca6/propcache-0.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59d61f6970ecbd8ff2e9360304d5c8876a6abd4530cb752c06586849ac8a9dc9", size = 227456, upload-time = "2025-06-09T22:54:11.828Z" },
+    { url = "https://files.pythonhosted.org/packages/de/03/07d992ccb6d930398689187e1b3c718339a1c06b8b145a8d9650e4726166/propcache-0.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62180e0b8dbb6b004baec00a7983e4cc52f5ada9cd11f48c3528d8cfa7b96a66", size = 225429, upload-time = "2025-06-09T22:54:13.823Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/e6/116ba39448753b1330f48ab8ba927dcd6cf0baea8a0ccbc512dfb49ba670/propcache-0.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c144ca294a204c470f18cf4c9d78887810d04a3e2fbb30eea903575a779159df", size = 213472, upload-time = "2025-06-09T22:54:15.232Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/85/f01f5d97e54e428885a5497ccf7f54404cbb4f906688a1690cd51bf597dc/propcache-0.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5c2a784234c28854878d68978265617aa6dc0780e53d44b4d67f3651a17a9a2", size = 204480, upload-time = "2025-06-09T22:54:17.104Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/79/7bf5ab9033b8b8194cc3f7cf1aaa0e9c3256320726f64a3e1f113a812dce/propcache-0.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5745bc7acdafa978ca1642891b82c19238eadc78ba2aaa293c6863b304e552d7", size = 214530, upload-time = "2025-06-09T22:54:18.512Z" },
+    { url = "https://files.pythonhosted.org/packages/31/0b/bd3e0c00509b609317df4a18e6b05a450ef2d9a963e1d8bc9c9415d86f30/propcache-0.3.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:c0075bf773d66fa8c9d41f66cc132ecc75e5bb9dd7cce3cfd14adc5ca184cb95", size = 205230, upload-time = "2025-06-09T22:54:19.947Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/23/fae0ff9b54b0de4e819bbe559508da132d5683c32d84d0dc2ccce3563ed4/propcache-0.3.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5f57aa0847730daceff0497f417c9de353c575d8da3579162cc74ac294c5369e", size = 206754, upload-time = "2025-06-09T22:54:21.716Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/7f/ad6a3c22630aaa5f618b4dc3c3598974a72abb4c18e45a50b3cdd091eb2f/propcache-0.3.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:eef914c014bf72d18efb55619447e0aecd5fb7c2e3fa7441e2e5d6099bddff7e", size = 218430, upload-time = "2025-06-09T22:54:23.17Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/2c/ba4f1c0e8a4b4c75910742f0d333759d441f65a1c7f34683b4a74c0ee015/propcache-0.3.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2a4092e8549031e82facf3decdbc0883755d5bbcc62d3aea9d9e185549936dcf", size = 223884, upload-time = "2025-06-09T22:54:25.539Z" },
+    { url = "https://files.pythonhosted.org/packages/88/e4/ebe30fc399e98572019eee82ad0caf512401661985cbd3da5e3140ffa1b0/propcache-0.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:85871b050f174bc0bfb437efbdb68aaf860611953ed12418e4361bc9c392749e", size = 211480, upload-time = "2025-06-09T22:54:26.892Z" },
+    { url = "https://files.pythonhosted.org/packages/96/0a/7d5260b914e01d1d0906f7f38af101f8d8ed0dc47426219eeaf05e8ea7c2/propcache-0.3.2-cp311-cp311-win32.whl", hash = "sha256:36c8d9b673ec57900c3554264e630d45980fd302458e4ac801802a7fd2ef7897", size = 37757, upload-time = "2025-06-09T22:54:28.241Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/2d/89fe4489a884bc0da0c3278c552bd4ffe06a1ace559db5ef02ef24ab446b/propcache-0.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53af8cb6a781b02d2ea079b5b853ba9430fcbe18a8e3ce647d5982a3ff69f39", size = 41500, upload-time = "2025-06-09T22:54:29.4Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/42/9ca01b0a6f48e81615dca4765a8f1dd2c057e0540f6116a27dc5ee01dfb6/propcache-0.3.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8de106b6c84506b31c27168582cd3cb3000a6412c16df14a8628e5871ff83c10", size = 73674, upload-time = "2025-06-09T22:54:30.551Z" },
+    { url = "https://files.pythonhosted.org/packages/af/6e/21293133beb550f9c901bbece755d582bfaf2176bee4774000bd4dd41884/propcache-0.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:28710b0d3975117239c76600ea351934ac7b5ff56e60953474342608dbbb6154", size = 43570, upload-time = "2025-06-09T22:54:32.296Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/c8/0393a0a3a2b8760eb3bde3c147f62b20044f0ddac81e9d6ed7318ec0d852/propcache-0.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce26862344bdf836650ed2487c3d724b00fbfec4233a1013f597b78c1cb73615", size = 43094, upload-time = "2025-06-09T22:54:33.929Z" },
+    { url = "https://files.pythonhosted.org/packages/37/2c/489afe311a690399d04a3e03b069225670c1d489eb7b044a566511c1c498/propcache-0.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bca54bd347a253af2cf4544bbec232ab982f4868de0dd684246b67a51bc6b1db", size = 226958, upload-time = "2025-06-09T22:54:35.186Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/ca/63b520d2f3d418c968bf596839ae26cf7f87bead026b6192d4da6a08c467/propcache-0.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55780d5e9a2ddc59711d727226bb1ba83a22dd32f64ee15594b9392b1f544eb1", size = 234894, upload-time = "2025-06-09T22:54:36.708Z" },
+    { url = "https://files.pythonhosted.org/packages/11/60/1d0ed6fff455a028d678df30cc28dcee7af77fa2b0e6962ce1df95c9a2a9/propcache-0.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:035e631be25d6975ed87ab23153db6a73426a48db688070d925aa27e996fe93c", size = 233672, upload-time = "2025-06-09T22:54:38.062Z" },
+    { url = "https://files.pythonhosted.org/packages/37/7c/54fd5301ef38505ab235d98827207176a5c9b2aa61939b10a460ca53e123/propcache-0.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee6f22b6eaa39297c751d0e80c0d3a454f112f5c6481214fcf4c092074cecd67", size = 224395, upload-time = "2025-06-09T22:54:39.634Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/1a/89a40e0846f5de05fdc6779883bf46ba980e6df4d2ff8fb02643de126592/propcache-0.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ca3aee1aa955438c4dba34fc20a9f390e4c79967257d830f137bd5a8a32ed3b", size = 212510, upload-time = "2025-06-09T22:54:41.565Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/33/ca98368586c9566a6b8d5ef66e30484f8da84c0aac3f2d9aec6d31a11bd5/propcache-0.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7a4f30862869fa2b68380d677cc1c5fcf1e0f2b9ea0cf665812895c75d0ca3b8", size = 222949, upload-time = "2025-06-09T22:54:43.038Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/11/ace870d0aafe443b33b2f0b7efdb872b7c3abd505bfb4890716ad7865e9d/propcache-0.3.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b77ec3c257d7816d9f3700013639db7491a434644c906a2578a11daf13176251", size = 217258, upload-time = "2025-06-09T22:54:44.376Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/d2/86fd6f7adffcfc74b42c10a6b7db721d1d9ca1055c45d39a1a8f2a740a21/propcache-0.3.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cab90ac9d3f14b2d5050928483d3d3b8fb6b4018893fc75710e6aa361ecb2474", size = 213036, upload-time = "2025-06-09T22:54:46.243Z" },
+    { url = "https://files.pythonhosted.org/packages/07/94/2d7d1e328f45ff34a0a284cf5a2847013701e24c2a53117e7c280a4316b3/propcache-0.3.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0b504d29f3c47cf6b9e936c1852246c83d450e8e063d50562115a6be6d3a2535", size = 227684, upload-time = "2025-06-09T22:54:47.63Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/05/37ae63a0087677e90b1d14710e532ff104d44bc1efa3b3970fff99b891dc/propcache-0.3.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ce2ac2675a6aa41ddb2a0c9cbff53780a617ac3d43e620f8fd77ba1c84dcfc06", size = 234562, upload-time = "2025-06-09T22:54:48.982Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/7c/3f539fcae630408d0bd8bf3208b9a647ccad10976eda62402a80adf8fc34/propcache-0.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b4239611205294cc433845b914131b2a1f03500ff3c1ed093ed216b82621e1", size = 222142, upload-time = "2025-06-09T22:54:50.424Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/d2/34b9eac8c35f79f8a962546b3e97e9d4b990c420ee66ac8255d5d9611648/propcache-0.3.2-cp312-cp312-win32.whl", hash = "sha256:df4a81b9b53449ebc90cc4deefb052c1dd934ba85012aa912c7ea7b7e38b60c1", size = 37711, upload-time = "2025-06-09T22:54:52.072Z" },
+    { url = "https://files.pythonhosted.org/packages/19/61/d582be5d226cf79071681d1b46b848d6cb03d7b70af7063e33a2787eaa03/propcache-0.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7046e79b989d7fe457bb755844019e10f693752d169076138abf17f31380800c", size = 41479, upload-time = "2025-06-09T22:54:53.234Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/d1/8c747fafa558c603c4ca19d8e20b288aa0c7cda74e9402f50f31eb65267e/propcache-0.3.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ca592ed634a73ca002967458187109265e980422116c0a107cf93d81f95af945", size = 71286, upload-time = "2025-06-09T22:54:54.369Z" },
+    { url = "https://files.pythonhosted.org/packages/61/99/d606cb7986b60d89c36de8a85d58764323b3a5ff07770a99d8e993b3fa73/propcache-0.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9ecb0aad4020e275652ba3975740f241bd12a61f1a784df044cf7477a02bc252", size = 42425, upload-time = "2025-06-09T22:54:55.642Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/96/ef98f91bbb42b79e9bb82bdd348b255eb9d65f14dbbe3b1594644c4073f7/propcache-0.3.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7f08f1cc28bd2eade7a8a3d2954ccc673bb02062e3e7da09bc75d843386b342f", size = 41846, upload-time = "2025-06-09T22:54:57.246Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/ad/3f0f9a705fb630d175146cd7b1d2bf5555c9beaed54e94132b21aac098a6/propcache-0.3.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1a342c834734edb4be5ecb1e9fb48cb64b1e2320fccbd8c54bf8da8f2a84c33", size = 208871, upload-time = "2025-06-09T22:54:58.975Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/38/2085cda93d2c8b6ec3e92af2c89489a36a5886b712a34ab25de9fbca7992/propcache-0.3.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8a544caaae1ac73f1fecfae70ded3e93728831affebd017d53449e3ac052ac1e", size = 215720, upload-time = "2025-06-09T22:55:00.471Z" },
+    { url = "https://files.pythonhosted.org/packages/61/c1/d72ea2dc83ac7f2c8e182786ab0fc2c7bd123a1ff9b7975bee671866fe5f/propcache-0.3.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:310d11aa44635298397db47a3ebce7db99a4cc4b9bbdfcf6c98a60c8d5261cf1", size = 215203, upload-time = "2025-06-09T22:55:01.834Z" },
+    { url = "https://files.pythonhosted.org/packages/af/81/b324c44ae60c56ef12007105f1460d5c304b0626ab0cc6b07c8f2a9aa0b8/propcache-0.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c1396592321ac83157ac03a2023aa6cc4a3cc3cfdecb71090054c09e5a7cce3", size = 206365, upload-time = "2025-06-09T22:55:03.199Z" },
+    { url = "https://files.pythonhosted.org/packages/09/73/88549128bb89e66d2aff242488f62869014ae092db63ccea53c1cc75a81d/propcache-0.3.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cabf5b5902272565e78197edb682017d21cf3b550ba0460ee473753f28d23c1", size = 196016, upload-time = "2025-06-09T22:55:04.518Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/3f/3bdd14e737d145114a5eb83cb172903afba7242f67c5877f9909a20d948d/propcache-0.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0a2f2235ac46a7aa25bdeb03a9e7060f6ecbd213b1f9101c43b3090ffb971ef6", size = 205596, upload-time = "2025-06-09T22:55:05.942Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/ca/2f4aa819c357d3107c3763d7ef42c03980f9ed5c48c82e01e25945d437c1/propcache-0.3.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:92b69e12e34869a6970fd2f3da91669899994b47c98f5d430b781c26f1d9f387", size = 200977, upload-time = "2025-06-09T22:55:07.792Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/4a/e65276c7477533c59085251ae88505caf6831c0e85ff8b2e31ebcbb949b1/propcache-0.3.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:54e02207c79968ebbdffc169591009f4474dde3b4679e16634d34c9363ff56b4", size = 197220, upload-time = "2025-06-09T22:55:09.173Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/54/fc7152e517cf5578278b242396ce4d4b36795423988ef39bb8cd5bf274c8/propcache-0.3.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4adfb44cb588001f68c5466579d3f1157ca07f7504fc91ec87862e2b8e556b88", size = 210642, upload-time = "2025-06-09T22:55:10.62Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/80/abeb4a896d2767bf5f1ea7b92eb7be6a5330645bd7fb844049c0e4045d9d/propcache-0.3.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fd3e6019dc1261cd0291ee8919dd91fbab7b169bb76aeef6c716833a3f65d206", size = 212789, upload-time = "2025-06-09T22:55:12.029Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/db/ea12a49aa7b2b6d68a5da8293dcf50068d48d088100ac016ad92a6a780e6/propcache-0.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4c181cad81158d71c41a2bce88edce078458e2dd5ffee7eddd6b05da85079f43", size = 205880, upload-time = "2025-06-09T22:55:13.45Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/e5/9076a0bbbfb65d1198007059c65639dfd56266cf8e477a9707e4b1999ff4/propcache-0.3.2-cp313-cp313-win32.whl", hash = "sha256:8a08154613f2249519e549de2330cf8e2071c2887309a7b07fb56098f5170a02", size = 37220, upload-time = "2025-06-09T22:55:15.284Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/f5/b369e026b09a26cd77aa88d8fffd69141d2ae00a2abaaf5380d2603f4b7f/propcache-0.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:e41671f1594fc4ab0a6dec1351864713cb3a279910ae8b58f884a88a0a632c05", size = 40678, upload-time = "2025-06-09T22:55:16.445Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/3a/6ece377b55544941a08d03581c7bc400a3c8cd3c2865900a68d5de79e21f/propcache-0.3.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:9a3cf035bbaf035f109987d9d55dc90e4b0e36e04bbbb95af3055ef17194057b", size = 76560, upload-time = "2025-06-09T22:55:17.598Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/da/64a2bb16418740fa634b0e9c3d29edff1db07f56d3546ca2d86ddf0305e1/propcache-0.3.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:156c03d07dc1323d8dacaa221fbe028c5c70d16709cdd63502778e6c3ccca1b0", size = 44676, upload-time = "2025-06-09T22:55:18.922Z" },
+    { url = "https://files.pythonhosted.org/packages/36/7b/f025e06ea51cb72c52fb87e9b395cced02786610b60a3ed51da8af017170/propcache-0.3.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74413c0ba02ba86f55cf60d18daab219f7e531620c15f1e23d95563f505efe7e", size = 44701, upload-time = "2025-06-09T22:55:20.106Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/00/faa1b1b7c3b74fc277f8642f32a4c72ba1d7b2de36d7cdfb676db7f4303e/propcache-0.3.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f066b437bb3fa39c58ff97ab2ca351db465157d68ed0440abecb21715eb24b28", size = 276934, upload-time = "2025-06-09T22:55:21.5Z" },
+    { url = "https://files.pythonhosted.org/packages/74/ab/935beb6f1756e0476a4d5938ff44bf0d13a055fed880caf93859b4f1baf4/propcache-0.3.2-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1304b085c83067914721e7e9d9917d41ad87696bf70f0bc7dee450e9c71ad0a", size = 278316, upload-time = "2025-06-09T22:55:22.918Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/9d/994a5c1ce4389610838d1caec74bdf0e98b306c70314d46dbe4fcf21a3e2/propcache-0.3.2-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab50cef01b372763a13333b4e54021bdcb291fc9a8e2ccb9c2df98be51bcde6c", size = 282619, upload-time = "2025-06-09T22:55:24.651Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/00/a10afce3d1ed0287cef2e09506d3be9822513f2c1e96457ee369adb9a6cd/propcache-0.3.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fad3b2a085ec259ad2c2842666b2a0a49dea8463579c606426128925af1ed725", size = 265896, upload-time = "2025-06-09T22:55:26.049Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/a8/2aa6716ffa566ca57c749edb909ad27884680887d68517e4be41b02299f3/propcache-0.3.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:261fa020c1c14deafd54c76b014956e2f86991af198c51139faf41c4d5e83892", size = 252111, upload-time = "2025-06-09T22:55:27.381Z" },
+    { url = "https://files.pythonhosted.org/packages/36/4f/345ca9183b85ac29c8694b0941f7484bf419c7f0fea2d1e386b4f7893eed/propcache-0.3.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:46d7f8aa79c927e5f987ee3a80205c987717d3659f035c85cf0c3680526bdb44", size = 268334, upload-time = "2025-06-09T22:55:28.747Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/ca/fcd54f78b59e3f97b3b9715501e3147f5340167733d27db423aa321e7148/propcache-0.3.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:6d8f3f0eebf73e3c0ff0e7853f68be638b4043c65a70517bb575eff54edd8dbe", size = 255026, upload-time = "2025-06-09T22:55:30.184Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/95/8e6a6bbbd78ac89c30c225210a5c687790e532ba4088afb8c0445b77ef37/propcache-0.3.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:03c89c1b14a5452cf15403e291c0ccd7751d5b9736ecb2c5bab977ad6c5bcd81", size = 250724, upload-time = "2025-06-09T22:55:31.646Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/b0/0dd03616142baba28e8b2d14ce5df6631b4673850a3d4f9c0f9dd714a404/propcache-0.3.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:0cc17efde71e12bbaad086d679ce575268d70bc123a5a71ea7ad76f70ba30bba", size = 268868, upload-time = "2025-06-09T22:55:33.209Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/98/2c12407a7e4fbacd94ddd32f3b1e3d5231e77c30ef7162b12a60e2dd5ce3/propcache-0.3.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:acdf05d00696bc0447e278bb53cb04ca72354e562cf88ea6f9107df8e7fd9770", size = 271322, upload-time = "2025-06-09T22:55:35.065Z" },
+    { url = "https://files.pythonhosted.org/packages/35/91/9cb56efbb428b006bb85db28591e40b7736847b8331d43fe335acf95f6c8/propcache-0.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4445542398bd0b5d32df908031cb1b30d43ac848e20470a878b770ec2dcc6330", size = 265778, upload-time = "2025-06-09T22:55:36.45Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/4c/b0fe775a2bdd01e176b14b574be679d84fc83958335790f7c9a686c1f468/propcache-0.3.2-cp313-cp313t-win32.whl", hash = "sha256:f86e5d7cd03afb3a1db8e9f9f6eff15794e79e791350ac48a8c924e6f439f394", size = 41175, upload-time = "2025-06-09T22:55:38.436Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/ff/47f08595e3d9b5e149c150f88d9714574f1a7cbd89fe2817158a952674bf/propcache-0.3.2-cp313-cp313t-win_amd64.whl", hash = "sha256:9704bedf6e7cbe3c65eca4379a9b53ee6a83749f047808cbb5044d40d7d72198", size = 44857, upload-time = "2025-06-09T22:55:39.687Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/39/8ea9bcfaaff16fd0b0fc901ee522e24c9ec44b4ca0229cfffb8066a06959/propcache-0.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a7fad897f14d92086d6b03fdd2eb844777b0c4d7ec5e3bac0fbae2ab0602bbe5", size = 74678, upload-time = "2025-06-09T22:55:41.227Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/85/cab84c86966e1d354cf90cdc4ba52f32f99a5bca92a1529d666d957d7686/propcache-0.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1f43837d4ca000243fd7fd6301947d7cb93360d03cd08369969450cc6b2ce3b4", size = 43829, upload-time = "2025-06-09T22:55:42.417Z" },
+    { url = "https://files.pythonhosted.org/packages/23/f7/9cb719749152d8b26d63801b3220ce2d3931312b2744d2b3a088b0ee9947/propcache-0.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:261df2e9474a5949c46e962065d88eb9b96ce0f2bd30e9d3136bcde84befd8f2", size = 43729, upload-time = "2025-06-09T22:55:43.651Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/a2/0b2b5a210ff311260002a315f6f9531b65a36064dfb804655432b2f7d3e3/propcache-0.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e514326b79e51f0a177daab1052bc164d9d9e54133797a3a58d24c9c87a3fe6d", size = 204483, upload-time = "2025-06-09T22:55:45.327Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/e0/7aff5de0c535f783b0c8be5bdb750c305c1961d69fbb136939926e155d98/propcache-0.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d4a996adb6904f85894570301939afeee65f072b4fd265ed7e569e8d9058e4ec", size = 217425, upload-time = "2025-06-09T22:55:46.729Z" },
+    { url = "https://files.pythonhosted.org/packages/92/1d/65fa889eb3b2a7d6e4ed3c2b568a9cb8817547a1450b572de7bf24872800/propcache-0.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:76cace5d6b2a54e55b137669b30f31aa15977eeed390c7cbfb1dafa8dfe9a701", size = 214723, upload-time = "2025-06-09T22:55:48.342Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/e2/eecf6989870988dfd731de408a6fa366e853d361a06c2133b5878ce821ad/propcache-0.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31248e44b81d59d6addbb182c4720f90b44e1efdc19f58112a3c3a1615fb47ef", size = 200166, upload-time = "2025-06-09T22:55:49.775Z" },
+    { url = "https://files.pythonhosted.org/packages/12/06/c32be4950967f18f77489268488c7cdc78cbfc65a8ba8101b15e526b83dc/propcache-0.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abb7fa19dbf88d3857363e0493b999b8011eea856b846305d8c0512dfdf8fbb1", size = 194004, upload-time = "2025-06-09T22:55:51.335Z" },
+    { url = "https://files.pythonhosted.org/packages/46/6c/17b521a6b3b7cbe277a4064ff0aa9129dd8c89f425a5a9b6b4dd51cc3ff4/propcache-0.3.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d81ac3ae39d38588ad0549e321e6f773a4e7cc68e7751524a22885d5bbadf886", size = 203075, upload-time = "2025-06-09T22:55:52.681Z" },
+    { url = "https://files.pythonhosted.org/packages/62/cb/3bdba2b736b3e45bc0e40f4370f745b3e711d439ffbffe3ae416393eece9/propcache-0.3.2-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:cc2782eb0f7a16462285b6f8394bbbd0e1ee5f928034e941ffc444012224171b", size = 195407, upload-time = "2025-06-09T22:55:54.048Z" },
+    { url = "https://files.pythonhosted.org/packages/29/bd/760c5c6a60a4a2c55a421bc34a25ba3919d49dee411ddb9d1493bb51d46e/propcache-0.3.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:db429c19a6c7e8a1c320e6a13c99799450f411b02251fb1b75e6217cf4a14fcb", size = 196045, upload-time = "2025-06-09T22:55:55.485Z" },
+    { url = "https://files.pythonhosted.org/packages/76/58/ced2757a46f55b8c84358d6ab8de4faf57cba831c51e823654da7144b13a/propcache-0.3.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:21d8759141a9e00a681d35a1f160892a36fb6caa715ba0b832f7747da48fb6ea", size = 208432, upload-time = "2025-06-09T22:55:56.884Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/ec/d98ea8d5a4d8fe0e372033f5254eddf3254344c0c5dc6c49ab84349e4733/propcache-0.3.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:2ca6d378f09adb13837614ad2754fa8afaee330254f404299611bce41a8438cb", size = 210100, upload-time = "2025-06-09T22:55:58.498Z" },
+    { url = "https://files.pythonhosted.org/packages/56/84/b6d8a7ecf3f62d7dd09d9d10bbf89fad6837970ef868b35b5ffa0d24d9de/propcache-0.3.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:34a624af06c048946709f4278b4176470073deda88d91342665d95f7c6270fbe", size = 200712, upload-time = "2025-06-09T22:55:59.906Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/32/889f4903ddfe4a9dc61da71ee58b763758cf2d608fe1decede06e6467f8d/propcache-0.3.2-cp39-cp39-win32.whl", hash = "sha256:4ba3fef1c30f306b1c274ce0b8baaa2c3cdd91f645c48f06394068f37d3837a1", size = 38187, upload-time = "2025-06-09T22:56:01.212Z" },
+    { url = "https://files.pythonhosted.org/packages/67/74/d666795fb9ba1dc139d30de64f3b6fd1ff9c9d3d96ccfdb992cd715ce5d2/propcache-0.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:7a2368eed65fc69a7a7a40b27f22e85e7627b74216f0846b04ba5c116e191ec9", size = 42025, upload-time = "2025-06-09T22:56:02.875Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/35/cc0aaecf278bb4575b8555f2b137de5ab821595ddae9da9d3cd1da4072c7/propcache-0.3.2-py3-none-any.whl", hash = "sha256:98f1ec44fb675f5052cccc8e609c46ed23a35a1cfd18545ad4e29002d858a43f", size = 12663, upload-time = "2025-06-09T22:56:04.484Z" },
+]
+
+[[package]]
+name = "protobuf"
+version = "3.19.6"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/51/d1/79bfd1f481469b661a2eddab551255536401892722189433282bfb13cfb1/protobuf-3.19.6.tar.gz", hash = "sha256:5f5540d57a43042389e87661c6eaa50f47c19c6176e8cf1c4f287aeefeccb5c4", size = 218071, upload-time = "2022-09-29T22:07:23.03Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/4b/3b/90f805b9e5ecacf8a216f2e5acabc2d3ad965b62803510be41804e6bfbfe/protobuf-3.19.6-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:010be24d5a44be7b0613750ab40bc8b8cedc796db468eae6c779b395f50d1fa1", size = 913631, upload-time = "2022-09-29T21:17:39.095Z" },
+    { url = "https://files.pythonhosted.org/packages/26/ef/bd6ba3b4ff9a35944bdd325e2c9ee56f71e855757f7d43938232499f0278/protobuf-3.19.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11478547958c2dfea921920617eb457bc26867b0d1aa065ab05f35080c5d9eb6", size = 1055327, upload-time = "2022-09-29T21:17:41.054Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/db/8b33c9558f1f27dd74e7f9ad730c6b32efab431419af556b1659e125b041/protobuf-3.19.6-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:30a15015d86b9c3b8d6bf78d5b8c7749f2512c29f168ca259c9d7727604d0e39", size = 913657, upload-time = "2022-09-29T21:18:18.359Z" },
+    { url = "https://files.pythonhosted.org/packages/51/61/e80b7a04f4e1b4eecc86582335205fd876abca0abafee4a6c001f70a375e/protobuf-3.19.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:878b4cd080a21ddda6ac6d1e163403ec6eea2e206cf225982ae04567d39be7b0", size = 1055457, upload-time = "2022-09-29T21:18:20.212Z" },
+    { url = "https://files.pythonhosted.org/packages/32/27/1141a8232723dcb10a595cc0ce4321dcbbd5215300bf4acfc142343205bf/protobuf-3.19.6-py2.py3-none-any.whl", hash = "sha256:14082457dc02be946f60b15aad35e9f5c69e738f80ebbc0900a19bc83734a5a4", size = 162648, upload-time = "2022-09-29T22:07:20.303Z" },
 ]
 
 [[package]]
 name = "protobuf"
 version = "6.32.1"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/fa/a4/cc17347aa2897568beece2e674674359f911d6fe21b0b8d6268cd42727ac/protobuf-6.32.1.tar.gz", hash = "sha256:ee2469e4a021474ab9baafea6cd070e5bf27c7d29433504ddea1a4ee5850f68d", size = 440635 }
+resolution-markers = [
+    "python_full_version >= '3.14'",
+    "python_full_version == '3.13.*'",
+    "python_full_version == '3.12.*'",
+    "python_full_version == '3.11.*'",
+    "python_full_version == '3.10.*'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/fa/a4/cc17347aa2897568beece2e674674359f911d6fe21b0b8d6268cd42727ac/protobuf-6.32.1.tar.gz", hash = "sha256:ee2469e4a021474ab9baafea6cd070e5bf27c7d29433504ddea1a4ee5850f68d", size = 440635, upload-time = "2025-09-11T21:38:42.935Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/3f/be/8dd0a927c559b37d7a6c8ab79034fd167dcc1f851595f2e641ad62be8643/protobuf-6.32.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:2f5b80a49e1eb7b86d85fcd23fe92df154b9730a725c3b38c4e43b9d77018bf4", size = 322874 },
-    { url = "https://files.pythonhosted.org/packages/5c/f6/88d77011b605ef979aace37b7703e4eefad066f7e84d935e5a696515c2dd/protobuf-6.32.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:b1864818300c297265c83a4982fd3169f97122c299f56a56e2445c3698d34710", size = 322013 },
-    { url = "https://files.pythonhosted.org/packages/97/b7/15cc7d93443d6c6a84626ae3258a91f4c6ac8c0edd5df35ea7658f71b79c/protobuf-6.32.1-py3-none-any.whl", hash = "sha256:2601b779fc7d32a866c6b4404f9d42a3f67c5b9f3f15b4db3cccabe06b95c346", size = 169289 },
+    { url = "https://files.pythonhosted.org/packages/3f/be/8dd0a927c559b37d7a6c8ab79034fd167dcc1f851595f2e641ad62be8643/protobuf-6.32.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:2f5b80a49e1eb7b86d85fcd23fe92df154b9730a725c3b38c4e43b9d77018bf4", size = 322874, upload-time = "2025-09-11T21:38:35.509Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/f6/88d77011b605ef979aace37b7703e4eefad066f7e84d935e5a696515c2dd/protobuf-6.32.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:b1864818300c297265c83a4982fd3169f97122c299f56a56e2445c3698d34710", size = 322013, upload-time = "2025-09-11T21:38:37.017Z" },
+    { url = "https://files.pythonhosted.org/packages/97/b7/15cc7d93443d6c6a84626ae3258a91f4c6ac8c0edd5df35ea7658f71b79c/protobuf-6.32.1-py3-none-any.whl", hash = "sha256:2601b779fc7d32a866c6b4404f9d42a3f67c5b9f3f15b4db3cccabe06b95c346", size = 169289, upload-time = "2025-09-11T21:38:41.234Z" },
 ]
 
 [[package]]
 name = "psutil"
 version = "7.1.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/b3/31/4723d756b59344b643542936e37a31d1d3204bcdc42a7daa8ee9eb06fb50/psutil-7.1.0.tar.gz", hash = "sha256:655708b3c069387c8b77b072fc429a57d0e214221d01c0a772df7dfedcb3bcd2", size = 497660 }
+sdist = { url = "https://files.pythonhosted.org/packages/b3/31/4723d756b59344b643542936e37a31d1d3204bcdc42a7daa8ee9eb06fb50/psutil-7.1.0.tar.gz", hash = "sha256:655708b3c069387c8b77b072fc429a57d0e214221d01c0a772df7dfedcb3bcd2", size = 497660, upload-time = "2025-09-17T20:14:52.902Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/46/62/ce4051019ee20ce0ed74432dd73a5bb087a6704284a470bb8adff69a0932/psutil-7.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76168cef4397494250e9f4e73eb3752b146de1dd950040b29186d0cce1d5ca13", size = 245242 },
-    { url = "https://files.pythonhosted.org/packages/38/61/f76959fba841bf5b61123fbf4b650886dc4094c6858008b5bf73d9057216/psutil-7.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:5d007560c8c372efdff9e4579c2846d71de737e4605f611437255e81efcca2c5", size = 246682 },
-    { url = "https://files.pythonhosted.org/packages/88/7a/37c99d2e77ec30d63398ffa6a660450b8a62517cabe44b3e9bae97696e8d/psutil-7.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22e4454970b32472ce7deaa45d045b34d3648ce478e26a04c7e858a0a6e75ff3", size = 287994 },
-    { url = "https://files.pythonhosted.org/packages/9d/de/04c8c61232f7244aa0a4b9a9fbd63a89d5aeaf94b2fc9d1d16e2faa5cbb0/psutil-7.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c70e113920d51e89f212dd7be06219a9b88014e63a4cec69b684c327bc474e3", size = 291163 },
-    { url = "https://files.pythonhosted.org/packages/f4/58/c4f976234bf6d4737bc8c02a81192f045c307b72cf39c9e5c5a2d78927f6/psutil-7.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7d4a113425c037300de3ac8b331637293da9be9713855c4fc9d2d97436d7259d", size = 293625 },
-    { url = "https://files.pythonhosted.org/packages/79/87/157c8e7959ec39ced1b11cc93c730c4fb7f9d408569a6c59dbd92ceb35db/psutil-7.1.0-cp37-abi3-win32.whl", hash = "sha256:09ad740870c8d219ed8daae0ad3b726d3bf9a028a198e7f3080f6a1888b99bca", size = 244812 },
-    { url = "https://files.pythonhosted.org/packages/bf/e9/b44c4f697276a7a95b8e94d0e320a7bf7f3318521b23de69035540b39838/psutil-7.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:57f5e987c36d3146c0dd2528cd42151cf96cd359b9d67cfff836995cc5df9a3d", size = 247965 },
-    { url = "https://files.pythonhosted.org/packages/26/65/1070a6e3c036f39142c2820c4b52e9243246fcfc3f96239ac84472ba361e/psutil-7.1.0-cp37-abi3-win_arm64.whl", hash = "sha256:6937cb68133e7c97b6cc9649a570c9a18ba0efebed46d8c5dae4c07fa1b67a07", size = 244971 },
+    { url = "https://files.pythonhosted.org/packages/46/62/ce4051019ee20ce0ed74432dd73a5bb087a6704284a470bb8adff69a0932/psutil-7.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76168cef4397494250e9f4e73eb3752b146de1dd950040b29186d0cce1d5ca13", size = 245242, upload-time = "2025-09-17T20:14:56.126Z" },
+    { url = "https://files.pythonhosted.org/packages/38/61/f76959fba841bf5b61123fbf4b650886dc4094c6858008b5bf73d9057216/psutil-7.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:5d007560c8c372efdff9e4579c2846d71de737e4605f611437255e81efcca2c5", size = 246682, upload-time = "2025-09-17T20:14:58.25Z" },
+    { url = "https://files.pythonhosted.org/packages/88/7a/37c99d2e77ec30d63398ffa6a660450b8a62517cabe44b3e9bae97696e8d/psutil-7.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22e4454970b32472ce7deaa45d045b34d3648ce478e26a04c7e858a0a6e75ff3", size = 287994, upload-time = "2025-09-17T20:14:59.901Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/de/04c8c61232f7244aa0a4b9a9fbd63a89d5aeaf94b2fc9d1d16e2faa5cbb0/psutil-7.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c70e113920d51e89f212dd7be06219a9b88014e63a4cec69b684c327bc474e3", size = 291163, upload-time = "2025-09-17T20:15:01.481Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/58/c4f976234bf6d4737bc8c02a81192f045c307b72cf39c9e5c5a2d78927f6/psutil-7.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7d4a113425c037300de3ac8b331637293da9be9713855c4fc9d2d97436d7259d", size = 293625, upload-time = "2025-09-17T20:15:04.492Z" },
+    { url = "https://files.pythonhosted.org/packages/79/87/157c8e7959ec39ced1b11cc93c730c4fb7f9d408569a6c59dbd92ceb35db/psutil-7.1.0-cp37-abi3-win32.whl", hash = "sha256:09ad740870c8d219ed8daae0ad3b726d3bf9a028a198e7f3080f6a1888b99bca", size = 244812, upload-time = "2025-09-17T20:15:07.462Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/e9/b44c4f697276a7a95b8e94d0e320a7bf7f3318521b23de69035540b39838/psutil-7.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:57f5e987c36d3146c0dd2528cd42151cf96cd359b9d67cfff836995cc5df9a3d", size = 247965, upload-time = "2025-09-17T20:15:09.673Z" },
+    { url = "https://files.pythonhosted.org/packages/26/65/1070a6e3c036f39142c2820c4b52e9243246fcfc3f96239ac84472ba361e/psutil-7.1.0-cp37-abi3-win_arm64.whl", hash = "sha256:6937cb68133e7c97b6cc9649a570c9a18ba0efebed46d8c5dae4c07fa1b67a07", size = 244971, upload-time = "2025-09-17T20:15:12.262Z" },
 ]
 
 [[package]]
 name = "py-cpuinfo"
 version = "9.0.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716 }
+sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335 },
+    { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" },
 ]
 
 [[package]]
@@ -1910,48 +2309,53 @@ name = "pyarrow"
 version = "21.0.0"
 source = { registry = "https://pypi.org/simple" }
 resolution-markers = [
-    "python_full_version == '3.13.*'",
-    "python_full_version == '3.12.*'",
-    "python_full_version == '3.11.*'",
-    "python_full_version < '3.11'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/ef/c2/ea068b8f00905c06329a3dfcd40d0fcc2b7d0f2e355bdb25b65e0a0e4cd4/pyarrow-21.0.0.tar.gz", hash = "sha256:5051f2dccf0e283ff56335760cbc8622cf52264d67e359d5569541ac11b6d5bc", size = 1133487 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e563271e2c5ff4d4a4cbeb2c83d5cf0d4938b891518e676025f7268c6fe5fe26", size = 31196837 },
-    { url = "https://files.pythonhosted.org/packages/df/5f/c1c1997613abf24fceb087e79432d24c19bc6f7259cab57c2c8e5e545fab/pyarrow-21.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:fee33b0ca46f4c85443d6c450357101e47d53e6c3f008d658c27a2d020d44c79", size = 32659470 },
-    { url = "https://files.pythonhosted.org/packages/3e/ed/b1589a777816ee33ba123ba1e4f8f02243a844fed0deec97bde9fb21a5cf/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:7be45519b830f7c24b21d630a31d48bcebfd5d4d7f9d3bdb49da9cdf6d764edb", size = 41055619 },
-    { url = "https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:26bfd95f6bff443ceae63c65dc7e048670b7e98bc892210acba7e4995d3d4b51", size = 42733488 },
-    { url = "https://files.pythonhosted.org/packages/f8/cc/de02c3614874b9089c94eac093f90ca5dfa6d5afe45de3ba847fd950fdf1/pyarrow-21.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:bd04ec08f7f8bd113c55868bd3fc442a9db67c27af098c5f814a3091e71cc61a", size = 43329159 },
-    { url = "https://files.pythonhosted.org/packages/a6/3e/99473332ac40278f196e105ce30b79ab8affab12f6194802f2593d6b0be2/pyarrow-21.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9b0b14b49ac10654332a805aedfc0147fb3469cbf8ea951b3d040dab12372594", size = 45050567 },
-    { url = "https://files.pythonhosted.org/packages/7b/f5/c372ef60593d713e8bfbb7e0c743501605f0ad00719146dc075faf11172b/pyarrow-21.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:9d9f8bcb4c3be7738add259738abdeddc363de1b80e3310e04067aa1ca596634", size = 26217959 },
-    { url = "https://files.pythonhosted.org/packages/94/dc/80564a3071a57c20b7c32575e4a0120e8a330ef487c319b122942d665960/pyarrow-21.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c077f48aab61738c237802836fc3844f85409a46015635198761b0d6a688f87b", size = 31243234 },
-    { url = "https://files.pythonhosted.org/packages/ea/cc/3b51cb2db26fe535d14f74cab4c79b191ed9a8cd4cbba45e2379b5ca2746/pyarrow-21.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:689f448066781856237eca8d1975b98cace19b8dd2ab6145bf49475478bcaa10", size = 32714370 },
-    { url = "https://files.pythonhosted.org/packages/24/11/a4431f36d5ad7d83b87146f515c063e4d07ef0b7240876ddb885e6b44f2e/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:479ee41399fcddc46159a551705b89c05f11e8b8cb8e968f7fec64f62d91985e", size = 41135424 },
-    { url = "https://files.pythonhosted.org/packages/74/dc/035d54638fc5d2971cbf1e987ccd45f1091c83bcf747281cf6cc25e72c88/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:40ebfcb54a4f11bcde86bc586cbd0272bac0d516cfa539c799c2453768477569", size = 42823810 },
-    { url = "https://files.pythonhosted.org/packages/2e/3b/89fced102448a9e3e0d4dded1f37fa3ce4700f02cdb8665457fcc8015f5b/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8d58d8497814274d3d20214fbb24abcad2f7e351474357d552a8d53bce70c70e", size = 43391538 },
-    { url = "https://files.pythonhosted.org/packages/fb/bb/ea7f1bd08978d39debd3b23611c293f64a642557e8141c80635d501e6d53/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:585e7224f21124dd57836b1530ac8f2df2afc43c861d7bf3d58a4870c42ae36c", size = 45120056 },
-    { url = "https://files.pythonhosted.org/packages/6e/0b/77ea0600009842b30ceebc3337639a7380cd946061b620ac1a2f3cb541e2/pyarrow-21.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:555ca6935b2cbca2c0e932bedd853e9bc523098c39636de9ad4693b5b1df86d6", size = 26220568 },
-    { url = "https://files.pythonhosted.org/packages/ca/d4/d4f817b21aacc30195cf6a46ba041dd1be827efa4a623cc8bf39a1c2a0c0/pyarrow-21.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:3a302f0e0963db37e0a24a70c56cf91a4faa0bca51c23812279ca2e23481fccd", size = 31160305 },
-    { url = "https://files.pythonhosted.org/packages/a2/9c/dcd38ce6e4b4d9a19e1d36914cb8e2b1da4e6003dd075474c4cfcdfe0601/pyarrow-21.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:b6b27cf01e243871390474a211a7922bfbe3bda21e39bc9160daf0da3fe48876", size = 32684264 },
-    { url = "https://files.pythonhosted.org/packages/4f/74/2a2d9f8d7a59b639523454bec12dba35ae3d0a07d8ab529dc0809f74b23c/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e72a8ec6b868e258a2cd2672d91f2860ad532d590ce94cdf7d5e7ec674ccf03d", size = 41108099 },
-    { url = "https://files.pythonhosted.org/packages/ad/90/2660332eeb31303c13b653ea566a9918484b6e4d6b9d2d46879a33ab0622/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b7ae0bbdc8c6674259b25bef5d2a1d6af5d39d7200c819cf99e07f7dfef1c51e", size = 42829529 },
-    { url = "https://files.pythonhosted.org/packages/33/27/1a93a25c92717f6aa0fca06eb4700860577d016cd3ae51aad0e0488ac899/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:58c30a1729f82d201627c173d91bd431db88ea74dcaa3885855bc6203e433b82", size = 43367883 },
-    { url = "https://files.pythonhosted.org/packages/05/d9/4d09d919f35d599bc05c6950095e358c3e15148ead26292dfca1fb659b0c/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:072116f65604b822a7f22945a7a6e581cfa28e3454fdcc6939d4ff6090126623", size = 45133802 },
-    { url = "https://files.pythonhosted.org/packages/71/30/f3795b6e192c3ab881325ffe172e526499eb3780e306a15103a2764916a2/pyarrow-21.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:cf56ec8b0a5c8c9d7021d6fd754e688104f9ebebf1bf4449613c9531f5346a18", size = 26203175 },
-    { url = "https://files.pythonhosted.org/packages/16/ca/c7eaa8e62db8fb37ce942b1ea0c6d7abfe3786ca193957afa25e71b81b66/pyarrow-21.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e99310a4ebd4479bcd1964dff9e14af33746300cb014aa4a3781738ac63baf4a", size = 31154306 },
-    { url = "https://files.pythonhosted.org/packages/ce/e8/e87d9e3b2489302b3a1aea709aaca4b781c5252fcb812a17ab6275a9a484/pyarrow-21.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:d2fe8e7f3ce329a71b7ddd7498b3cfac0eeb200c2789bd840234f0dc271a8efe", size = 32680622 },
-    { url = "https://files.pythonhosted.org/packages/84/52/79095d73a742aa0aba370c7942b1b655f598069489ab387fe47261a849e1/pyarrow-21.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f522e5709379d72fb3da7785aa489ff0bb87448a9dc5a75f45763a795a089ebd", size = 41104094 },
-    { url = "https://files.pythonhosted.org/packages/89/4b/7782438b551dbb0468892a276b8c789b8bbdb25ea5c5eb27faadd753e037/pyarrow-21.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:69cbbdf0631396e9925e048cfa5bce4e8c3d3b41562bbd70c685a8eb53a91e61", size = 42825576 },
-    { url = "https://files.pythonhosted.org/packages/b3/62/0f29de6e0a1e33518dec92c65be0351d32d7ca351e51ec5f4f837a9aab91/pyarrow-21.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:731c7022587006b755d0bdb27626a1a3bb004bb56b11fb30d98b6c1b4718579d", size = 43368342 },
-    { url = "https://files.pythonhosted.org/packages/90/c7/0fa1f3f29cf75f339768cc698c8ad4ddd2481c1742e9741459911c9ac477/pyarrow-21.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc56bc708f2d8ac71bd1dcb927e458c93cec10b98eb4120206a4091db7b67b99", size = 45131218 },
-    { url = "https://files.pythonhosted.org/packages/01/63/581f2076465e67b23bc5a37d4a2abff8362d389d29d8105832e82c9c811c/pyarrow-21.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:186aa00bca62139f75b7de8420f745f2af12941595bbbfa7ed3870ff63e25636", size = 26087551 },
-    { url = "https://files.pythonhosted.org/packages/c9/ab/357d0d9648bb8241ee7348e564f2479d206ebe6e1c47ac5027c2e31ecd39/pyarrow-21.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:a7a102574faa3f421141a64c10216e078df467ab9576684d5cd696952546e2da", size = 31290064 },
-    { url = "https://files.pythonhosted.org/packages/3f/8a/5685d62a990e4cac2043fc76b4661bf38d06efed55cf45a334b455bd2759/pyarrow-21.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:1e005378c4a2c6db3ada3ad4c217b381f6c886f0a80d6a316fe586b90f77efd7", size = 32727837 },
-    { url = "https://files.pythonhosted.org/packages/fc/de/c0828ee09525c2bafefd3e736a248ebe764d07d0fd762d4f0929dbc516c9/pyarrow-21.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:65f8e85f79031449ec8706b74504a316805217b35b6099155dd7e227eef0d4b6", size = 41014158 },
-    { url = "https://files.pythonhosted.org/packages/6e/26/a2865c420c50b7a3748320b614f3484bfcde8347b2639b2b903b21ce6a72/pyarrow-21.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:3a81486adc665c7eb1a2bde0224cfca6ceaba344a82a971ef059678417880eb8", size = 42667885 },
-    { url = "https://files.pythonhosted.org/packages/0a/f9/4ee798dc902533159250fb4321267730bc0a107d8c6889e07c3add4fe3a5/pyarrow-21.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fc0d2f88b81dcf3ccf9a6ae17f89183762c8a94a5bdcfa09e05cfe413acf0503", size = 43276625 },
-    { url = "https://files.pythonhosted.org/packages/5a/da/e02544d6997037a4b0d22d8e5f66bc9315c3671371a8b18c79ade1cefe14/pyarrow-21.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6299449adf89df38537837487a4f8d3bd91ec94354fdd2a7d30bc11c48ef6e79", size = 44951890 },
-    { url = "https://files.pythonhosted.org/packages/e5/4e/519c1bc1876625fe6b71e9a28287c43ec2f20f73c658b9ae1d485c0c206e/pyarrow-21.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:222c39e2c70113543982c6b34f3077962b44fca38c0bd9e68bb6781534425c10", size = 26371006 },
+    "python_full_version == '3.10.*'",
+    "python_full_version < '3.10'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ef/c2/ea068b8f00905c06329a3dfcd40d0fcc2b7d0f2e355bdb25b65e0a0e4cd4/pyarrow-21.0.0.tar.gz", hash = "sha256:5051f2dccf0e283ff56335760cbc8622cf52264d67e359d5569541ac11b6d5bc", size = 1133487, upload-time = "2025-07-18T00:57:31.761Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e563271e2c5ff4d4a4cbeb2c83d5cf0d4938b891518e676025f7268c6fe5fe26", size = 31196837, upload-time = "2025-07-18T00:54:34.755Z" },
+    { url = "https://files.pythonhosted.org/packages/df/5f/c1c1997613abf24fceb087e79432d24c19bc6f7259cab57c2c8e5e545fab/pyarrow-21.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:fee33b0ca46f4c85443d6c450357101e47d53e6c3f008d658c27a2d020d44c79", size = 32659470, upload-time = "2025-07-18T00:54:38.329Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/ed/b1589a777816ee33ba123ba1e4f8f02243a844fed0deec97bde9fb21a5cf/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:7be45519b830f7c24b21d630a31d48bcebfd5d4d7f9d3bdb49da9cdf6d764edb", size = 41055619, upload-time = "2025-07-18T00:54:42.172Z" },
+    { url = "https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:26bfd95f6bff443ceae63c65dc7e048670b7e98bc892210acba7e4995d3d4b51", size = 42733488, upload-time = "2025-07-18T00:54:47.132Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/cc/de02c3614874b9089c94eac093f90ca5dfa6d5afe45de3ba847fd950fdf1/pyarrow-21.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:bd04ec08f7f8bd113c55868bd3fc442a9db67c27af098c5f814a3091e71cc61a", size = 43329159, upload-time = "2025-07-18T00:54:51.686Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/3e/99473332ac40278f196e105ce30b79ab8affab12f6194802f2593d6b0be2/pyarrow-21.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9b0b14b49ac10654332a805aedfc0147fb3469cbf8ea951b3d040dab12372594", size = 45050567, upload-time = "2025-07-18T00:54:56.679Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/f5/c372ef60593d713e8bfbb7e0c743501605f0ad00719146dc075faf11172b/pyarrow-21.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:9d9f8bcb4c3be7738add259738abdeddc363de1b80e3310e04067aa1ca596634", size = 26217959, upload-time = "2025-07-18T00:55:00.482Z" },
+    { url = "https://files.pythonhosted.org/packages/94/dc/80564a3071a57c20b7c32575e4a0120e8a330ef487c319b122942d665960/pyarrow-21.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c077f48aab61738c237802836fc3844f85409a46015635198761b0d6a688f87b", size = 31243234, upload-time = "2025-07-18T00:55:03.812Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/cc/3b51cb2db26fe535d14f74cab4c79b191ed9a8cd4cbba45e2379b5ca2746/pyarrow-21.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:689f448066781856237eca8d1975b98cace19b8dd2ab6145bf49475478bcaa10", size = 32714370, upload-time = "2025-07-18T00:55:07.495Z" },
+    { url = "https://files.pythonhosted.org/packages/24/11/a4431f36d5ad7d83b87146f515c063e4d07ef0b7240876ddb885e6b44f2e/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:479ee41399fcddc46159a551705b89c05f11e8b8cb8e968f7fec64f62d91985e", size = 41135424, upload-time = "2025-07-18T00:55:11.461Z" },
+    { url = "https://files.pythonhosted.org/packages/74/dc/035d54638fc5d2971cbf1e987ccd45f1091c83bcf747281cf6cc25e72c88/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:40ebfcb54a4f11bcde86bc586cbd0272bac0d516cfa539c799c2453768477569", size = 42823810, upload-time = "2025-07-18T00:55:16.301Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/3b/89fced102448a9e3e0d4dded1f37fa3ce4700f02cdb8665457fcc8015f5b/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8d58d8497814274d3d20214fbb24abcad2f7e351474357d552a8d53bce70c70e", size = 43391538, upload-time = "2025-07-18T00:55:23.82Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/bb/ea7f1bd08978d39debd3b23611c293f64a642557e8141c80635d501e6d53/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:585e7224f21124dd57836b1530ac8f2df2afc43c861d7bf3d58a4870c42ae36c", size = 45120056, upload-time = "2025-07-18T00:55:28.231Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/0b/77ea0600009842b30ceebc3337639a7380cd946061b620ac1a2f3cb541e2/pyarrow-21.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:555ca6935b2cbca2c0e932bedd853e9bc523098c39636de9ad4693b5b1df86d6", size = 26220568, upload-time = "2025-07-18T00:55:32.122Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/d4/d4f817b21aacc30195cf6a46ba041dd1be827efa4a623cc8bf39a1c2a0c0/pyarrow-21.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:3a302f0e0963db37e0a24a70c56cf91a4faa0bca51c23812279ca2e23481fccd", size = 31160305, upload-time = "2025-07-18T00:55:35.373Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/9c/dcd38ce6e4b4d9a19e1d36914cb8e2b1da4e6003dd075474c4cfcdfe0601/pyarrow-21.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:b6b27cf01e243871390474a211a7922bfbe3bda21e39bc9160daf0da3fe48876", size = 32684264, upload-time = "2025-07-18T00:55:39.303Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/74/2a2d9f8d7a59b639523454bec12dba35ae3d0a07d8ab529dc0809f74b23c/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e72a8ec6b868e258a2cd2672d91f2860ad532d590ce94cdf7d5e7ec674ccf03d", size = 41108099, upload-time = "2025-07-18T00:55:42.889Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/90/2660332eeb31303c13b653ea566a9918484b6e4d6b9d2d46879a33ab0622/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b7ae0bbdc8c6674259b25bef5d2a1d6af5d39d7200c819cf99e07f7dfef1c51e", size = 42829529, upload-time = "2025-07-18T00:55:47.069Z" },
+    { url = "https://files.pythonhosted.org/packages/33/27/1a93a25c92717f6aa0fca06eb4700860577d016cd3ae51aad0e0488ac899/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:58c30a1729f82d201627c173d91bd431db88ea74dcaa3885855bc6203e433b82", size = 43367883, upload-time = "2025-07-18T00:55:53.069Z" },
+    { url = "https://files.pythonhosted.org/packages/05/d9/4d09d919f35d599bc05c6950095e358c3e15148ead26292dfca1fb659b0c/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:072116f65604b822a7f22945a7a6e581cfa28e3454fdcc6939d4ff6090126623", size = 45133802, upload-time = "2025-07-18T00:55:57.714Z" },
+    { url = "https://files.pythonhosted.org/packages/71/30/f3795b6e192c3ab881325ffe172e526499eb3780e306a15103a2764916a2/pyarrow-21.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:cf56ec8b0a5c8c9d7021d6fd754e688104f9ebebf1bf4449613c9531f5346a18", size = 26203175, upload-time = "2025-07-18T00:56:01.364Z" },
+    { url = "https://files.pythonhosted.org/packages/16/ca/c7eaa8e62db8fb37ce942b1ea0c6d7abfe3786ca193957afa25e71b81b66/pyarrow-21.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e99310a4ebd4479bcd1964dff9e14af33746300cb014aa4a3781738ac63baf4a", size = 31154306, upload-time = "2025-07-18T00:56:04.42Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/e8/e87d9e3b2489302b3a1aea709aaca4b781c5252fcb812a17ab6275a9a484/pyarrow-21.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:d2fe8e7f3ce329a71b7ddd7498b3cfac0eeb200c2789bd840234f0dc271a8efe", size = 32680622, upload-time = "2025-07-18T00:56:07.505Z" },
+    { url = "https://files.pythonhosted.org/packages/84/52/79095d73a742aa0aba370c7942b1b655f598069489ab387fe47261a849e1/pyarrow-21.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f522e5709379d72fb3da7785aa489ff0bb87448a9dc5a75f45763a795a089ebd", size = 41104094, upload-time = "2025-07-18T00:56:10.994Z" },
+    { url = "https://files.pythonhosted.org/packages/89/4b/7782438b551dbb0468892a276b8c789b8bbdb25ea5c5eb27faadd753e037/pyarrow-21.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:69cbbdf0631396e9925e048cfa5bce4e8c3d3b41562bbd70c685a8eb53a91e61", size = 42825576, upload-time = "2025-07-18T00:56:15.569Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/62/0f29de6e0a1e33518dec92c65be0351d32d7ca351e51ec5f4f837a9aab91/pyarrow-21.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:731c7022587006b755d0bdb27626a1a3bb004bb56b11fb30d98b6c1b4718579d", size = 43368342, upload-time = "2025-07-18T00:56:19.531Z" },
+    { url = "https://files.pythonhosted.org/packages/90/c7/0fa1f3f29cf75f339768cc698c8ad4ddd2481c1742e9741459911c9ac477/pyarrow-21.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc56bc708f2d8ac71bd1dcb927e458c93cec10b98eb4120206a4091db7b67b99", size = 45131218, upload-time = "2025-07-18T00:56:23.347Z" },
+    { url = "https://files.pythonhosted.org/packages/01/63/581f2076465e67b23bc5a37d4a2abff8362d389d29d8105832e82c9c811c/pyarrow-21.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:186aa00bca62139f75b7de8420f745f2af12941595bbbfa7ed3870ff63e25636", size = 26087551, upload-time = "2025-07-18T00:56:26.758Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/ab/357d0d9648bb8241ee7348e564f2479d206ebe6e1c47ac5027c2e31ecd39/pyarrow-21.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:a7a102574faa3f421141a64c10216e078df467ab9576684d5cd696952546e2da", size = 31290064, upload-time = "2025-07-18T00:56:30.214Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/8a/5685d62a990e4cac2043fc76b4661bf38d06efed55cf45a334b455bd2759/pyarrow-21.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:1e005378c4a2c6db3ada3ad4c217b381f6c886f0a80d6a316fe586b90f77efd7", size = 32727837, upload-time = "2025-07-18T00:56:33.935Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/de/c0828ee09525c2bafefd3e736a248ebe764d07d0fd762d4f0929dbc516c9/pyarrow-21.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:65f8e85f79031449ec8706b74504a316805217b35b6099155dd7e227eef0d4b6", size = 41014158, upload-time = "2025-07-18T00:56:37.528Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/26/a2865c420c50b7a3748320b614f3484bfcde8347b2639b2b903b21ce6a72/pyarrow-21.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:3a81486adc665c7eb1a2bde0224cfca6ceaba344a82a971ef059678417880eb8", size = 42667885, upload-time = "2025-07-18T00:56:41.483Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/f9/4ee798dc902533159250fb4321267730bc0a107d8c6889e07c3add4fe3a5/pyarrow-21.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fc0d2f88b81dcf3ccf9a6ae17f89183762c8a94a5bdcfa09e05cfe413acf0503", size = 43276625, upload-time = "2025-07-18T00:56:48.002Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/da/e02544d6997037a4b0d22d8e5f66bc9315c3671371a8b18c79ade1cefe14/pyarrow-21.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6299449adf89df38537837487a4f8d3bd91ec94354fdd2a7d30bc11c48ef6e79", size = 44951890, upload-time = "2025-07-18T00:56:52.568Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/4e/519c1bc1876625fe6b71e9a28287c43ec2f20f73c658b9ae1d485c0c206e/pyarrow-21.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:222c39e2c70113543982c6b34f3077962b44fca38c0bd9e68bb6781534425c10", size = 26371006, upload-time = "2025-07-18T00:56:56.379Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/cc/ce4939f4b316457a083dc5718b3982801e8c33f921b3c98e7a93b7c7491f/pyarrow-21.0.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:a7f6524e3747e35f80744537c78e7302cd41deee8baa668d56d55f77d9c464b3", size = 31211248, upload-time = "2025-07-18T00:56:59.7Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/c2/7a860931420d73985e2f340f06516b21740c15b28d24a0e99a900bb27d2b/pyarrow-21.0.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:203003786c9fd253ebcafa44b03c06983c9c8d06c3145e37f1b76a1f317aeae1", size = 32676896, upload-time = "2025-07-18T00:57:03.884Z" },
+    { url = "https://files.pythonhosted.org/packages/68/a8/197f989b9a75e59b4ca0db6a13c56f19a0ad8a298c68da9cc28145e0bb97/pyarrow-21.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:3b4d97e297741796fead24867a8dabf86c87e4584ccc03167e4a811f50fdf74d", size = 41067862, upload-time = "2025-07-18T00:57:07.587Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/82/6ecfa89487b35aa21accb014b64e0a6b814cc860d5e3170287bf5135c7d8/pyarrow-21.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:898afce396b80fdda05e3086b4256f8677c671f7b1d27a6976fa011d3fd0a86e", size = 42747508, upload-time = "2025-07-18T00:57:13.917Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/b7/ba252f399bbf3addc731e8643c05532cf32e74cebb5e32f8f7409bc243cf/pyarrow-21.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:067c66ca29aaedae08218569a114e413b26e742171f526e828e1064fcdec13f4", size = 43345293, upload-time = "2025-07-18T00:57:19.828Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/0a/a20819795bd702b9486f536a8eeb70a6aa64046fce32071c19ec8230dbaa/pyarrow-21.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0c4e75d13eb76295a49e0ea056eb18dbd87d81450bfeb8afa19a7e5a75ae2ad7", size = 45060670, upload-time = "2025-07-18T00:57:24.477Z" },
+    { url = "https://files.pythonhosted.org/packages/10/15/6b30e77872012bbfe8265d42a01d5b3c17ef0ac0f2fae531ad91b6a6c02e/pyarrow-21.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdc4c17afda4dab2a9c0b79148a43a7f4e1094916b3e18d8975bfd6d6d52241f", size = 26227521, upload-time = "2025-07-18T00:57:29.119Z" },
 ]
 
 [[package]]
@@ -1960,58 +2364,61 @@ version = "23.0.1"
 source = { registry = "https://pypi.org/simple" }
 resolution-markers = [
     "python_full_version >= '3.14'",
+    "python_full_version == '3.13.*'",
+    "python_full_version == '3.12.*'",
+    "python_full_version == '3.11.*'",
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/bc/a8/24e5dc6855f50a62936ceb004e6e9645e4219a8065f304145d7fb8a79d5d/pyarrow-23.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:3fab8f82571844eb3c460f90a75583801d14ca0cc32b1acc8c361650e006fd56", size = 34307390 },
-    { url = "https://files.pythonhosted.org/packages/bc/8e/4be5617b4aaae0287f621ad31c6036e5f63118cfca0dc57d42121ff49b51/pyarrow-23.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:3f91c038b95f71ddfc865f11d5876c42f343b4495535bd262c7b321b0b94507c", size = 35853761 },
-    { url = "https://files.pythonhosted.org/packages/2e/08/3e56a18819462210432ae37d10f5c8eed3828be1d6c751b6e6a2e93c286a/pyarrow-23.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:d0744403adabef53c985a7f8a082b502a368510c40d184df349a0a8754533258", size = 44493116 },
-    { url = "https://files.pythonhosted.org/packages/f8/82/c40b68001dbec8a3faa4c08cd8c200798ac732d2854537c5449dc859f55a/pyarrow-23.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c33b5bf406284fd0bba436ed6f6c3ebe8e311722b441d89397c54f871c6863a2", size = 47564532 },
-    { url = "https://files.pythonhosted.org/packages/20/bc/73f611989116b6f53347581b02177f9f620efdf3cd3f405d0e83cdf53a83/pyarrow-23.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ddf743e82f69dcd6dbbcb63628895d7161e04e56794ef80550ac6f3315eeb1d5", size = 48183685 },
-    { url = "https://files.pythonhosted.org/packages/b0/cc/6c6b3ecdae2a8c3aced99956187e8302fc954cc2cca2a37cf2111dad16ce/pyarrow-23.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e052a211c5ac9848ae15d5ec875ed0943c0221e2fcfe69eee80b604b4e703222", size = 50605582 },
-    { url = "https://files.pythonhosted.org/packages/8d/94/d359e708672878d7638a04a0448edf7c707f9e5606cee11e15aaa5c7535a/pyarrow-23.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:5abde149bb3ce524782d838eb67ac095cd3fd6090eba051130589793f1a7f76d", size = 27521148 },
-    { url = "https://files.pythonhosted.org/packages/b0/41/8e6b6ef7e225d4ceead8459427a52afdc23379768f54dd3566014d7618c1/pyarrow-23.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb", size = 34302230 },
-    { url = "https://files.pythonhosted.org/packages/bf/4a/1472c00392f521fea03ae93408bf445cc7bfa1ab81683faf9bc188e36629/pyarrow-23.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:0ae6e17c828455b6265d590100c295193f93cc5675eb0af59e49dbd00d2de350", size = 35850050 },
-    { url = "https://files.pythonhosted.org/packages/0c/b2/bd1f2f05ded56af7f54d702c8364c9c43cd6abb91b0e9933f3d77b4f4132/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fed7020203e9ef273360b9e45be52a2a47d3103caf156a30ace5247ffb51bdbd", size = 44491918 },
-    { url = "https://files.pythonhosted.org/packages/0b/62/96459ef5b67957eac38a90f541d1c28833d1b367f014a482cb63f3b7cd2d/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:26d50dee49d741ac0e82185033488d28d35be4d763ae6f321f97d1140eb7a0e9", size = 47562811 },
-    { url = "https://files.pythonhosted.org/packages/7d/94/1170e235add1f5f45a954e26cd0e906e7e74e23392dcb560de471f7366ec/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c30143b17161310f151f4a2bcfe41b5ff744238c1039338779424e38579d701", size = 48183766 },
-    { url = "https://files.pythonhosted.org/packages/0e/2d/39a42af4570377b99774cdb47f63ee6c7da7616bd55b3d5001aa18edfe4f/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db2190fa79c80a23fdd29fef4b8992893f024ae7c17d2f5f4db7171fa30c2c78", size = 50607669 },
-    { url = "https://files.pythonhosted.org/packages/00/ca/db94101c187f3df742133ac837e93b1f269ebdac49427f8310ee40b6a58f/pyarrow-23.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:f00f993a8179e0e1c9713bcc0baf6d6c01326a406a9c23495ec1ba9c9ebf2919", size = 27527698 },
-    { url = "https://files.pythonhosted.org/packages/9a/4b/4166bb5abbfe6f750fc60ad337c43ecf61340fa52ab386da6e8dbf9e63c4/pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f", size = 34214575 },
-    { url = "https://files.pythonhosted.org/packages/e1/da/3f941e3734ac8088ea588b53e860baeddac8323ea40ce22e3d0baa865cc9/pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7", size = 35832540 },
-    { url = "https://files.pythonhosted.org/packages/88/7c/3d841c366620e906d54430817531b877ba646310296df42ef697308c2705/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9", size = 44470940 },
-    { url = "https://files.pythonhosted.org/packages/2c/a5/da83046273d990f256cb79796a190bbf7ec999269705ddc609403f8c6b06/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05", size = 47586063 },
-    { url = "https://files.pythonhosted.org/packages/5b/3c/b7d2ebcff47a514f47f9da1e74b7949138c58cfeb108cdd4ee62f43f0cf3/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67", size = 48173045 },
-    { url = "https://files.pythonhosted.org/packages/43/b2/b40961262213beaba6acfc88698eb773dfce32ecdf34d19291db94c2bd73/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730", size = 50621741 },
-    { url = "https://files.pythonhosted.org/packages/f6/70/1fdda42d65b28b078e93d75d371b2185a61da89dda4def8ba6ba41ebdeb4/pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0", size = 27620678 },
-    { url = "https://files.pythonhosted.org/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066 },
-    { url = "https://files.pythonhosted.org/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526 },
-    { url = "https://files.pythonhosted.org/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279 },
-    { url = "https://files.pythonhosted.org/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798 },
-    { url = "https://files.pythonhosted.org/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446 },
-    { url = "https://files.pythonhosted.org/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972 },
-    { url = "https://files.pythonhosted.org/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749 },
-    { url = "https://files.pythonhosted.org/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544 },
-    { url = "https://files.pythonhosted.org/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911 },
-    { url = "https://files.pythonhosted.org/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337 },
-    { url = "https://files.pythonhosted.org/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944 },
-    { url = "https://files.pythonhosted.org/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269 },
-    { url = "https://files.pythonhosted.org/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794 },
-    { url = "https://files.pythonhosted.org/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642 },
-    { url = "https://files.pythonhosted.org/packages/8d/1b/6da9a89583ce7b23ac611f183ae4843cd3a6cf54f079549b0e8c14031e73/pyarrow-23.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5df1161da23636a70838099d4aaa65142777185cc0cdba4037a18cee7d8db9ca", size = 34238755 },
-    { url = "https://files.pythonhosted.org/packages/ae/b5/d58a241fbe324dbaeb8df07be6af8752c846192d78d2272e551098f74e88/pyarrow-23.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:fa8e51cb04b9f8c9c5ace6bab63af9a1f88d35c0d6cbf53e8c17c098552285e1", size = 35847826 },
-    { url = "https://files.pythonhosted.org/packages/54/a5/8cbc83f04aba433ca7b331b38f39e000efd9f0c7ce47128670e737542996/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b95a3994f015be13c63148fef8832e8a23938128c185ee951c98908a696e0eb", size = 44536859 },
-    { url = "https://files.pythonhosted.org/packages/36/2e/c0f017c405fcdc252dbccafbe05e36b0d0eb1ea9a958f081e01c6972927f/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4982d71350b1a6e5cfe1af742c53dfb759b11ce14141870d05d9e540d13bc5d1", size = 47614443 },
-    { url = "https://files.pythonhosted.org/packages/af/6b/2314a78057912f5627afa13ba43809d9d653e6630859618b0fd81a4e0759/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c250248f1fe266db627921c89b47b7c06fee0489ad95b04d50353537d74d6886", size = 48232991 },
-    { url = "https://files.pythonhosted.org/packages/40/f2/1bcb1d3be3460832ef3370d621142216e15a2c7c62602a4ea19ec240dd64/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f4763b83c11c16e5f4c15601ba6dfa849e20723b46aa2617cb4bffe8768479f", size = 50645077 },
-    { url = "https://files.pythonhosted.org/packages/eb/3f/b1da7b61cd66566a4d4c8383d376c606d1c34a906c3f1cb35c479f59d1aa/pyarrow-23.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:3a4c85ef66c134161987c17b147d6bffdca4566f9a4c1d81a0a01cdf08414ea5", size = 28234271 },
-    { url = "https://files.pythonhosted.org/packages/b5/78/07f67434e910a0f7323269be7bfbf58699bd0c1d080b18a1ab49ba943fe8/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:17cd28e906c18af486a499422740298c52d7c6795344ea5002a7720b4eadf16d", size = 34488692 },
-    { url = "https://files.pythonhosted.org/packages/50/76/34cf7ae93ece1f740a04910d9f7e80ba166b9b4ab9596a953e9e62b90fe1/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:76e823d0e86b4fb5e1cf4a58d293036e678b5a4b03539be933d3b31f9406859f", size = 35964383 },
-    { url = "https://files.pythonhosted.org/packages/46/90/459b827238936d4244214be7c684e1b366a63f8c78c380807ae25ed92199/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a62e1899e3078bf65943078b3ad2a6ddcacf2373bc06379aac61b1e548a75814", size = 44538119 },
-    { url = "https://files.pythonhosted.org/packages/28/a1/93a71ae5881e99d1f9de1d4554a87be37da11cd6b152239fb5bd924fdc64/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:df088e8f640c9fae3b1f495b3c64755c4e719091caf250f3a74d095ddf3c836d", size = 47571199 },
-    { url = "https://files.pythonhosted.org/packages/88/a3/d2c462d4ef313521eaf2eff04d204ac60775263f1fb08c374b543f79f610/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:46718a220d64677c93bc243af1d44b55998255427588e400677d7192671845c7", size = 48259435 },
-    { url = "https://files.pythonhosted.org/packages/cc/f1/11a544b8c3d38a759eb3fbb022039117fd633e9a7b19e4841cc3da091915/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a09f3876e87f48bc2f13583ab551f0379e5dfb83210391e68ace404181a20690", size = 50629149 },
-    { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807 },
+sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/bc/a8/24e5dc6855f50a62936ceb004e6e9645e4219a8065f304145d7fb8a79d5d/pyarrow-23.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:3fab8f82571844eb3c460f90a75583801d14ca0cc32b1acc8c361650e006fd56", size = 34307390, upload-time = "2026-02-16T10:08:08.654Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/8e/4be5617b4aaae0287f621ad31c6036e5f63118cfca0dc57d42121ff49b51/pyarrow-23.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:3f91c038b95f71ddfc865f11d5876c42f343b4495535bd262c7b321b0b94507c", size = 35853761, upload-time = "2026-02-16T10:08:17.811Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/08/3e56a18819462210432ae37d10f5c8eed3828be1d6c751b6e6a2e93c286a/pyarrow-23.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:d0744403adabef53c985a7f8a082b502a368510c40d184df349a0a8754533258", size = 44493116, upload-time = "2026-02-16T10:08:25.792Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/82/c40b68001dbec8a3faa4c08cd8c200798ac732d2854537c5449dc859f55a/pyarrow-23.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c33b5bf406284fd0bba436ed6f6c3ebe8e311722b441d89397c54f871c6863a2", size = 47564532, upload-time = "2026-02-16T10:08:34.27Z" },
+    { url = "https://files.pythonhosted.org/packages/20/bc/73f611989116b6f53347581b02177f9f620efdf3cd3f405d0e83cdf53a83/pyarrow-23.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ddf743e82f69dcd6dbbcb63628895d7161e04e56794ef80550ac6f3315eeb1d5", size = 48183685, upload-time = "2026-02-16T10:08:42.889Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/cc/6c6b3ecdae2a8c3aced99956187e8302fc954cc2cca2a37cf2111dad16ce/pyarrow-23.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e052a211c5ac9848ae15d5ec875ed0943c0221e2fcfe69eee80b604b4e703222", size = 50605582, upload-time = "2026-02-16T10:08:51.641Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/94/d359e708672878d7638a04a0448edf7c707f9e5606cee11e15aaa5c7535a/pyarrow-23.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:5abde149bb3ce524782d838eb67ac095cd3fd6090eba051130589793f1a7f76d", size = 27521148, upload-time = "2026-02-16T10:08:58.077Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/41/8e6b6ef7e225d4ceead8459427a52afdc23379768f54dd3566014d7618c1/pyarrow-23.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb", size = 34302230, upload-time = "2026-02-16T10:09:03.859Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/4a/1472c00392f521fea03ae93408bf445cc7bfa1ab81683faf9bc188e36629/pyarrow-23.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:0ae6e17c828455b6265d590100c295193f93cc5675eb0af59e49dbd00d2de350", size = 35850050, upload-time = "2026-02-16T10:09:11.877Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/b2/bd1f2f05ded56af7f54d702c8364c9c43cd6abb91b0e9933f3d77b4f4132/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fed7020203e9ef273360b9e45be52a2a47d3103caf156a30ace5247ffb51bdbd", size = 44491918, upload-time = "2026-02-16T10:09:18.144Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/62/96459ef5b67957eac38a90f541d1c28833d1b367f014a482cb63f3b7cd2d/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:26d50dee49d741ac0e82185033488d28d35be4d763ae6f321f97d1140eb7a0e9", size = 47562811, upload-time = "2026-02-16T10:09:25.792Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/94/1170e235add1f5f45a954e26cd0e906e7e74e23392dcb560de471f7366ec/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c30143b17161310f151f4a2bcfe41b5ff744238c1039338779424e38579d701", size = 48183766, upload-time = "2026-02-16T10:09:34.645Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/2d/39a42af4570377b99774cdb47f63ee6c7da7616bd55b3d5001aa18edfe4f/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db2190fa79c80a23fdd29fef4b8992893f024ae7c17d2f5f4db7171fa30c2c78", size = 50607669, upload-time = "2026-02-16T10:09:44.153Z" },
+    { url = "https://files.pythonhosted.org/packages/00/ca/db94101c187f3df742133ac837e93b1f269ebdac49427f8310ee40b6a58f/pyarrow-23.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:f00f993a8179e0e1c9713bcc0baf6d6c01326a406a9c23495ec1ba9c9ebf2919", size = 27527698, upload-time = "2026-02-16T10:09:50.263Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/4b/4166bb5abbfe6f750fc60ad337c43ecf61340fa52ab386da6e8dbf9e63c4/pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f", size = 34214575, upload-time = "2026-02-16T10:09:56.225Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/da/3f941e3734ac8088ea588b53e860baeddac8323ea40ce22e3d0baa865cc9/pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7", size = 35832540, upload-time = "2026-02-16T10:10:03.428Z" },
+    { url = "https://files.pythonhosted.org/packages/88/7c/3d841c366620e906d54430817531b877ba646310296df42ef697308c2705/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9", size = 44470940, upload-time = "2026-02-16T10:10:10.704Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/a5/da83046273d990f256cb79796a190bbf7ec999269705ddc609403f8c6b06/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05", size = 47586063, upload-time = "2026-02-16T10:10:17.95Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/3c/b7d2ebcff47a514f47f9da1e74b7949138c58cfeb108cdd4ee62f43f0cf3/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67", size = 48173045, upload-time = "2026-02-16T10:10:25.363Z" },
+    { url = "https://files.pythonhosted.org/packages/43/b2/b40961262213beaba6acfc88698eb773dfce32ecdf34d19291db94c2bd73/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730", size = 50621741, upload-time = "2026-02-16T10:10:33.477Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/70/1fdda42d65b28b078e93d75d371b2185a61da89dda4def8ba6ba41ebdeb4/pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0", size = 27620678, upload-time = "2026-02-16T10:10:39.31Z" },
+    { url = "https://files.pythonhosted.org/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066, upload-time = "2026-02-16T10:10:45.487Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526, upload-time = "2026-02-16T10:10:52.266Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279, upload-time = "2026-02-16T10:11:01.557Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798, upload-time = "2026-02-16T10:11:09.401Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446, upload-time = "2026-02-16T10:11:17.781Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972, upload-time = "2026-02-16T10:11:26.185Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749, upload-time = "2026-02-16T10:12:23.297Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544, upload-time = "2026-02-16T10:11:32.535Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911, upload-time = "2026-02-16T10:11:39.813Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337, upload-time = "2026-02-16T10:11:47.764Z" },
+    { url = "https://files.pythonhosted.org/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944, upload-time = "2026-02-16T10:11:56.607Z" },
+    { url = "https://files.pythonhosted.org/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269, upload-time = "2026-02-16T10:12:04.47Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794, upload-time = "2026-02-16T10:12:11.797Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642, upload-time = "2026-02-16T10:12:17.746Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/1b/6da9a89583ce7b23ac611f183ae4843cd3a6cf54f079549b0e8c14031e73/pyarrow-23.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5df1161da23636a70838099d4aaa65142777185cc0cdba4037a18cee7d8db9ca", size = 34238755, upload-time = "2026-02-16T10:12:32.819Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/b5/d58a241fbe324dbaeb8df07be6af8752c846192d78d2272e551098f74e88/pyarrow-23.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:fa8e51cb04b9f8c9c5ace6bab63af9a1f88d35c0d6cbf53e8c17c098552285e1", size = 35847826, upload-time = "2026-02-16T10:12:38.949Z" },
+    { url = "https://files.pythonhosted.org/packages/54/a5/8cbc83f04aba433ca7b331b38f39e000efd9f0c7ce47128670e737542996/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b95a3994f015be13c63148fef8832e8a23938128c185ee951c98908a696e0eb", size = 44536859, upload-time = "2026-02-16T10:12:45.467Z" },
+    { url = "https://files.pythonhosted.org/packages/36/2e/c0f017c405fcdc252dbccafbe05e36b0d0eb1ea9a958f081e01c6972927f/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4982d71350b1a6e5cfe1af742c53dfb759b11ce14141870d05d9e540d13bc5d1", size = 47614443, upload-time = "2026-02-16T10:12:55.525Z" },
+    { url = "https://files.pythonhosted.org/packages/af/6b/2314a78057912f5627afa13ba43809d9d653e6630859618b0fd81a4e0759/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c250248f1fe266db627921c89b47b7c06fee0489ad95b04d50353537d74d6886", size = 48232991, upload-time = "2026-02-16T10:13:04.729Z" },
+    { url = "https://files.pythonhosted.org/packages/40/f2/1bcb1d3be3460832ef3370d621142216e15a2c7c62602a4ea19ec240dd64/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f4763b83c11c16e5f4c15601ba6dfa849e20723b46aa2617cb4bffe8768479f", size = 50645077, upload-time = "2026-02-16T10:13:14.147Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/3f/b1da7b61cd66566a4d4c8383d376c606d1c34a906c3f1cb35c479f59d1aa/pyarrow-23.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:3a4c85ef66c134161987c17b147d6bffdca4566f9a4c1d81a0a01cdf08414ea5", size = 28234271, upload-time = "2026-02-16T10:14:09.397Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/78/07f67434e910a0f7323269be7bfbf58699bd0c1d080b18a1ab49ba943fe8/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:17cd28e906c18af486a499422740298c52d7c6795344ea5002a7720b4eadf16d", size = 34488692, upload-time = "2026-02-16T10:13:21.541Z" },
+    { url = "https://files.pythonhosted.org/packages/50/76/34cf7ae93ece1f740a04910d9f7e80ba166b9b4ab9596a953e9e62b90fe1/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:76e823d0e86b4fb5e1cf4a58d293036e678b5a4b03539be933d3b31f9406859f", size = 35964383, upload-time = "2026-02-16T10:13:28.63Z" },
+    { url = "https://files.pythonhosted.org/packages/46/90/459b827238936d4244214be7c684e1b366a63f8c78c380807ae25ed92199/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a62e1899e3078bf65943078b3ad2a6ddcacf2373bc06379aac61b1e548a75814", size = 44538119, upload-time = "2026-02-16T10:13:35.506Z" },
+    { url = "https://files.pythonhosted.org/packages/28/a1/93a71ae5881e99d1f9de1d4554a87be37da11cd6b152239fb5bd924fdc64/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:df088e8f640c9fae3b1f495b3c64755c4e719091caf250f3a74d095ddf3c836d", size = 47571199, upload-time = "2026-02-16T10:13:42.504Z" },
+    { url = "https://files.pythonhosted.org/packages/88/a3/d2c462d4ef313521eaf2eff04d204ac60775263f1fb08c374b543f79f610/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:46718a220d64677c93bc243af1d44b55998255427588e400677d7192671845c7", size = 48259435, upload-time = "2026-02-16T10:13:49.226Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/f1/11a544b8c3d38a759eb3fbb022039117fd633e9a7b19e4841cc3da091915/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a09f3876e87f48bc2f13583ab551f0379e5dfb83210391e68ace404181a20690", size = 50629149, upload-time = "2026-02-16T10:13:57.238Z" },
+    { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" },
 ]
 
 [[package]]
@@ -2024,9 +2431,9 @@ dependencies = [
     { name = "typing-extensions" },
     { name = "typing-inspection" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/96/ad/a17bc283d7d81837c061c49e3eaa27a45991759a1b7eae1031921c6bd924/pydantic-2.12.4.tar.gz", hash = "sha256:0f8cb9555000a4b5b617f66bfd2566264c4984b27589d3b845685983e8ea85ac", size = 821038 }
+sdist = { url = "https://files.pythonhosted.org/packages/96/ad/a17bc283d7d81837c061c49e3eaa27a45991759a1b7eae1031921c6bd924/pydantic-2.12.4.tar.gz", hash = "sha256:0f8cb9555000a4b5b617f66bfd2566264c4984b27589d3b845685983e8ea85ac", size = 821038, upload-time = "2025-11-05T10:50:08.59Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/82/2f/e68750da9b04856e2a7ec56fc6f034a5a79775e9b9a81882252789873798/pydantic-2.12.4-py3-none-any.whl", hash = "sha256:92d3d202a745d46f9be6df459ac5a064fdaa3c1c4cd8adcfa332ccf3c05f871e", size = 463400 },
+    { url = "https://files.pythonhosted.org/packages/82/2f/e68750da9b04856e2a7ec56fc6f034a5a79775e9b9a81882252789873798/pydantic-2.12.4-py3-none-any.whl", hash = "sha256:92d3d202a745d46f9be6df459ac5a064fdaa3c1c4cd8adcfa332ccf3c05f871e", size = 463400, upload-time = "2025-11-05T10:50:06.732Z" },
 ]
 
 [[package]]
@@ -2036,116 +2443,137 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/c6/90/32c9941e728d564b411d574d8ee0cf09b12ec978cb22b294995bae5549a5/pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146", size = 2107298 },
-    { url = "https://files.pythonhosted.org/packages/fb/a8/61c96a77fe28993d9a6fb0f4127e05430a267b235a124545d79fea46dd65/pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2", size = 1901475 },
-    { url = "https://files.pythonhosted.org/packages/5d/b6/338abf60225acc18cdc08b4faef592d0310923d19a87fba1faf05af5346e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97", size = 1918815 },
-    { url = "https://files.pythonhosted.org/packages/d1/1c/2ed0433e682983d8e8cba9c8d8ef274d4791ec6a6f24c58935b90e780e0a/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9", size = 2065567 },
-    { url = "https://files.pythonhosted.org/packages/b3/24/cf84974ee7d6eae06b9e63289b7b8f6549d416b5c199ca2d7ce13bbcf619/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52", size = 2230442 },
-    { url = "https://files.pythonhosted.org/packages/fd/21/4e287865504b3edc0136c89c9c09431be326168b1eb7841911cbc877a995/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941", size = 2350956 },
-    { url = "https://files.pythonhosted.org/packages/a8/76/7727ef2ffa4b62fcab916686a68a0426b9b790139720e1934e8ba797e238/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a", size = 2068253 },
-    { url = "https://files.pythonhosted.org/packages/d5/8c/a4abfc79604bcb4c748e18975c44f94f756f08fb04218d5cb87eb0d3a63e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c", size = 2177050 },
-    { url = "https://files.pythonhosted.org/packages/67/b1/de2e9a9a79b480f9cb0b6e8b6ba4c50b18d4e89852426364c66aa82bb7b3/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2", size = 2147178 },
-    { url = "https://files.pythonhosted.org/packages/16/c1/dfb33f837a47b20417500efaa0378adc6635b3c79e8369ff7a03c494b4ac/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556", size = 2341833 },
-    { url = "https://files.pythonhosted.org/packages/47/36/00f398642a0f4b815a9a558c4f1dca1b4020a7d49562807d7bc9ff279a6c/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49", size = 2321156 },
-    { url = "https://files.pythonhosted.org/packages/7e/70/cad3acd89fde2010807354d978725ae111ddf6d0ea46d1ea1775b5c1bd0c/pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba", size = 1989378 },
-    { url = "https://files.pythonhosted.org/packages/76/92/d338652464c6c367e5608e4488201702cd1cbb0f33f7b6a85a60fe5f3720/pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9", size = 2013622 },
-    { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873 },
-    { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826 },
-    { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869 },
-    { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890 },
-    { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740 },
-    { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021 },
-    { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378 },
-    { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761 },
-    { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303 },
-    { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355 },
-    { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875 },
-    { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549 },
-    { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305 },
-    { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902 },
-    { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990 },
-    { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003 },
-    { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200 },
-    { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578 },
-    { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504 },
-    { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816 },
-    { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366 },
-    { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698 },
-    { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603 },
-    { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591 },
-    { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068 },
-    { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908 },
-    { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145 },
-    { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179 },
-    { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403 },
-    { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206 },
-    { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307 },
-    { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258 },
-    { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917 },
-    { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186 },
-    { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164 },
-    { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146 },
-    { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788 },
-    { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133 },
-    { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852 },
-    { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679 },
-    { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766 },
-    { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005 },
-    { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622 },
-    { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725 },
-    { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040 },
-    { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691 },
-    { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897 },
-    { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302 },
-    { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877 },
-    { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680 },
-    { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960 },
-    { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102 },
-    { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039 },
-    { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126 },
-    { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489 },
-    { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288 },
-    { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255 },
-    { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760 },
-    { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092 },
-    { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385 },
-    { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832 },
-    { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585 },
-    { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078 },
-    { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914 },
-    { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560 },
-    { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244 },
-    { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955 },
-    { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906 },
-    { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607 },
-    { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769 },
-    { url = "https://files.pythonhosted.org/packages/e6/b0/1a2aa41e3b5a4ba11420aba2d091b2d17959c8d1519ece3627c371951e73/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8", size = 2103351 },
-    { url = "https://files.pythonhosted.org/packages/a4/ee/31b1f0020baaf6d091c87900ae05c6aeae101fa4e188e1613c80e4f1ea31/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a", size = 1925363 },
-    { url = "https://files.pythonhosted.org/packages/e1/89/ab8e86208467e467a80deaca4e434adac37b10a9d134cd2f99b28a01e483/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b", size = 2135615 },
-    { url = "https://files.pythonhosted.org/packages/99/0a/99a53d06dd0348b2008f2f30884b34719c323f16c3be4e6cc1203b74a91d/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2", size = 2175369 },
-    { url = "https://files.pythonhosted.org/packages/6d/94/30ca3b73c6d485b9bb0bc66e611cff4a7138ff9736b7e66bcf0852151636/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093", size = 2144218 },
-    { url = "https://files.pythonhosted.org/packages/87/57/31b4f8e12680b739a91f472b5671294236b82586889ef764b5fbc6669238/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a", size = 2329951 },
-    { url = "https://files.pythonhosted.org/packages/7d/73/3c2c8edef77b8f7310e6fb012dbc4b8551386ed575b9eb6fb2506e28a7eb/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963", size = 2318428 },
-    { url = "https://files.pythonhosted.org/packages/2f/02/8559b1f26ee0d502c74f9cca5c0d2fd97e967e083e006bbbb4e97f3a043a/pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a", size = 2147009 },
-    { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980 },
-    { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865 },
-    { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256 },
-    { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762 },
-    { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141 },
-    { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317 },
-    { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992 },
-    { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302 },
+sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c6/90/32c9941e728d564b411d574d8ee0cf09b12ec978cb22b294995bae5549a5/pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146", size = 2107298, upload-time = "2025-11-04T13:39:04.116Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/a8/61c96a77fe28993d9a6fb0f4127e05430a267b235a124545d79fea46dd65/pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2", size = 1901475, upload-time = "2025-11-04T13:39:06.055Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/b6/338abf60225acc18cdc08b4faef592d0310923d19a87fba1faf05af5346e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97", size = 1918815, upload-time = "2025-11-04T13:39:10.41Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/1c/2ed0433e682983d8e8cba9c8d8ef274d4791ec6a6f24c58935b90e780e0a/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9", size = 2065567, upload-time = "2025-11-04T13:39:12.244Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/24/cf84974ee7d6eae06b9e63289b7b8f6549d416b5c199ca2d7ce13bbcf619/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52", size = 2230442, upload-time = "2025-11-04T13:39:13.962Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/21/4e287865504b3edc0136c89c9c09431be326168b1eb7841911cbc877a995/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941", size = 2350956, upload-time = "2025-11-04T13:39:15.889Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/76/7727ef2ffa4b62fcab916686a68a0426b9b790139720e1934e8ba797e238/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a", size = 2068253, upload-time = "2025-11-04T13:39:17.403Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/8c/a4abfc79604bcb4c748e18975c44f94f756f08fb04218d5cb87eb0d3a63e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c", size = 2177050, upload-time = "2025-11-04T13:39:19.351Z" },
+    { url = "https://files.pythonhosted.org/packages/67/b1/de2e9a9a79b480f9cb0b6e8b6ba4c50b18d4e89852426364c66aa82bb7b3/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2", size = 2147178, upload-time = "2025-11-04T13:39:21Z" },
+    { url = "https://files.pythonhosted.org/packages/16/c1/dfb33f837a47b20417500efaa0378adc6635b3c79e8369ff7a03c494b4ac/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556", size = 2341833, upload-time = "2025-11-04T13:39:22.606Z" },
+    { url = "https://files.pythonhosted.org/packages/47/36/00f398642a0f4b815a9a558c4f1dca1b4020a7d49562807d7bc9ff279a6c/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49", size = 2321156, upload-time = "2025-11-04T13:39:25.843Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/70/cad3acd89fde2010807354d978725ae111ddf6d0ea46d1ea1775b5c1bd0c/pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba", size = 1989378, upload-time = "2025-11-04T13:39:27.92Z" },
+    { url = "https://files.pythonhosted.org/packages/76/92/d338652464c6c367e5608e4488201702cd1cbb0f33f7b6a85a60fe5f3720/pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9", size = 2013622, upload-time = "2025-11-04T13:39:29.848Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" },
+    { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" },
+    { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" },
+    { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" },
+    { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" },
+    { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" },
+    { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" },
+    { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" },
+    { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" },
+    { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" },
+    { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" },
+    { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" },
+    { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" },
+    { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" },
+    { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" },
+    { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" },
+    { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" },
+    { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" },
+    { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" },
+    { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" },
+    { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" },
+    { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" },
+    { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" },
+    { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" },
+    { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" },
+    { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" },
+    { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" },
+    { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" },
+    { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" },
+    { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" },
+    { url = "https://files.pythonhosted.org/packages/54/db/160dffb57ed9a3705c4cbcbff0ac03bdae45f1ca7d58ab74645550df3fbd/pydantic_core-2.41.5-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8bfeaf8735be79f225f3fefab7f941c712aaca36f1128c9d7e2352ee1aa87bdf", size = 2107999, upload-time = "2025-11-04T13:42:03.885Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/7d/88e7de946f60d9263cc84819f32513520b85c0f8322f9b8f6e4afc938383/pydantic_core-2.41.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:346285d28e4c8017da95144c7f3acd42740d637ff41946af5ce6e5e420502dd5", size = 1929745, upload-time = "2025-11-04T13:42:06.075Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/c2/aef51e5b283780e85e99ff19db0f05842d2d4a8a8cd15e63b0280029b08f/pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a75dafbf87d6276ddc5b2bf6fae5254e3d0876b626eb24969a574fff9149ee5d", size = 1920220, upload-time = "2025-11-04T13:42:08.457Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/97/492ab10f9ac8695cd76b2fdb24e9e61f394051df71594e9bcc891c9f586e/pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7b93a4d08587e2b7e7882de461e82b6ed76d9026ce91ca7915e740ecc7855f60", size = 2067296, upload-time = "2025-11-04T13:42:10.817Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/23/984149650e5269c59a2a4c41d234a9570adc68ab29981825cfaf4cfad8f4/pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8465ab91a4bd96d36dde3263f06caa6a8a6019e4113f24dc753d79a8b3a3f82", size = 2231548, upload-time = "2025-11-04T13:42:13.843Z" },
+    { url = "https://files.pythonhosted.org/packages/71/0c/85bcbb885b9732c28bec67a222dbed5ed2d77baee1f8bba2002e8cd00c5c/pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:299e0a22e7ae2b85c1a57f104538b2656e8ab1873511fd718a1c1c6f149b77b5", size = 2362571, upload-time = "2025-11-04T13:42:16.208Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/4a/412d2048be12c334003e9b823a3fa3d038e46cc2d64dd8aab50b31b65499/pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:707625ef0983fcfb461acfaf14de2067c5942c6bb0f3b4c99158bed6fedd3cf3", size = 2068175, upload-time = "2025-11-04T13:42:18.911Z" },
+    { url = "https://files.pythonhosted.org/packages/73/f4/c58b6a776b502d0a5540ad02e232514285513572060f0d78f7832ca3c98b/pydantic_core-2.41.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f41eb9797986d6ebac5e8edff36d5cef9de40def462311b3eb3eeded1431e425", size = 2177203, upload-time = "2025-11-04T13:42:22.578Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/ae/f06ea4c7e7a9eead3d165e7623cd2ea0cb788e277e4f935af63fc98fa4e6/pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0384e2e1021894b1ff5a786dbf94771e2986ebe2869533874d7e43bc79c6f504", size = 2148191, upload-time = "2025-11-04T13:42:24.89Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/57/25a11dcdc656bf5f8b05902c3c2934ac3ea296257cc4a3f79a6319e61856/pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:f0cd744688278965817fd0839c4a4116add48d23890d468bc436f78beb28abf5", size = 2343907, upload-time = "2025-11-04T13:42:27.683Z" },
+    { url = "https://files.pythonhosted.org/packages/96/82/e33d5f4933d7a03327c0c43c65d575e5919d4974ffc026bc917a5f7b9f61/pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:753e230374206729bf0a807954bcc6c150d3743928a73faffee51ac6557a03c3", size = 2322174, upload-time = "2025-11-04T13:42:30.776Z" },
+    { url = "https://files.pythonhosted.org/packages/81/45/4091be67ce9f469e81656f880f3506f6a5624121ec5eb3eab37d7581897d/pydantic_core-2.41.5-cp39-cp39-win32.whl", hash = "sha256:873e0d5b4fb9b89ef7c2d2a963ea7d02879d9da0da8d9d4933dee8ee86a8b460", size = 1990353, upload-time = "2025-11-04T13:42:33.111Z" },
+    { url = "https://files.pythonhosted.org/packages/44/8a/a98aede18db6e9cd5d66bcacd8a409fcf8134204cdede2e7de35c5a2c5ef/pydantic_core-2.41.5-cp39-cp39-win_amd64.whl", hash = "sha256:e4f4a984405e91527a0d62649ee21138f8e3d0ef103be488c1dc11a80d7f184b", size = 2015698, upload-time = "2025-11-04T13:42:35.484Z" },
+    { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" },
+    { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" },
+    { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/b0/1a2aa41e3b5a4ba11420aba2d091b2d17959c8d1519ece3627c371951e73/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8", size = 2103351, upload-time = "2025-11-04T13:43:02.058Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/ee/31b1f0020baaf6d091c87900ae05c6aeae101fa4e188e1613c80e4f1ea31/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a", size = 1925363, upload-time = "2025-11-04T13:43:05.159Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/89/ab8e86208467e467a80deaca4e434adac37b10a9d134cd2f99b28a01e483/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b", size = 2135615, upload-time = "2025-11-04T13:43:08.116Z" },
+    { url = "https://files.pythonhosted.org/packages/99/0a/99a53d06dd0348b2008f2f30884b34719c323f16c3be4e6cc1203b74a91d/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2", size = 2175369, upload-time = "2025-11-04T13:43:12.49Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/94/30ca3b73c6d485b9bb0bc66e611cff4a7138ff9736b7e66bcf0852151636/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093", size = 2144218, upload-time = "2025-11-04T13:43:15.431Z" },
+    { url = "https://files.pythonhosted.org/packages/87/57/31b4f8e12680b739a91f472b5671294236b82586889ef764b5fbc6669238/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a", size = 2329951, upload-time = "2025-11-04T13:43:18.062Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/73/3c2c8edef77b8f7310e6fb012dbc4b8551386ed575b9eb6fb2506e28a7eb/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963", size = 2318428, upload-time = "2025-11-04T13:43:20.679Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/02/8559b1f26ee0d502c74f9cca5c0d2fd97e967e083e006bbbb4e97f3a043a/pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a", size = 2147009, upload-time = "2025-11-04T13:43:23.286Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" },
+    { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" },
+    { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" },
+    { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" },
+    { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" },
 ]
 
 [[package]]
 name = "pygments"
 version = "2.19.2"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631 }
+sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217 },
+    { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" },
 ]
 
 [[package]]
@@ -2153,10 +2581,11 @@ name = "pylance"
 source = { editable = "." }
 dependencies = [
     { name = "lance-namespace" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" },
-    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" },
+    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
 ]
 
 [package.optional-dependencies]
@@ -2173,8 +2602,9 @@ geo = [
 ]
 tests = [
     { name = "boto3" },
-    { name = "datafusion" },
-    { name = "datasets" },
+    { name = "datafusion", marker = "python_full_version >= '3.10'" },
+    { name = "datasets", version = "0.0.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "datasets", version = "4.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
     { name = "duckdb" },
     { name = "ml-dtypes" },
     { name = "pandas" },
@@ -2182,7 +2612,8 @@ tests = [
     { name = "polars", extra = ["pandas", "pyarrow"] },
     { name = "psutil" },
     { name = "pytest" },
-    { name = "tensorflow", marker = "sys_platform == 'linux'" },
+    { name = "tensorflow", version = "2.7.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and sys_platform == 'linux'" },
+    { name = "tensorflow", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and sys_platform == 'linux'" },
     { name = "tqdm" },
 ]
 torch = [
@@ -2192,12 +2623,12 @@ torch = [
 [package.metadata]
 requires-dist = [
     { name = "boto3", marker = "extra == 'tests'" },
-    { name = "datafusion", marker = "extra == 'tests'", specifier = ">=52,<53" },
+    { name = "datafusion", marker = "python_full_version >= '3.10' and extra == 'tests'", specifier = ">=52,<53" },
     { name = "datasets", marker = "extra == 'tests'" },
     { name = "duckdb", marker = "extra == 'tests'" },
     { name = "geoarrow-rust-core", marker = "extra == 'geo'" },
     { name = "geoarrow-rust-io", marker = "extra == 'geo'" },
-    { name = "lance-namespace", specifier = ">=0.5.2" },
+    { name = "lance-namespace", specifier = ">=0.6.1" },
     { name = "ml-dtypes", marker = "extra == 'tests'" },
     { name = "numpy", specifier = ">=1.22" },
     { name = "pandas", marker = "extra == 'tests'" },
@@ -2208,56 +2639,97 @@ requires-dist = [
     { name = "pyright", marker = "extra == 'dev'" },
     { name = "pytest", marker = "extra == 'tests'" },
     { name = "pytest-benchmark", marker = "extra == 'benchmarks'" },
-    { name = "ruff", marker = "extra == 'dev'", specifier = "==0.4.1" },
+    { name = "ruff", marker = "extra == 'dev'", specifier = "==0.11.2" },
     { name = "tensorflow", marker = "sys_platform == 'linux' and extra == 'tests'" },
     { name = "torch", marker = "extra == 'torch'", specifier = ">=2.0" },
     { name = "tqdm", marker = "extra == 'tests'" },
 ]
+provides-extras = ["benchmarks", "dev", "geo", "tests", "torch"]
+
+[[package]]
+name = "pyproj"
+version = "3.6.1"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+dependencies = [
+    { name = "certifi", marker = "python_full_version < '3.10'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/7d/84/2b39bbf888c753ea48b40d47511548c77aa03445465c35cc4c4e9649b643/pyproj-3.6.1.tar.gz", hash = "sha256:44aa7c704c2b7d8fb3d483bbf75af6cb2350d30a63b144279a09b75fead501bf", size = 225131, upload-time = "2023-09-21T02:07:51.593Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c5/32/63cf474f4a8d4804b3bdf7c16b8589f38142e8e2f8319dcea27e0bc21a87/pyproj-3.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ab7aa4d9ff3c3acf60d4b285ccec134167a948df02347585fdd934ebad8811b4", size = 6142763, upload-time = "2023-09-21T02:07:12.844Z" },
+    { url = "https://files.pythonhosted.org/packages/18/86/2e7cb9de40492f1bafbf11f4c9072edc394509a40b5e4c52f8139546f039/pyproj-3.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4bc0472302919e59114aa140fd7213c2370d848a7249d09704f10f5b062031fe", size = 4877123, upload-time = "2023-09-21T02:10:37.905Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/c5/928d5a26995dbefbebd7507d982141cd9153bc7e4392b334fff722c4af12/pyproj-3.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5279586013b8d6582e22b6f9e30c49796966770389a9d5b85e25a4223286cd3f", size = 6190576, upload-time = "2023-09-21T02:17:08.637Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/2b/b60cf73b0720abca313bfffef34e34f7f7dae23852b2853cf0368d49426b/pyproj-3.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80fafd1f3eb421694857f254a9bdbacd1eb22fc6c24ca74b136679f376f97d35", size = 8328075, upload-time = "2023-09-21T02:07:15.353Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/a8/7193f46032636be917bc775506ae987aad72c931b1f691b775ca812a2917/pyproj-3.6.1-cp310-cp310-win32.whl", hash = "sha256:c41e80ddee130450dcb8829af7118f1ab69eaf8169c4bf0ee8d52b72f098dc2f", size = 5635713, upload-time = "2023-09-21T02:07:17.548Z" },
+    { url = "https://files.pythonhosted.org/packages/89/8f/27350c8fba71a37cd0d316f100fbd96bf139cc2b5ff1ab0dcbc7ac64010a/pyproj-3.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:db3aedd458e7f7f21d8176f0a1d924f1ae06d725228302b872885a1c34f3119e", size = 6087932, upload-time = "2023-09-21T02:07:19.793Z" },
+    { url = "https://files.pythonhosted.org/packages/84/a6/a300c1b14b2112e966e9f90b18f9c13b586bdcf417207cee913ae9005da3/pyproj-3.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ebfbdbd0936e178091309f6cd4fcb4decd9eab12aa513cdd9add89efa3ec2882", size = 6147442, upload-time = "2023-09-21T02:07:21.879Z" },
+    { url = "https://files.pythonhosted.org/packages/30/bd/b9bd3761f08754e8dbb34c5a647db2099b348ab5da338e90980caf280e37/pyproj-3.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:447db19c7efad70ff161e5e46a54ab9cc2399acebb656b6ccf63e4bc4a04b97a", size = 4880331, upload-time = "2023-09-21T02:10:40.828Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/0a/d82aeeb605b5d6870bc72307c3b5e044e632eb7720df8885e144f51a8eac/pyproj-3.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7e13c40183884ec7f94eb8e0f622f08f1d5716150b8d7a134de48c6110fee85", size = 6192425, upload-time = "2023-09-21T02:17:09.049Z" },
+    { url = "https://files.pythonhosted.org/packages/64/90/dfe5c00de1ca4dbb82606e79790659d4ed7f0ed8d372bccb3baca2a5abe0/pyproj-3.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65ad699e0c830e2b8565afe42bd58cc972b47d829b2e0e48ad9638386d994915", size = 8571478, upload-time = "2023-09-21T02:07:23.771Z" },
+    { url = "https://files.pythonhosted.org/packages/14/6d/ae373629a1723f0db80d7b8c93598b00d9ecb930ed9ebf4f35826a33e97c/pyproj-3.6.1-cp311-cp311-win32.whl", hash = "sha256:8b8acc31fb8702c54625f4d5a2a6543557bec3c28a0ef638778b7ab1d1772132", size = 5634575, upload-time = "2023-09-21T02:07:26.535Z" },
+    { url = "https://files.pythonhosted.org/packages/79/95/eb68113c5b5737c342bde1bab92705dabe69c16299c5a122616e50f1fbd6/pyproj-3.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:38a3361941eb72b82bd9a18f60c78b0df8408416f9340521df442cebfc4306e2", size = 6088494, upload-time = "2023-09-21T02:07:28.75Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/64/93232511a7906a492b1b7dfdfc17f4e95982d76a24ef4f86d18cfe7ae2c9/pyproj-3.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1e9fbaf920f0f9b4ee62aab832be3ae3968f33f24e2e3f7fbb8c6728ef1d9746", size = 6135280, upload-time = "2023-09-21T02:07:30.911Z" },
+    { url = "https://files.pythonhosted.org/packages/10/f2/b550b1f65cc7e51c9116b220b50aade60c439103432a3fd5b12efbc77e15/pyproj-3.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d227a865356f225591b6732430b1d1781e946893789a609bb34f59d09b8b0f8", size = 4880030, upload-time = "2023-09-21T02:10:43.067Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/4b/2f8f6f94643b9fe2083338eff294feda84d916409b5840b7a402d2be93f8/pyproj-3.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83039e5ae04e5afc974f7d25ee0870a80a6bd6b7957c3aca5613ccbe0d3e72bf", size = 6184439, upload-time = "2023-09-21T02:17:43.499Z" },
+    { url = "https://files.pythonhosted.org/packages/19/9b/c57569132174786aa3f72275ac306956859a639dad0ce8d95c8411ce8209/pyproj-3.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb059ba3bced6f6725961ba758649261d85ed6ce670d3e3b0a26e81cf1aa8d", size = 8660747, upload-time = "2023-09-21T02:07:32.586Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/ab/1c2159ec757677c5a6b8803f6be45c2b550dc42c84ec4a228dc219849bbb/pyproj-3.6.1-cp312-cp312-win32.whl", hash = "sha256:2d6ff73cc6dbbce3766b6c0bce70ce070193105d8de17aa2470009463682a8eb", size = 5626805, upload-time = "2023-09-21T02:07:35.28Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/f3/2f32fe143cd7ba1d4d68f1b6dce9ca402d909cbd5a5830e3a8fa3d1acbbf/pyproj-3.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:7a27151ddad8e1439ba70c9b4b2b617b290c39395fa9ddb7411ebb0eb86d6fb0", size = 6079779, upload-time = "2023-09-21T02:07:37.486Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/50/d369bbe62d7a0d1e2cb40bc211da86a3f6e0f3c99f872957a72c3d5492d6/pyproj-3.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4ba1f9b03d04d8cab24d6375609070580a26ce76eaed54631f03bab00a9c737b", size = 6144755, upload-time = "2023-09-21T02:07:39.611Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/c2/8d4f61065dfed965e53badd41201ad86a05af0c1bbc75dffb12ef0f5a7dd/pyproj-3.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18faa54a3ca475bfe6255156f2f2874e9a1c8917b0004eee9f664b86ccc513d3", size = 4879187, upload-time = "2023-09-21T02:10:45.519Z" },
+    { url = "https://files.pythonhosted.org/packages/31/38/2cf8777cb2d5622a78195e690281b7029098795fde4751aec8128238b8bb/pyproj-3.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd43bd9a9b9239805f406fd82ba6b106bf4838d9ef37c167d3ed70383943ade1", size = 6192339, upload-time = "2023-09-21T02:17:09.942Z" },
+    { url = "https://files.pythonhosted.org/packages/97/0a/b1525be9680369cc06dd288e12c59d24d5798b4afcdcf1b0915836e1caa6/pyproj-3.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50100b2726a3ca946906cbaa789dd0749f213abf0cbb877e6de72ca7aa50e1ae", size = 8332638, upload-time = "2023-09-21T02:07:41.777Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/e8/e826e0a962f36bd925a933829cf6ef218efe2055db5ea292be40974a929d/pyproj-3.6.1-cp39-cp39-win32.whl", hash = "sha256:9274880263256f6292ff644ca92c46d96aa7e57a75c6df3f11d636ce845a1877", size = 5638159, upload-time = "2023-09-21T02:07:43.49Z" },
+    { url = "https://files.pythonhosted.org/packages/43/d0/cbe29a4dcf38ee7e72bf695d0d3f2bee21b4f22ee6cf579ad974de9edfc8/pyproj-3.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:36b64c2cb6ea1cc091f329c5bd34f9c01bb5da8c8e4492c709bda6a09f96808f", size = 6090565, upload-time = "2023-09-21T02:07:45.735Z" },
+    { url = "https://files.pythonhosted.org/packages/43/28/e8d2ca71dd56c27cbe668e4226963d61956cded222a2e839e6fec1ab6d82/pyproj-3.6.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fd93c1a0c6c4aedc77c0fe275a9f2aba4d59b8acf88cebfc19fe3c430cfabf4f", size = 6034252, upload-time = "2023-09-21T02:07:47.906Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/39/1ce27cb86f51a1f5aed3a1617802a6131b59ea78492141d1fbe36722595e/pyproj-3.6.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6420ea8e7d2a88cb148b124429fba8cd2e0fae700a2d96eab7083c0928a85110", size = 6386263, upload-time = "2023-09-21T02:07:49.586Z" },
+]
 
 [[package]]
 name = "pyproj"
 version = "3.7.1"
 source = { registry = "https://pypi.org/simple" }
 resolution-markers = [
-    "python_full_version < '3.11'",
+    "python_full_version == '3.10.*'",
 ]
 dependencies = [
-    { name = "certifi", marker = "python_full_version < '3.11'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/67/10/a8480ea27ea4bbe896c168808854d00f2a9b49f95c0319ddcbba693c8a90/pyproj-3.7.1.tar.gz", hash = "sha256:60d72facd7b6b79853f19744779abcd3f804c4e0d4fa8815469db20c9f640a47", size = 226339 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/25/a3/c4cd4bba5b336075f145fe784fcaf4ef56ffbc979833303303e7a659dda2/pyproj-3.7.1-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:bf09dbeb333c34e9c546364e7df1ff40474f9fddf9e70657ecb0e4f670ff0b0e", size = 6262524 },
-    { url = "https://files.pythonhosted.org/packages/40/45/4fdf18f4cc1995f1992771d2a51cf186a9d7a8ec973c9693f8453850c707/pyproj-3.7.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:6575b2e53cc9e3e461ad6f0692a5564b96e7782c28631c7771c668770915e169", size = 4665102 },
-    { url = "https://files.pythonhosted.org/packages/0c/d2/360eb127380106cee83569954ae696b88a891c804d7a93abe3fbc15f5976/pyproj-3.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cb516ee35ed57789b46b96080edf4e503fdb62dbb2e3c6581e0d6c83fca014b", size = 9432667 },
-    { url = "https://files.pythonhosted.org/packages/76/a5/c6e11b9a99ce146741fb4d184d5c468446c6d6015b183cae82ac822a6cfa/pyproj-3.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e47c4e93b88d99dd118875ee3ca0171932444cdc0b52d493371b5d98d0f30ee", size = 9259185 },
-    { url = "https://files.pythonhosted.org/packages/41/56/a3c15c42145797a99363fa0fdb4e9805dccb8b4a76a6d7b2cdf36ebcc2a1/pyproj-3.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3e8d276caeae34fcbe4813855d0d97b9b825bab8d7a8b86d859c24a6213a5a0d", size = 10469103 },
-    { url = "https://files.pythonhosted.org/packages/ef/73/c9194c2802fefe2a4fd4230bdd5ab083e7604e93c64d0356fa49c363bad6/pyproj-3.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f173f851ee75e54acdaa053382b6825b400cb2085663a9bb073728a59c60aebb", size = 10401391 },
-    { url = "https://files.pythonhosted.org/packages/c5/1d/ce8bb5b9251b04d7c22d63619bb3db3d2397f79000a9ae05b3fd86a5837e/pyproj-3.7.1-cp310-cp310-win32.whl", hash = "sha256:f550281ed6e5ea88fcf04a7c6154e246d5714be495c50c9e8e6b12d3fb63e158", size = 5869997 },
-    { url = "https://files.pythonhosted.org/packages/09/6a/ca145467fd2e5b21e3d5b8c2b9645dcfb3b68f08b62417699a1f5689008e/pyproj-3.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:3537668992a709a2e7f068069192138618c00d0ba113572fdd5ee5ffde8222f3", size = 6278581 },
-    { url = "https://files.pythonhosted.org/packages/ab/0d/63670fc527e664068b70b7cab599aa38b7420dd009bdc29ea257e7f3dfb3/pyproj-3.7.1-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:a94e26c1a4950cea40116775588a2ca7cf56f1f434ff54ee35a84718f3841a3d", size = 6264315 },
-    { url = "https://files.pythonhosted.org/packages/25/9d/cbaf82cfb290d1f1fa42feb9ba9464013bb3891e40c4199f8072112e4589/pyproj-3.7.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:263b54ba5004b6b957d55757d846fc5081bc02980caa0279c4fc95fa0fff6067", size = 4666267 },
-    { url = "https://files.pythonhosted.org/packages/79/53/24f9f9b8918c0550f3ff49ad5de4cf3f0688c9f91ff191476db8979146fe/pyproj-3.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6d6a2ccd5607cd15ef990c51e6f2dd27ec0a741e72069c387088bba3aab60fa", size = 9680510 },
-    { url = "https://files.pythonhosted.org/packages/3c/ac/12fab74a908d40b63174dc704587febd0729414804bbfd873cabe504ff2d/pyproj-3.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c5dcf24ede53d8abab7d8a77f69ff1936c6a8843ef4fcc574646e4be66e5739", size = 9493619 },
-    { url = "https://files.pythonhosted.org/packages/c4/45/26311d6437135da2153a178125db5dfb6abce831ce04d10ec207eabac70a/pyproj-3.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c2e7449840a44ce860d8bea2c6c1c4bc63fa07cba801dcce581d14dcb031a02", size = 10709755 },
-    { url = "https://files.pythonhosted.org/packages/99/52/4ecd0986f27d0e6c8ee3a7bc5c63da15acd30ac23034f871325b297e61fd/pyproj-3.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0829865c1d3a3543f918b3919dc601eea572d6091c0dd175e1a054db9c109274", size = 10642970 },
-    { url = "https://files.pythonhosted.org/packages/3f/a5/d3bfc018fc92195a000d1d28acc1f3f1df15ff9f09ece68f45a2636c0134/pyproj-3.7.1-cp311-cp311-win32.whl", hash = "sha256:6181960b4b812e82e588407fe5c9c68ada267c3b084db078f248db5d7f45d18a", size = 5868295 },
-    { url = "https://files.pythonhosted.org/packages/92/39/ef6f06a5b223dbea308cfcbb7a0f72e7b506aef1850e061b2c73b0818715/pyproj-3.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ad0ff443a785d84e2b380869fdd82e6bfc11eba6057d25b4409a9bbfa867970", size = 6279871 },
-    { url = "https://files.pythonhosted.org/packages/e6/c9/876d4345b8d17f37ac59ebd39f8fa52fc6a6a9891a420f72d050edb6b899/pyproj-3.7.1-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:2781029d90df7f8d431e29562a3f2d8eafdf233c4010d6fc0381858dc7373217", size = 6264087 },
-    { url = "https://files.pythonhosted.org/packages/ff/e6/5f8691f8c90e7f402cc80a6276eb19d2ec1faa150d5ae2dd9c7b0a254da8/pyproj-3.7.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:d61bf8ab04c73c1da08eedaf21a103b72fa5b0a9b854762905f65ff8b375d394", size = 4669628 },
-    { url = "https://files.pythonhosted.org/packages/42/ec/16475bbb79c1c68845c0a0d9c60c4fb31e61b8a2a20bc18b1a81e81c7f68/pyproj-3.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04abc517a8555d1b05fcee768db3280143fe42ec39fdd926a2feef31631a1f2f", size = 9721415 },
-    { url = "https://files.pythonhosted.org/packages/b3/a3/448f05b15e318bd6bea9a32cfaf11e886c4ae61fa3eee6e09ed5c3b74bb2/pyproj-3.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084c0a475688f934d386c2ab3b6ce03398a473cd48adfda70d9ab8f87f2394a0", size = 9556447 },
-    { url = "https://files.pythonhosted.org/packages/6a/ae/bd15fe8d8bd914ead6d60bca7f895a4e6f8ef7e3928295134ff9a7dad14c/pyproj-3.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a20727a23b1e49c7dc7fe3c3df8e56a8a7acdade80ac2f5cca29d7ca5564c145", size = 10758317 },
-    { url = "https://files.pythonhosted.org/packages/9d/d9/5ccefb8bca925f44256b188a91c31238cae29ab6ee7f53661ecc04616146/pyproj-3.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bf84d766646f1ebd706d883755df4370aaf02b48187cedaa7e4239f16bc8213d", size = 10771259 },
-    { url = "https://files.pythonhosted.org/packages/2a/7d/31dedff9c35fa703162f922eeb0baa6c44a3288469a5fd88d209e2892f9e/pyproj-3.7.1-cp312-cp312-win32.whl", hash = "sha256:5f0da2711364d7cb9f115b52289d4a9b61e8bca0da57f44a3a9d6fc9bdeb7274", size = 5859914 },
-    { url = "https://files.pythonhosted.org/packages/3e/47/c6ab03d6564a7c937590cff81a2742b5990f096cce7c1a622d325be340ee/pyproj-3.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:aee664a9d806612af30a19dba49e55a7a78ebfec3e9d198f6a6176e1d140ec98", size = 6273196 },
-    { url = "https://files.pythonhosted.org/packages/ef/01/984828464c9960036c602753fc0f21f24f0aa9043c18fa3f2f2b66a86340/pyproj-3.7.1-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:5f8d02ef4431dee414d1753d13fa82a21a2f61494737b5f642ea668d76164d6d", size = 6253062 },
-    { url = "https://files.pythonhosted.org/packages/68/65/6ecdcdc829811a2c160cdfe2f068a009fc572fd4349664f758ccb0853a7c/pyproj-3.7.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:0b853ae99bda66cbe24b4ccfe26d70601d84375940a47f553413d9df570065e0", size = 4660548 },
-    { url = "https://files.pythonhosted.org/packages/67/da/dda94c4490803679230ba4c17a12f151b307a0d58e8110820405ca2d98db/pyproj-3.7.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83db380c52087f9e9bdd8a527943b2e7324f275881125e39475c4f9277bdeec4", size = 9662464 },
-    { url = "https://files.pythonhosted.org/packages/6f/57/f61b7d22c91ae1d12ee00ac4c0038714e774ebcd851b9133e5f4f930dd40/pyproj-3.7.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b35ed213892e211a3ce2bea002aa1183e1a2a9b79e51bb3c6b15549a831ae528", size = 9497461 },
-    { url = "https://files.pythonhosted.org/packages/b7/f6/932128236f79d2ac7d39fe1a19667fdf7155d9a81d31fb9472a7a497790f/pyproj-3.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a8b15b0463d1303bab113d1a6af2860a0d79013c3a66fcc5475ce26ef717fd4f", size = 10708869 },
-    { url = "https://files.pythonhosted.org/packages/1d/0d/07ac7712994454a254c383c0d08aff9916a2851e6512d59da8dc369b1b02/pyproj-3.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:87229e42b75e89f4dad6459200f92988c5998dfb093c7c631fb48524c86cd5dc", size = 10729260 },
-    { url = "https://files.pythonhosted.org/packages/b0/d0/9c604bc72c37ba69b867b6df724d6a5af6789e8c375022c952f65b2af558/pyproj-3.7.1-cp313-cp313-win32.whl", hash = "sha256:d666c3a3faaf3b1d7fc4a544059c4eab9d06f84a604b070b7aa2f318e227798e", size = 5855462 },
-    { url = "https://files.pythonhosted.org/packages/98/df/68a2b7f5fb6400c64aad82d72bcc4bc531775e62eedff993a77c780defd0/pyproj-3.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:d3caac7473be22b6d6e102dde6c46de73b96bc98334e577dfaee9886f102ea2e", size = 6266573 },
+    { name = "certifi", marker = "python_full_version == '3.10.*'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/67/10/a8480ea27ea4bbe896c168808854d00f2a9b49f95c0319ddcbba693c8a90/pyproj-3.7.1.tar.gz", hash = "sha256:60d72facd7b6b79853f19744779abcd3f804c4e0d4fa8815469db20c9f640a47", size = 226339, upload-time = "2025-02-16T04:28:46.621Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/25/a3/c4cd4bba5b336075f145fe784fcaf4ef56ffbc979833303303e7a659dda2/pyproj-3.7.1-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:bf09dbeb333c34e9c546364e7df1ff40474f9fddf9e70657ecb0e4f670ff0b0e", size = 6262524, upload-time = "2025-02-16T04:27:19.725Z" },
+    { url = "https://files.pythonhosted.org/packages/40/45/4fdf18f4cc1995f1992771d2a51cf186a9d7a8ec973c9693f8453850c707/pyproj-3.7.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:6575b2e53cc9e3e461ad6f0692a5564b96e7782c28631c7771c668770915e169", size = 4665102, upload-time = "2025-02-16T04:27:24.428Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/d2/360eb127380106cee83569954ae696b88a891c804d7a93abe3fbc15f5976/pyproj-3.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cb516ee35ed57789b46b96080edf4e503fdb62dbb2e3c6581e0d6c83fca014b", size = 9432667, upload-time = "2025-02-16T04:27:27.04Z" },
+    { url = "https://files.pythonhosted.org/packages/76/a5/c6e11b9a99ce146741fb4d184d5c468446c6d6015b183cae82ac822a6cfa/pyproj-3.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e47c4e93b88d99dd118875ee3ca0171932444cdc0b52d493371b5d98d0f30ee", size = 9259185, upload-time = "2025-02-16T04:27:30.35Z" },
+    { url = "https://files.pythonhosted.org/packages/41/56/a3c15c42145797a99363fa0fdb4e9805dccb8b4a76a6d7b2cdf36ebcc2a1/pyproj-3.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3e8d276caeae34fcbe4813855d0d97b9b825bab8d7a8b86d859c24a6213a5a0d", size = 10469103, upload-time = "2025-02-16T04:27:33.542Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/73/c9194c2802fefe2a4fd4230bdd5ab083e7604e93c64d0356fa49c363bad6/pyproj-3.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f173f851ee75e54acdaa053382b6825b400cb2085663a9bb073728a59c60aebb", size = 10401391, upload-time = "2025-02-16T04:27:36.051Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/1d/ce8bb5b9251b04d7c22d63619bb3db3d2397f79000a9ae05b3fd86a5837e/pyproj-3.7.1-cp310-cp310-win32.whl", hash = "sha256:f550281ed6e5ea88fcf04a7c6154e246d5714be495c50c9e8e6b12d3fb63e158", size = 5869997, upload-time = "2025-02-16T04:27:38.302Z" },
+    { url = "https://files.pythonhosted.org/packages/09/6a/ca145467fd2e5b21e3d5b8c2b9645dcfb3b68f08b62417699a1f5689008e/pyproj-3.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:3537668992a709a2e7f068069192138618c00d0ba113572fdd5ee5ffde8222f3", size = 6278581, upload-time = "2025-02-16T04:27:41.051Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/0d/63670fc527e664068b70b7cab599aa38b7420dd009bdc29ea257e7f3dfb3/pyproj-3.7.1-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:a94e26c1a4950cea40116775588a2ca7cf56f1f434ff54ee35a84718f3841a3d", size = 6264315, upload-time = "2025-02-16T04:27:44.539Z" },
+    { url = "https://files.pythonhosted.org/packages/25/9d/cbaf82cfb290d1f1fa42feb9ba9464013bb3891e40c4199f8072112e4589/pyproj-3.7.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:263b54ba5004b6b957d55757d846fc5081bc02980caa0279c4fc95fa0fff6067", size = 4666267, upload-time = "2025-02-16T04:27:47.019Z" },
+    { url = "https://files.pythonhosted.org/packages/79/53/24f9f9b8918c0550f3ff49ad5de4cf3f0688c9f91ff191476db8979146fe/pyproj-3.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6d6a2ccd5607cd15ef990c51e6f2dd27ec0a741e72069c387088bba3aab60fa", size = 9680510, upload-time = "2025-02-16T04:27:49.239Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/ac/12fab74a908d40b63174dc704587febd0729414804bbfd873cabe504ff2d/pyproj-3.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c5dcf24ede53d8abab7d8a77f69ff1936c6a8843ef4fcc574646e4be66e5739", size = 9493619, upload-time = "2025-02-16T04:27:52.65Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/45/26311d6437135da2153a178125db5dfb6abce831ce04d10ec207eabac70a/pyproj-3.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c2e7449840a44ce860d8bea2c6c1c4bc63fa07cba801dcce581d14dcb031a02", size = 10709755, upload-time = "2025-02-16T04:27:55.239Z" },
+    { url = "https://files.pythonhosted.org/packages/99/52/4ecd0986f27d0e6c8ee3a7bc5c63da15acd30ac23034f871325b297e61fd/pyproj-3.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0829865c1d3a3543f918b3919dc601eea572d6091c0dd175e1a054db9c109274", size = 10642970, upload-time = "2025-02-16T04:27:58.343Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/a5/d3bfc018fc92195a000d1d28acc1f3f1df15ff9f09ece68f45a2636c0134/pyproj-3.7.1-cp311-cp311-win32.whl", hash = "sha256:6181960b4b812e82e588407fe5c9c68ada267c3b084db078f248db5d7f45d18a", size = 5868295, upload-time = "2025-02-16T04:28:01.712Z" },
+    { url = "https://files.pythonhosted.org/packages/92/39/ef6f06a5b223dbea308cfcbb7a0f72e7b506aef1850e061b2c73b0818715/pyproj-3.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ad0ff443a785d84e2b380869fdd82e6bfc11eba6057d25b4409a9bbfa867970", size = 6279871, upload-time = "2025-02-16T04:28:04.988Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/c9/876d4345b8d17f37ac59ebd39f8fa52fc6a6a9891a420f72d050edb6b899/pyproj-3.7.1-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:2781029d90df7f8d431e29562a3f2d8eafdf233c4010d6fc0381858dc7373217", size = 6264087, upload-time = "2025-02-16T04:28:09.036Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/e6/5f8691f8c90e7f402cc80a6276eb19d2ec1faa150d5ae2dd9c7b0a254da8/pyproj-3.7.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:d61bf8ab04c73c1da08eedaf21a103b72fa5b0a9b854762905f65ff8b375d394", size = 4669628, upload-time = "2025-02-16T04:28:10.944Z" },
+    { url = "https://files.pythonhosted.org/packages/42/ec/16475bbb79c1c68845c0a0d9c60c4fb31e61b8a2a20bc18b1a81e81c7f68/pyproj-3.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04abc517a8555d1b05fcee768db3280143fe42ec39fdd926a2feef31631a1f2f", size = 9721415, upload-time = "2025-02-16T04:28:13.342Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/a3/448f05b15e318bd6bea9a32cfaf11e886c4ae61fa3eee6e09ed5c3b74bb2/pyproj-3.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084c0a475688f934d386c2ab3b6ce03398a473cd48adfda70d9ab8f87f2394a0", size = 9556447, upload-time = "2025-02-16T04:28:15.818Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/ae/bd15fe8d8bd914ead6d60bca7f895a4e6f8ef7e3928295134ff9a7dad14c/pyproj-3.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a20727a23b1e49c7dc7fe3c3df8e56a8a7acdade80ac2f5cca29d7ca5564c145", size = 10758317, upload-time = "2025-02-16T04:28:18.338Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/d9/5ccefb8bca925f44256b188a91c31238cae29ab6ee7f53661ecc04616146/pyproj-3.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bf84d766646f1ebd706d883755df4370aaf02b48187cedaa7e4239f16bc8213d", size = 10771259, upload-time = "2025-02-16T04:28:20.822Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/7d/31dedff9c35fa703162f922eeb0baa6c44a3288469a5fd88d209e2892f9e/pyproj-3.7.1-cp312-cp312-win32.whl", hash = "sha256:5f0da2711364d7cb9f115b52289d4a9b61e8bca0da57f44a3a9d6fc9bdeb7274", size = 5859914, upload-time = "2025-02-16T04:28:23.303Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/47/c6ab03d6564a7c937590cff81a2742b5990f096cce7c1a622d325be340ee/pyproj-3.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:aee664a9d806612af30a19dba49e55a7a78ebfec3e9d198f6a6176e1d140ec98", size = 6273196, upload-time = "2025-02-16T04:28:25.227Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/01/984828464c9960036c602753fc0f21f24f0aa9043c18fa3f2f2b66a86340/pyproj-3.7.1-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:5f8d02ef4431dee414d1753d13fa82a21a2f61494737b5f642ea668d76164d6d", size = 6253062, upload-time = "2025-02-16T04:28:27.861Z" },
+    { url = "https://files.pythonhosted.org/packages/68/65/6ecdcdc829811a2c160cdfe2f068a009fc572fd4349664f758ccb0853a7c/pyproj-3.7.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:0b853ae99bda66cbe24b4ccfe26d70601d84375940a47f553413d9df570065e0", size = 4660548, upload-time = "2025-02-16T04:28:29.526Z" },
+    { url = "https://files.pythonhosted.org/packages/67/da/dda94c4490803679230ba4c17a12f151b307a0d58e8110820405ca2d98db/pyproj-3.7.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83db380c52087f9e9bdd8a527943b2e7324f275881125e39475c4f9277bdeec4", size = 9662464, upload-time = "2025-02-16T04:28:31.437Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/57/f61b7d22c91ae1d12ee00ac4c0038714e774ebcd851b9133e5f4f930dd40/pyproj-3.7.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b35ed213892e211a3ce2bea002aa1183e1a2a9b79e51bb3c6b15549a831ae528", size = 9497461, upload-time = "2025-02-16T04:28:33.848Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/f6/932128236f79d2ac7d39fe1a19667fdf7155d9a81d31fb9472a7a497790f/pyproj-3.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a8b15b0463d1303bab113d1a6af2860a0d79013c3a66fcc5475ce26ef717fd4f", size = 10708869, upload-time = "2025-02-16T04:28:37.34Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/0d/07ac7712994454a254c383c0d08aff9916a2851e6512d59da8dc369b1b02/pyproj-3.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:87229e42b75e89f4dad6459200f92988c5998dfb093c7c631fb48524c86cd5dc", size = 10729260, upload-time = "2025-02-16T04:28:40.639Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/d0/9c604bc72c37ba69b867b6df724d6a5af6789e8c375022c952f65b2af558/pyproj-3.7.1-cp313-cp313-win32.whl", hash = "sha256:d666c3a3faaf3b1d7fc4a544059c4eab9d06f84a604b070b7aa2f318e227798e", size = 5855462, upload-time = "2025-02-16T04:28:42.827Z" },
+    { url = "https://files.pythonhosted.org/packages/98/df/68a2b7f5fb6400c64aad82d72bcc4bc531775e62eedff993a77c780defd0/pyproj-3.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:d3caac7473be22b6d6e102dde6c46de73b96bc98334e577dfaee9886f102ea2e", size = 6266573, upload-time = "2025-02-16T04:28:44.727Z" },
 ]
 
 [[package]]
@@ -2273,62 +2745,62 @@ resolution-markers = [
 dependencies = [
     { name = "certifi", marker = "python_full_version >= '3.11'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/04/90/67bd7260b4ea9b8b20b4f58afef6c223ecb3abf368eb4ec5bc2cdef81b49/pyproj-3.7.2.tar.gz", hash = "sha256:39a0cf1ecc7e282d1d30f36594ebd55c9fae1fda8a2622cee5d100430628f88c", size = 226279 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/a6/bd/f205552cd1713b08f93b09e39a3ec99edef0b3ebbbca67b486fdf1abe2de/pyproj-3.7.2-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:2514d61f24c4e0bb9913e2c51487ecdaeca5f8748d8313c933693416ca41d4d5", size = 6227022 },
-    { url = "https://files.pythonhosted.org/packages/75/4c/9a937e659b8b418ab573c6d340d27e68716928953273e0837e7922fcac34/pyproj-3.7.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:8693ca3892d82e70de077701ee76dd13d7bca4ae1c9d1e739d72004df015923a", size = 4625810 },
-    { url = "https://files.pythonhosted.org/packages/c0/7d/a9f41e814dc4d1dc54e95b2ccaf0b3ebe3eb18b1740df05fe334724c3d89/pyproj-3.7.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:5e26484d80fea56273ed1555abaea161e9661d81a6c07815d54b8e883d4ceb25", size = 9638694 },
-    { url = "https://files.pythonhosted.org/packages/ad/ab/9bdb4a6216b712a1f9aab1c0fcbee5d3726f34a366f29c3e8c08a78d6b70/pyproj-3.7.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:281cb92847814e8018010c48b4069ff858a30236638631c1a91dd7bfa68f8a8a", size = 9493977 },
-    { url = "https://files.pythonhosted.org/packages/c9/db/2db75b1b6190f1137b1c4e8ef6a22e1c338e46320f6329bfac819143e063/pyproj-3.7.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9c8577f0b7bb09118ec2e57e3babdc977127dd66326d6c5d755c76b063e6d9dc", size = 10841151 },
-    { url = "https://files.pythonhosted.org/packages/89/f7/989643394ba23a286e9b7b3f09981496172f9e0d4512457ffea7dc47ffc7/pyproj-3.7.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a23f59904fac3a5e7364b3aa44d288234af267ca041adb2c2b14a903cd5d3ac5", size = 10751585 },
-    { url = "https://files.pythonhosted.org/packages/53/6d/ad928fe975a6c14a093c92e6a319ca18f479f3336bb353a740bdba335681/pyproj-3.7.2-cp311-cp311-win32.whl", hash = "sha256:f2af4ed34b2cf3e031a2d85b067a3ecbd38df073c567e04b52fa7a0202afde8a", size = 5908533 },
-    { url = "https://files.pythonhosted.org/packages/79/e0/b95584605cec9ed50b7ebaf7975d1c4ddeec5a86b7a20554ed8b60042bd7/pyproj-3.7.2-cp311-cp311-win_amd64.whl", hash = "sha256:0b7cb633565129677b2a183c4d807c727d1c736fcb0568a12299383056e67433", size = 6320742 },
-    { url = "https://files.pythonhosted.org/packages/b7/4d/536e8f93bca808175c2d0a5ac9fdf69b960d8ab6b14f25030dccb07464d7/pyproj-3.7.2-cp311-cp311-win_arm64.whl", hash = "sha256:38b08d85e3a38e455625b80e9eb9f78027c8e2649a21dec4df1f9c3525460c71", size = 6245772 },
-    { url = "https://files.pythonhosted.org/packages/8d/ab/9893ea9fb066be70ed9074ae543914a618c131ed8dff2da1e08b3a4df4db/pyproj-3.7.2-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:0a9bb26a6356fb5b033433a6d1b4542158fb71e3c51de49b4c318a1dff3aeaab", size = 6219832 },
-    { url = "https://files.pythonhosted.org/packages/53/78/4c64199146eed7184eb0e85bedec60a4aa8853b6ffe1ab1f3a8b962e70a0/pyproj-3.7.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:567caa03021178861fad27fabde87500ec6d2ee173dd32f3e2d9871e40eebd68", size = 4620650 },
-    { url = "https://files.pythonhosted.org/packages/b6/ac/14a78d17943898a93ef4f8c6a9d4169911c994e3161e54a7cedeba9d8dde/pyproj-3.7.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c203101d1dc3c038a56cff0447acc515dd29d6e14811406ac539c21eed422b2a", size = 9667087 },
-    { url = "https://files.pythonhosted.org/packages/b8/be/212882c450bba74fc8d7d35cbd57e4af84792f0a56194819d98106b075af/pyproj-3.7.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:1edc34266c0c23ced85f95a1ee8b47c9035eae6aca5b6b340327250e8e281630", size = 9552797 },
-    { url = "https://files.pythonhosted.org/packages/ba/c0/c0f25c87b5d2a8686341c53c1792a222a480d6c9caf60311fec12c99ec26/pyproj-3.7.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa9f26c21bc0e2dc3d224cb1eb4020cf23e76af179a7c66fea49b828611e4260", size = 10837036 },
-    { url = "https://files.pythonhosted.org/packages/5d/37/5cbd6772addde2090c91113332623a86e8c7d583eccb2ad02ea634c4a89f/pyproj-3.7.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f9428b318530625cb389b9ddc9c51251e172808a4af79b82809376daaeabe5e9", size = 10775952 },
-    { url = "https://files.pythonhosted.org/packages/69/a1/dc250e3cf83eb4b3b9a2cf86fdb5e25288bd40037ae449695550f9e96b2f/pyproj-3.7.2-cp312-cp312-win32.whl", hash = "sha256:b3d99ed57d319da042f175f4554fc7038aa4bcecc4ac89e217e350346b742c9d", size = 5898872 },
-    { url = "https://files.pythonhosted.org/packages/4a/a6/6fe724b72b70f2b00152d77282e14964d60ab092ec225e67c196c9b463e5/pyproj-3.7.2-cp312-cp312-win_amd64.whl", hash = "sha256:11614a054cd86a2ed968a657d00987a86eeb91fdcbd9ad3310478685dc14a128", size = 6312176 },
-    { url = "https://files.pythonhosted.org/packages/5d/68/915cc32c02a91e76d02c8f55d5a138d6ef9e47a0d96d259df98f4842e558/pyproj-3.7.2-cp312-cp312-win_arm64.whl", hash = "sha256:509a146d1398bafe4f53273398c3bb0b4732535065fa995270e52a9d3676bca3", size = 6233452 },
-    { url = "https://files.pythonhosted.org/packages/be/14/faf1b90d267cea68d7e70662e7f88cefdb1bc890bd596c74b959e0517a72/pyproj-3.7.2-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:19466e529b1b15eeefdf8ff26b06fa745856c044f2f77bf0edbae94078c1dfa1", size = 6214580 },
-    { url = "https://files.pythonhosted.org/packages/35/48/da9a45b184d375f62667f62eba0ca68569b0bd980a0bb7ffcc1d50440520/pyproj-3.7.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:c79b9b84c4a626c5dc324c0d666be0bfcebd99f7538d66e8898c2444221b3da7", size = 4615388 },
-    { url = "https://files.pythonhosted.org/packages/5e/e7/d2b459a4a64bca328b712c1b544e109df88e5c800f7c143cfbc404d39bfb/pyproj-3.7.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:ceecf374cacca317bc09e165db38ac548ee3cad07c3609442bd70311c59c21aa", size = 9628455 },
-    { url = "https://files.pythonhosted.org/packages/f8/85/c2b1706e51942de19076eff082f8495e57d5151364e78b5bef4af4a1d94a/pyproj-3.7.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5141a538ffdbe4bfd157421828bb2e07123a90a7a2d6f30fa1462abcfb5ce681", size = 9514269 },
-    { url = "https://files.pythonhosted.org/packages/34/38/07a9b89ae7467872f9a476883a5bad9e4f4d1219d31060f0f2b282276cbe/pyproj-3.7.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f000841e98ea99acbb7b8ca168d67773b0191de95187228a16110245c5d954d5", size = 10808437 },
-    { url = "https://files.pythonhosted.org/packages/12/56/fda1daeabbd39dec5b07f67233d09f31facb762587b498e6fc4572be9837/pyproj-3.7.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8115faf2597f281a42ab608ceac346b4eb1383d3b45ab474fd37341c4bf82a67", size = 10745540 },
-    { url = "https://files.pythonhosted.org/packages/0d/90/c793182cbba65a39a11db2ac6b479fe76c59e6509ae75e5744c344a0da9d/pyproj-3.7.2-cp313-cp313-win32.whl", hash = "sha256:f18c0579dd6be00b970cb1a6719197fceecc407515bab37da0066f0184aafdf3", size = 5896506 },
-    { url = "https://files.pythonhosted.org/packages/be/0f/747974129cf0d800906f81cd25efd098c96509026e454d4b66868779ab04/pyproj-3.7.2-cp313-cp313-win_amd64.whl", hash = "sha256:bb41c29d5f60854b1075853fe80c58950b398d4ebb404eb532536ac8d2834ed7", size = 6310195 },
-    { url = "https://files.pythonhosted.org/packages/82/64/fc7598a53172c4931ec6edf5228280663063150625d3f6423b4c20f9daff/pyproj-3.7.2-cp313-cp313-win_arm64.whl", hash = "sha256:2b617d573be4118c11cd96b8891a0b7f65778fa7733ed8ecdb297a447d439100", size = 6230748 },
-    { url = "https://files.pythonhosted.org/packages/aa/f0/611dd5cddb0d277f94b7af12981f56e1441bf8d22695065d4f0df5218498/pyproj-3.7.2-cp313-cp313t-macosx_13_0_x86_64.whl", hash = "sha256:d27b48f0e81beeaa2b4d60c516c3a1cfbb0c7ff6ef71256d8e9c07792f735279", size = 6241729 },
-    { url = "https://files.pythonhosted.org/packages/15/93/40bd4a6c523ff9965e480870611aed7eda5aa2c6128c6537345a2b77b542/pyproj-3.7.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:55a3610d75023c7b1c6e583e48ef8f62918e85a2ae81300569d9f104d6684bb6", size = 4652497 },
-    { url = "https://files.pythonhosted.org/packages/1b/ae/7150ead53c117880b35e0d37960d3138fe640a235feb9605cb9386f50bb0/pyproj-3.7.2-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:8d7349182fa622696787cc9e195508d2a41a64765da9b8a6bee846702b9e6220", size = 9942610 },
-    { url = "https://files.pythonhosted.org/packages/d8/17/7a4a7eafecf2b46ab64e5c08176c20ceb5844b503eaa551bf12ccac77322/pyproj-3.7.2-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:d230b186eb876ed4f29a7c5ee310144c3a0e44e89e55f65fb3607e13f6db337c", size = 9692390 },
-    { url = "https://files.pythonhosted.org/packages/c3/55/ae18f040f6410f0ea547a21ada7ef3e26e6c82befa125b303b02759c0e9d/pyproj-3.7.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:237499c7862c578d0369e2b8ac56eec550e391a025ff70e2af8417139dabb41c", size = 11047596 },
-    { url = "https://files.pythonhosted.org/packages/e6/2e/d3fff4d2909473f26ae799f9dda04caa322c417a51ff3b25763f7d03b233/pyproj-3.7.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8c225f5978abd506fd9a78eaaf794435e823c9156091cabaab5374efb29d7f69", size = 10896975 },
-    { url = "https://files.pythonhosted.org/packages/f2/bc/8fc7d3963d87057b7b51ebe68c1e7c51c23129eee5072ba6b86558544a46/pyproj-3.7.2-cp313-cp313t-win32.whl", hash = "sha256:2da731876d27639ff9d2d81c151f6ab90a1546455fabd93368e753047be344a2", size = 5953057 },
-    { url = "https://files.pythonhosted.org/packages/cc/27/ea9809966cc47d2d51e6d5ae631ea895f7c7c7b9b3c29718f900a8f7d197/pyproj-3.7.2-cp313-cp313t-win_amd64.whl", hash = "sha256:f54d91ae18dd23b6c0ab48126d446820e725419da10617d86a1b69ada6d881d3", size = 6375414 },
-    { url = "https://files.pythonhosted.org/packages/5b/f8/1ef0129fba9a555c658e22af68989f35e7ba7b9136f25758809efec0cd6e/pyproj-3.7.2-cp313-cp313t-win_arm64.whl", hash = "sha256:fc52ba896cfc3214dc9f9ca3c0677a623e8fdd096b257c14a31e719d21ff3fdd", size = 6262501 },
-    { url = "https://files.pythonhosted.org/packages/42/17/c2b050d3f5b71b6edd0d96ae16c990fdc42a5f1366464a5c2772146de33a/pyproj-3.7.2-cp314-cp314-macosx_13_0_x86_64.whl", hash = "sha256:2aaa328605ace41db050d06bac1adc11f01b71fe95c18661497763116c3a0f02", size = 6214541 },
-    { url = "https://files.pythonhosted.org/packages/03/68/68ada9c8aea96ded09a66cfd9bf87aa6db8c2edebe93f5bf9b66b0143fbc/pyproj-3.7.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:35dccbce8201313c596a970fde90e33605248b66272595c061b511c8100ccc08", size = 4617456 },
-    { url = "https://files.pythonhosted.org/packages/81/e4/4c50ceca7d0e937977866b02cb64e6ccf4df979a5871e521f9e255df6073/pyproj-3.7.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:25b0b7cb0042444c29a164b993c45c1b8013d6c48baa61dc1160d834a277e83b", size = 9615590 },
-    { url = "https://files.pythonhosted.org/packages/05/1e/ada6fb15a1d75b5bd9b554355a69a798c55a7dcc93b8d41596265c1772e3/pyproj-3.7.2-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:85def3a6388e9ba51f964619aa002a9d2098e77c6454ff47773bb68871024281", size = 9474960 },
-    { url = "https://files.pythonhosted.org/packages/51/07/9d48ad0a8db36e16f842f2c8a694c1d9d7dcf9137264846bef77585a71f3/pyproj-3.7.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b1bccefec3875ab81eabf49059e2b2ea77362c178b66fd3528c3e4df242f1516", size = 10799478 },
-    { url = "https://files.pythonhosted.org/packages/85/cf/2f812b529079f72f51ff2d6456b7fef06c01735e5cfd62d54ffb2b548028/pyproj-3.7.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d5371ca114d6990b675247355a801925814eca53e6c4b2f1b5c0a956336ee36e", size = 10710030 },
-    { url = "https://files.pythonhosted.org/packages/99/9b/4626a19e1f03eba4c0e77b91a6cf0f73aa9cb5d51a22ee385c22812bcc2c/pyproj-3.7.2-cp314-cp314-win32.whl", hash = "sha256:77f066626030f41be543274f5ac79f2a511fe89860ecd0914f22131b40a0ec25", size = 5991181 },
-    { url = "https://files.pythonhosted.org/packages/04/b2/5a6610554306a83a563080c2cf2c57565563eadd280e15388efa00fb5b33/pyproj-3.7.2-cp314-cp314-win_amd64.whl", hash = "sha256:5a964da1696b8522806f4276ab04ccfff8f9eb95133a92a25900697609d40112", size = 6434721 },
-    { url = "https://files.pythonhosted.org/packages/ae/ce/6c910ea2e1c74ef673c5d48c482564b8a7824a44c4e35cca2e765b68cfcc/pyproj-3.7.2-cp314-cp314-win_arm64.whl", hash = "sha256:e258ab4dbd3cf627809067c0ba8f9884ea76c8e5999d039fb37a1619c6c3e1f6", size = 6363821 },
-    { url = "https://files.pythonhosted.org/packages/e4/e4/5532f6f7491812ba782a2177fe9de73fd8e2912b59f46a1d056b84b9b8f2/pyproj-3.7.2-cp314-cp314t-macosx_13_0_x86_64.whl", hash = "sha256:bbbac2f930c6d266f70ec75df35ef851d96fdb3701c674f42fd23a9314573b37", size = 6241773 },
-    { url = "https://files.pythonhosted.org/packages/20/1f/0938c3f2bbbef1789132d1726d9b0e662f10cfc22522743937f421ad664e/pyproj-3.7.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:b7544e0a3d6339dc9151e9c8f3ea62a936ab7cc446a806ec448bbe86aebb979b", size = 4652537 },
-    { url = "https://files.pythonhosted.org/packages/c7/a8/488b1ed47d25972f33874f91f09ca8f2227902f05f63a2b80dc73e7b1c97/pyproj-3.7.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f7f5133dca4c703e8acadf6f30bc567d39a42c6af321e7f81975c2518f3ed357", size = 9940864 },
-    { url = "https://files.pythonhosted.org/packages/c7/cc/7f4c895d0cb98e47b6a85a6d79eaca03eb266129eed2f845125c09cf31ff/pyproj-3.7.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:5aff3343038d7426aa5076f07feb88065f50e0502d1b0d7c22ddfdd2c75a3f81", size = 9688868 },
-    { url = "https://files.pythonhosted.org/packages/b2/b7/c7e306b8bb0f071d9825b753ee4920f066c40fbfcce9372c4f3cfb2fc4ed/pyproj-3.7.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b0552178c61f2ac1c820d087e8ba6e62b29442debddbb09d51c4bf8acc84d888", size = 11045910 },
-    { url = "https://files.pythonhosted.org/packages/42/fb/538a4d2df695980e2dde5c04d965fbdd1fe8c20a3194dc4aaa3952a4d1be/pyproj-3.7.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:47d87db2d2c436c5fd0409b34d70bb6cdb875cca2ebe7a9d1c442367b0ab8d59", size = 10895724 },
-    { url = "https://files.pythonhosted.org/packages/e8/8b/a3f0618b03957de9db5489a04558a8826f43906628bb0b766033aa3b5548/pyproj-3.7.2-cp314-cp314t-win32.whl", hash = "sha256:c9b6f1d8ad3e80a0ee0903a778b6ece7dca1d1d40f6d114ae01bc8ddbad971aa", size = 6056848 },
-    { url = "https://files.pythonhosted.org/packages/bc/56/413240dd5149dd3291eda55aa55a659da4431244a2fd1319d0ae89407cfb/pyproj-3.7.2-cp314-cp314t-win_amd64.whl", hash = "sha256:1914e29e27933ba6f9822663ee0600f169014a2859f851c054c88cf5ea8a333c", size = 6517676 },
-    { url = "https://files.pythonhosted.org/packages/15/73/a7141a1a0559bf1a7aa42a11c879ceb19f02f5c6c371c6d57fd86cefd4d1/pyproj-3.7.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d9d25bae416a24397e0d85739f84d323b55f6511e45a522dd7d7eae70d10c7e4", size = 6391844 },
+sdist = { url = "https://files.pythonhosted.org/packages/04/90/67bd7260b4ea9b8b20b4f58afef6c223ecb3abf368eb4ec5bc2cdef81b49/pyproj-3.7.2.tar.gz", hash = "sha256:39a0cf1ecc7e282d1d30f36594ebd55c9fae1fda8a2622cee5d100430628f88c", size = 226279, upload-time = "2025-08-14T12:05:42.18Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a6/bd/f205552cd1713b08f93b09e39a3ec99edef0b3ebbbca67b486fdf1abe2de/pyproj-3.7.2-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:2514d61f24c4e0bb9913e2c51487ecdaeca5f8748d8313c933693416ca41d4d5", size = 6227022, upload-time = "2025-08-14T12:03:51.474Z" },
+    { url = "https://files.pythonhosted.org/packages/75/4c/9a937e659b8b418ab573c6d340d27e68716928953273e0837e7922fcac34/pyproj-3.7.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:8693ca3892d82e70de077701ee76dd13d7bca4ae1c9d1e739d72004df015923a", size = 4625810, upload-time = "2025-08-14T12:03:53.808Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/7d/a9f41e814dc4d1dc54e95b2ccaf0b3ebe3eb18b1740df05fe334724c3d89/pyproj-3.7.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:5e26484d80fea56273ed1555abaea161e9661d81a6c07815d54b8e883d4ceb25", size = 9638694, upload-time = "2025-08-14T12:03:55.669Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/ab/9bdb4a6216b712a1f9aab1c0fcbee5d3726f34a366f29c3e8c08a78d6b70/pyproj-3.7.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:281cb92847814e8018010c48b4069ff858a30236638631c1a91dd7bfa68f8a8a", size = 9493977, upload-time = "2025-08-14T12:03:57.937Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/db/2db75b1b6190f1137b1c4e8ef6a22e1c338e46320f6329bfac819143e063/pyproj-3.7.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9c8577f0b7bb09118ec2e57e3babdc977127dd66326d6c5d755c76b063e6d9dc", size = 10841151, upload-time = "2025-08-14T12:04:00.271Z" },
+    { url = "https://files.pythonhosted.org/packages/89/f7/989643394ba23a286e9b7b3f09981496172f9e0d4512457ffea7dc47ffc7/pyproj-3.7.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a23f59904fac3a5e7364b3aa44d288234af267ca041adb2c2b14a903cd5d3ac5", size = 10751585, upload-time = "2025-08-14T12:04:02.228Z" },
+    { url = "https://files.pythonhosted.org/packages/53/6d/ad928fe975a6c14a093c92e6a319ca18f479f3336bb353a740bdba335681/pyproj-3.7.2-cp311-cp311-win32.whl", hash = "sha256:f2af4ed34b2cf3e031a2d85b067a3ecbd38df073c567e04b52fa7a0202afde8a", size = 5908533, upload-time = "2025-08-14T12:04:04.821Z" },
+    { url = "https://files.pythonhosted.org/packages/79/e0/b95584605cec9ed50b7ebaf7975d1c4ddeec5a86b7a20554ed8b60042bd7/pyproj-3.7.2-cp311-cp311-win_amd64.whl", hash = "sha256:0b7cb633565129677b2a183c4d807c727d1c736fcb0568a12299383056e67433", size = 6320742, upload-time = "2025-08-14T12:04:06.357Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/4d/536e8f93bca808175c2d0a5ac9fdf69b960d8ab6b14f25030dccb07464d7/pyproj-3.7.2-cp311-cp311-win_arm64.whl", hash = "sha256:38b08d85e3a38e455625b80e9eb9f78027c8e2649a21dec4df1f9c3525460c71", size = 6245772, upload-time = "2025-08-14T12:04:08.365Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/ab/9893ea9fb066be70ed9074ae543914a618c131ed8dff2da1e08b3a4df4db/pyproj-3.7.2-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:0a9bb26a6356fb5b033433a6d1b4542158fb71e3c51de49b4c318a1dff3aeaab", size = 6219832, upload-time = "2025-08-14T12:04:10.264Z" },
+    { url = "https://files.pythonhosted.org/packages/53/78/4c64199146eed7184eb0e85bedec60a4aa8853b6ffe1ab1f3a8b962e70a0/pyproj-3.7.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:567caa03021178861fad27fabde87500ec6d2ee173dd32f3e2d9871e40eebd68", size = 4620650, upload-time = "2025-08-14T12:04:11.978Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/ac/14a78d17943898a93ef4f8c6a9d4169911c994e3161e54a7cedeba9d8dde/pyproj-3.7.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c203101d1dc3c038a56cff0447acc515dd29d6e14811406ac539c21eed422b2a", size = 9667087, upload-time = "2025-08-14T12:04:13.964Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/be/212882c450bba74fc8d7d35cbd57e4af84792f0a56194819d98106b075af/pyproj-3.7.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:1edc34266c0c23ced85f95a1ee8b47c9035eae6aca5b6b340327250e8e281630", size = 9552797, upload-time = "2025-08-14T12:04:16.624Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/c0/c0f25c87b5d2a8686341c53c1792a222a480d6c9caf60311fec12c99ec26/pyproj-3.7.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa9f26c21bc0e2dc3d224cb1eb4020cf23e76af179a7c66fea49b828611e4260", size = 10837036, upload-time = "2025-08-14T12:04:18.733Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/37/5cbd6772addde2090c91113332623a86e8c7d583eccb2ad02ea634c4a89f/pyproj-3.7.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f9428b318530625cb389b9ddc9c51251e172808a4af79b82809376daaeabe5e9", size = 10775952, upload-time = "2025-08-14T12:04:20.709Z" },
+    { url = "https://files.pythonhosted.org/packages/69/a1/dc250e3cf83eb4b3b9a2cf86fdb5e25288bd40037ae449695550f9e96b2f/pyproj-3.7.2-cp312-cp312-win32.whl", hash = "sha256:b3d99ed57d319da042f175f4554fc7038aa4bcecc4ac89e217e350346b742c9d", size = 5898872, upload-time = "2025-08-14T12:04:22.485Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/a6/6fe724b72b70f2b00152d77282e14964d60ab092ec225e67c196c9b463e5/pyproj-3.7.2-cp312-cp312-win_amd64.whl", hash = "sha256:11614a054cd86a2ed968a657d00987a86eeb91fdcbd9ad3310478685dc14a128", size = 6312176, upload-time = "2025-08-14T12:04:24.736Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/68/915cc32c02a91e76d02c8f55d5a138d6ef9e47a0d96d259df98f4842e558/pyproj-3.7.2-cp312-cp312-win_arm64.whl", hash = "sha256:509a146d1398bafe4f53273398c3bb0b4732535065fa995270e52a9d3676bca3", size = 6233452, upload-time = "2025-08-14T12:04:27.287Z" },
+    { url = "https://files.pythonhosted.org/packages/be/14/faf1b90d267cea68d7e70662e7f88cefdb1bc890bd596c74b959e0517a72/pyproj-3.7.2-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:19466e529b1b15eeefdf8ff26b06fa745856c044f2f77bf0edbae94078c1dfa1", size = 6214580, upload-time = "2025-08-14T12:04:28.804Z" },
+    { url = "https://files.pythonhosted.org/packages/35/48/da9a45b184d375f62667f62eba0ca68569b0bd980a0bb7ffcc1d50440520/pyproj-3.7.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:c79b9b84c4a626c5dc324c0d666be0bfcebd99f7538d66e8898c2444221b3da7", size = 4615388, upload-time = "2025-08-14T12:04:30.553Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/e7/d2b459a4a64bca328b712c1b544e109df88e5c800f7c143cfbc404d39bfb/pyproj-3.7.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:ceecf374cacca317bc09e165db38ac548ee3cad07c3609442bd70311c59c21aa", size = 9628455, upload-time = "2025-08-14T12:04:32.435Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/85/c2b1706e51942de19076eff082f8495e57d5151364e78b5bef4af4a1d94a/pyproj-3.7.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5141a538ffdbe4bfd157421828bb2e07123a90a7a2d6f30fa1462abcfb5ce681", size = 9514269, upload-time = "2025-08-14T12:04:34.599Z" },
+    { url = "https://files.pythonhosted.org/packages/34/38/07a9b89ae7467872f9a476883a5bad9e4f4d1219d31060f0f2b282276cbe/pyproj-3.7.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f000841e98ea99acbb7b8ca168d67773b0191de95187228a16110245c5d954d5", size = 10808437, upload-time = "2025-08-14T12:04:36.485Z" },
+    { url = "https://files.pythonhosted.org/packages/12/56/fda1daeabbd39dec5b07f67233d09f31facb762587b498e6fc4572be9837/pyproj-3.7.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8115faf2597f281a42ab608ceac346b4eb1383d3b45ab474fd37341c4bf82a67", size = 10745540, upload-time = "2025-08-14T12:04:38.568Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/90/c793182cbba65a39a11db2ac6b479fe76c59e6509ae75e5744c344a0da9d/pyproj-3.7.2-cp313-cp313-win32.whl", hash = "sha256:f18c0579dd6be00b970cb1a6719197fceecc407515bab37da0066f0184aafdf3", size = 5896506, upload-time = "2025-08-14T12:04:41.059Z" },
+    { url = "https://files.pythonhosted.org/packages/be/0f/747974129cf0d800906f81cd25efd098c96509026e454d4b66868779ab04/pyproj-3.7.2-cp313-cp313-win_amd64.whl", hash = "sha256:bb41c29d5f60854b1075853fe80c58950b398d4ebb404eb532536ac8d2834ed7", size = 6310195, upload-time = "2025-08-14T12:04:42.974Z" },
+    { url = "https://files.pythonhosted.org/packages/82/64/fc7598a53172c4931ec6edf5228280663063150625d3f6423b4c20f9daff/pyproj-3.7.2-cp313-cp313-win_arm64.whl", hash = "sha256:2b617d573be4118c11cd96b8891a0b7f65778fa7733ed8ecdb297a447d439100", size = 6230748, upload-time = "2025-08-14T12:04:44.491Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/f0/611dd5cddb0d277f94b7af12981f56e1441bf8d22695065d4f0df5218498/pyproj-3.7.2-cp313-cp313t-macosx_13_0_x86_64.whl", hash = "sha256:d27b48f0e81beeaa2b4d60c516c3a1cfbb0c7ff6ef71256d8e9c07792f735279", size = 6241729, upload-time = "2025-08-14T12:04:46.274Z" },
+    { url = "https://files.pythonhosted.org/packages/15/93/40bd4a6c523ff9965e480870611aed7eda5aa2c6128c6537345a2b77b542/pyproj-3.7.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:55a3610d75023c7b1c6e583e48ef8f62918e85a2ae81300569d9f104d6684bb6", size = 4652497, upload-time = "2025-08-14T12:04:48.203Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/ae/7150ead53c117880b35e0d37960d3138fe640a235feb9605cb9386f50bb0/pyproj-3.7.2-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:8d7349182fa622696787cc9e195508d2a41a64765da9b8a6bee846702b9e6220", size = 9942610, upload-time = "2025-08-14T12:04:49.652Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/17/7a4a7eafecf2b46ab64e5c08176c20ceb5844b503eaa551bf12ccac77322/pyproj-3.7.2-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:d230b186eb876ed4f29a7c5ee310144c3a0e44e89e55f65fb3607e13f6db337c", size = 9692390, upload-time = "2025-08-14T12:04:51.731Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/55/ae18f040f6410f0ea547a21ada7ef3e26e6c82befa125b303b02759c0e9d/pyproj-3.7.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:237499c7862c578d0369e2b8ac56eec550e391a025ff70e2af8417139dabb41c", size = 11047596, upload-time = "2025-08-14T12:04:53.748Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/2e/d3fff4d2909473f26ae799f9dda04caa322c417a51ff3b25763f7d03b233/pyproj-3.7.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8c225f5978abd506fd9a78eaaf794435e823c9156091cabaab5374efb29d7f69", size = 10896975, upload-time = "2025-08-14T12:04:55.875Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/bc/8fc7d3963d87057b7b51ebe68c1e7c51c23129eee5072ba6b86558544a46/pyproj-3.7.2-cp313-cp313t-win32.whl", hash = "sha256:2da731876d27639ff9d2d81c151f6ab90a1546455fabd93368e753047be344a2", size = 5953057, upload-time = "2025-08-14T12:04:58.466Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/27/ea9809966cc47d2d51e6d5ae631ea895f7c7c7b9b3c29718f900a8f7d197/pyproj-3.7.2-cp313-cp313t-win_amd64.whl", hash = "sha256:f54d91ae18dd23b6c0ab48126d446820e725419da10617d86a1b69ada6d881d3", size = 6375414, upload-time = "2025-08-14T12:04:59.861Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/f8/1ef0129fba9a555c658e22af68989f35e7ba7b9136f25758809efec0cd6e/pyproj-3.7.2-cp313-cp313t-win_arm64.whl", hash = "sha256:fc52ba896cfc3214dc9f9ca3c0677a623e8fdd096b257c14a31e719d21ff3fdd", size = 6262501, upload-time = "2025-08-14T12:05:01.39Z" },
+    { url = "https://files.pythonhosted.org/packages/42/17/c2b050d3f5b71b6edd0d96ae16c990fdc42a5f1366464a5c2772146de33a/pyproj-3.7.2-cp314-cp314-macosx_13_0_x86_64.whl", hash = "sha256:2aaa328605ace41db050d06bac1adc11f01b71fe95c18661497763116c3a0f02", size = 6214541, upload-time = "2025-08-14T12:05:03.166Z" },
+    { url = "https://files.pythonhosted.org/packages/03/68/68ada9c8aea96ded09a66cfd9bf87aa6db8c2edebe93f5bf9b66b0143fbc/pyproj-3.7.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:35dccbce8201313c596a970fde90e33605248b66272595c061b511c8100ccc08", size = 4617456, upload-time = "2025-08-14T12:05:04.563Z" },
+    { url = "https://files.pythonhosted.org/packages/81/e4/4c50ceca7d0e937977866b02cb64e6ccf4df979a5871e521f9e255df6073/pyproj-3.7.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:25b0b7cb0042444c29a164b993c45c1b8013d6c48baa61dc1160d834a277e83b", size = 9615590, upload-time = "2025-08-14T12:05:06.094Z" },
+    { url = "https://files.pythonhosted.org/packages/05/1e/ada6fb15a1d75b5bd9b554355a69a798c55a7dcc93b8d41596265c1772e3/pyproj-3.7.2-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:85def3a6388e9ba51f964619aa002a9d2098e77c6454ff47773bb68871024281", size = 9474960, upload-time = "2025-08-14T12:05:07.973Z" },
+    { url = "https://files.pythonhosted.org/packages/51/07/9d48ad0a8db36e16f842f2c8a694c1d9d7dcf9137264846bef77585a71f3/pyproj-3.7.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b1bccefec3875ab81eabf49059e2b2ea77362c178b66fd3528c3e4df242f1516", size = 10799478, upload-time = "2025-08-14T12:05:14.102Z" },
+    { url = "https://files.pythonhosted.org/packages/85/cf/2f812b529079f72f51ff2d6456b7fef06c01735e5cfd62d54ffb2b548028/pyproj-3.7.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d5371ca114d6990b675247355a801925814eca53e6c4b2f1b5c0a956336ee36e", size = 10710030, upload-time = "2025-08-14T12:05:16.317Z" },
+    { url = "https://files.pythonhosted.org/packages/99/9b/4626a19e1f03eba4c0e77b91a6cf0f73aa9cb5d51a22ee385c22812bcc2c/pyproj-3.7.2-cp314-cp314-win32.whl", hash = "sha256:77f066626030f41be543274f5ac79f2a511fe89860ecd0914f22131b40a0ec25", size = 5991181, upload-time = "2025-08-14T12:05:19.492Z" },
+    { url = "https://files.pythonhosted.org/packages/04/b2/5a6610554306a83a563080c2cf2c57565563eadd280e15388efa00fb5b33/pyproj-3.7.2-cp314-cp314-win_amd64.whl", hash = "sha256:5a964da1696b8522806f4276ab04ccfff8f9eb95133a92a25900697609d40112", size = 6434721, upload-time = "2025-08-14T12:05:21.022Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/ce/6c910ea2e1c74ef673c5d48c482564b8a7824a44c4e35cca2e765b68cfcc/pyproj-3.7.2-cp314-cp314-win_arm64.whl", hash = "sha256:e258ab4dbd3cf627809067c0ba8f9884ea76c8e5999d039fb37a1619c6c3e1f6", size = 6363821, upload-time = "2025-08-14T12:05:22.627Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/e4/5532f6f7491812ba782a2177fe9de73fd8e2912b59f46a1d056b84b9b8f2/pyproj-3.7.2-cp314-cp314t-macosx_13_0_x86_64.whl", hash = "sha256:bbbac2f930c6d266f70ec75df35ef851d96fdb3701c674f42fd23a9314573b37", size = 6241773, upload-time = "2025-08-14T12:05:24.577Z" },
+    { url = "https://files.pythonhosted.org/packages/20/1f/0938c3f2bbbef1789132d1726d9b0e662f10cfc22522743937f421ad664e/pyproj-3.7.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:b7544e0a3d6339dc9151e9c8f3ea62a936ab7cc446a806ec448bbe86aebb979b", size = 4652537, upload-time = "2025-08-14T12:05:26.391Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/a8/488b1ed47d25972f33874f91f09ca8f2227902f05f63a2b80dc73e7b1c97/pyproj-3.7.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f7f5133dca4c703e8acadf6f30bc567d39a42c6af321e7f81975c2518f3ed357", size = 9940864, upload-time = "2025-08-14T12:05:27.985Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/cc/7f4c895d0cb98e47b6a85a6d79eaca03eb266129eed2f845125c09cf31ff/pyproj-3.7.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:5aff3343038d7426aa5076f07feb88065f50e0502d1b0d7c22ddfdd2c75a3f81", size = 9688868, upload-time = "2025-08-14T12:05:30.425Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/b7/c7e306b8bb0f071d9825b753ee4920f066c40fbfcce9372c4f3cfb2fc4ed/pyproj-3.7.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b0552178c61f2ac1c820d087e8ba6e62b29442debddbb09d51c4bf8acc84d888", size = 11045910, upload-time = "2025-08-14T12:05:32.507Z" },
+    { url = "https://files.pythonhosted.org/packages/42/fb/538a4d2df695980e2dde5c04d965fbdd1fe8c20a3194dc4aaa3952a4d1be/pyproj-3.7.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:47d87db2d2c436c5fd0409b34d70bb6cdb875cca2ebe7a9d1c442367b0ab8d59", size = 10895724, upload-time = "2025-08-14T12:05:35.465Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/8b/a3f0618b03957de9db5489a04558a8826f43906628bb0b766033aa3b5548/pyproj-3.7.2-cp314-cp314t-win32.whl", hash = "sha256:c9b6f1d8ad3e80a0ee0903a778b6ece7dca1d1d40f6d114ae01bc8ddbad971aa", size = 6056848, upload-time = "2025-08-14T12:05:37.553Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/56/413240dd5149dd3291eda55aa55a659da4431244a2fd1319d0ae89407cfb/pyproj-3.7.2-cp314-cp314t-win_amd64.whl", hash = "sha256:1914e29e27933ba6f9822663ee0600f169014a2859f851c054c88cf5ea8a333c", size = 6517676, upload-time = "2025-08-14T12:05:39.126Z" },
+    { url = "https://files.pythonhosted.org/packages/15/73/a7141a1a0559bf1a7aa42a11c879ceb19f02f5c6c371c6d57fd86cefd4d1/pyproj-3.7.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d9d25bae416a24397e0d85739f84d323b55f6511e45a522dd7d7eae70d10c7e4", size = 6391844, upload-time = "2025-08-14T12:05:40.745Z" },
 ]
 
 [[package]]
@@ -2339,9 +2811,9 @@ dependencies = [
     { name = "nodeenv" },
     { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/f7/16/6b4fbdd1fef59a0292cbb99f790b44983e390321eccbc5921b4d161da5d1/pyright-1.1.406.tar.gz", hash = "sha256:c4872bc58c9643dac09e8a2e74d472c62036910b3bd37a32813989ef7576ea2c", size = 4113151 }
+sdist = { url = "https://files.pythonhosted.org/packages/f7/16/6b4fbdd1fef59a0292cbb99f790b44983e390321eccbc5921b4d161da5d1/pyright-1.1.406.tar.gz", hash = "sha256:c4872bc58c9643dac09e8a2e74d472c62036910b3bd37a32813989ef7576ea2c", size = 4113151, upload-time = "2025-10-02T01:04:45.488Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f6/a2/e309afbb459f50507103793aaef85ca4348b66814c86bc73908bdeb66d12/pyright-1.1.406-py3-none-any.whl", hash = "sha256:1d81fb43c2407bf566e97e57abb01c811973fdb21b2df8df59f870f688bdca71", size = 5980982 },
+    { url = "https://files.pythonhosted.org/packages/f6/a2/e309afbb459f50507103793aaef85ca4348b66814c86bc73908bdeb66d12/pyright-1.1.406-py3-none-any.whl", hash = "sha256:1d81fb43c2407bf566e97e57abb01c811973fdb21b2df8df59f870f688bdca71", size = 5980982, upload-time = "2025-10-02T01:04:43.137Z" },
 ]
 
 [[package]]
@@ -2357,9 +2829,9 @@ dependencies = [
     { name = "pygments" },
     { name = "tomli", marker = "python_full_version < '3.11'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618 }
+sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750 },
+    { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" },
 ]
 
 [[package]]
@@ -2370,9 +2842,9 @@ dependencies = [
     { name = "py-cpuinfo" },
     { name = "pytest" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810 }
+sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810, upload-time = "2024-10-30T11:51:48.521Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259 },
+    { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259, upload-time = "2024-10-30T11:51:45.94Z" },
 ]
 
 [[package]]
@@ -2382,97 +2854,106 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "six" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432 }
+sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892 },
+    { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
 ]
 
 [[package]]
 name = "pytz"
 version = "2025.2"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884 }
+sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225 },
+    { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" },
 ]
 
 [[package]]
 name = "pyyaml"
 version = "6.0.3"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227 },
-    { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019 },
-    { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646 },
-    { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793 },
-    { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293 },
-    { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872 },
-    { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828 },
-    { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415 },
-    { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561 },
-    { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826 },
-    { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577 },
-    { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556 },
-    { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114 },
-    { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638 },
-    { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463 },
-    { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986 },
-    { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543 },
-    { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763 },
-    { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063 },
-    { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973 },
-    { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116 },
-    { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011 },
-    { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870 },
-    { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089 },
-    { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181 },
-    { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658 },
-    { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003 },
-    { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344 },
-    { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669 },
-    { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252 },
-    { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081 },
-    { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159 },
-    { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626 },
-    { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613 },
-    { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115 },
-    { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427 },
-    { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090 },
-    { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246 },
-    { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814 },
-    { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809 },
-    { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454 },
-    { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355 },
-    { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175 },
-    { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228 },
-    { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194 },
-    { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429 },
-    { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912 },
-    { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108 },
-    { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641 },
-    { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901 },
-    { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132 },
-    { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261 },
-    { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272 },
-    { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923 },
-    { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062 },
-    { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341 },
+sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" },
+    { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" },
+    { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" },
+    { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" },
+    { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" },
+    { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" },
+    { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" },
+    { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" },
+    { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" },
+    { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" },
+    { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" },
+    { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" },
+    { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" },
+    { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" },
+    { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" },
+    { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" },
+    { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" },
+    { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" },
+    { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" },
+    { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" },
+    { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" },
+    { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" },
+    { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" },
+    { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" },
+    { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/62/67fc8e68a75f738c9200422bf65693fb79a4cd0dc5b23310e5202e978090/pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da", size = 184450, upload-time = "2025-09-25T21:33:00.618Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/92/861f152ce87c452b11b9d0977952259aa7df792d71c1053365cc7b09cc08/pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917", size = 174319, upload-time = "2025-09-25T21:33:02.086Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/cd/f0cfc8c74f8a030017a2b9c771b7f47e5dd702c3e28e5b2071374bda2948/pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9", size = 737631, upload-time = "2025-09-25T21:33:03.25Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/b2/18f2bd28cd2055a79a46c9b0895c0b3d987ce40ee471cecf58a1a0199805/pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5", size = 836795, upload-time = "2025-09-25T21:33:05.014Z" },
+    { url = "https://files.pythonhosted.org/packages/73/b9/793686b2d54b531203c160ef12bec60228a0109c79bae6c1277961026770/pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a", size = 750767, upload-time = "2025-09-25T21:33:06.398Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/86/a137b39a611def2ed78b0e66ce2fe13ee701a07c07aebe55c340ed2a050e/pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926", size = 727982, upload-time = "2025-09-25T21:33:08.708Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/62/71c27c94f457cf4418ef8ccc71735324c549f7e3ea9d34aba50874563561/pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7", size = 755677, upload-time = "2025-09-25T21:33:09.876Z" },
+    { url = "https://files.pythonhosted.org/packages/29/3d/6f5e0d58bd924fb0d06c3a6bad00effbdae2de5adb5cda5648006ffbd8d3/pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0", size = 142592, upload-time = "2025-09-25T21:33:10.983Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/0c/25113e0b5e103d7f1490c0e947e303fe4a696c10b501dea7a9f49d4e876c/pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007", size = 158777, upload-time = "2025-09-25T21:33:15.55Z" },
 ]
 
 [[package]]
 name = "requests"
-version = "2.32.5"
+version = "2.33.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "certifi" },
-    { name = "charset-normalizer" },
-    { name = "idna" },
-    { name = "urllib3" },
+    { name = "certifi", marker = "python_full_version >= '3.10'" },
+    { name = "charset-normalizer", marker = "python_full_version >= '3.10'" },
+    { name = "idna", marker = "python_full_version >= '3.10'" },
+    { name = "urllib3", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517 }
+sdist = { url = "https://files.pythonhosted.org/packages/34/64/8860370b167a9721e8956ae116825caff829224fbca0ca6e7bf8ddef8430/requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652", size = 134232, upload-time = "2026-03-25T15:10:41.586Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738 },
+    { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" },
 ]
 
 [[package]]
@@ -2480,36 +2961,37 @@ name = "rich"
 version = "14.1.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "markdown-it-py" },
-    { name = "pygments" },
+    { name = "markdown-it-py", marker = "python_full_version >= '3.10'" },
+    { name = "pygments", marker = "python_full_version >= '3.10'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/fe/75/af448d8e52bf1d8fa6a9d089ca6c07ff4453d86c65c145d0a300bb073b9b/rich-14.1.0.tar.gz", hash = "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8", size = 224441 }
+sdist = { url = "https://files.pythonhosted.org/packages/fe/75/af448d8e52bf1d8fa6a9d089ca6c07ff4453d86c65c145d0a300bb073b9b/rich-14.1.0.tar.gz", hash = "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8", size = 224441, upload-time = "2025-07-25T07:32:58.125Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/e3/30/3c4d035596d3cf444529e0b2953ad0466f6049528a879d27534700580395/rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", size = 243368 },
+    { url = "https://files.pythonhosted.org/packages/e3/30/3c4d035596d3cf444529e0b2953ad0466f6049528a879d27534700580395/rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", size = 243368, upload-time = "2025-07-25T07:32:56.73Z" },
 ]
 
 [[package]]
 name = "ruff"
-version = "0.4.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/cd/76/667f7536232ff6c3769a13f8d67911e038367350f7c3b3e09c4d98648fbc/ruff-0.4.1.tar.gz", hash = "sha256:d592116cdbb65f8b1b7e2a2b48297eb865f6bdc20641879aa9d7b9c11d86db79", size = 2309884 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/84/7a/1eea0f76c900b824f50631645dd84a1e4e3bb52b44642c1b82e808375259/ruff-0.4.1-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2d9ef6231e3fbdc0b8c72404a1a0c46fd0dcea84efca83beb4681c318ea6a953", size = 16559105 },
-    { url = "https://files.pythonhosted.org/packages/52/29/ce2d1aa82f0c8db7b1468fd4adf921c8572dfc2c95d25df2a7980edc4764/ruff-0.4.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9485f54a7189e6f7433e0058cf8581bee45c31a25cd69009d2a040d1bd4bfaef", size = 8496889 },
-    { url = "https://files.pythonhosted.org/packages/b0/19/a1c7c7b9f15c58195675abad31ac4b4555c8b94d147ba8c7e9f6fcb9043f/ruff-0.4.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2921ac03ce1383e360e8a95442ffb0d757a6a7ddd9a5be68561a671e0e5807e", size = 8133815 },
-    { url = "https://files.pythonhosted.org/packages/fa/b5/d48020b41bd05a47be6c53d59e5fa8cbbe3bda597535286948d27415c1f6/ruff-0.4.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eec8d185fe193ad053eda3a6be23069e0c8ba8c5d20bc5ace6e3b9e37d246d3f", size = 7514442 },
-    { url = "https://files.pythonhosted.org/packages/b1/f5/4f81560b8b555fda93ac624d5e534cba8c2362aa29eebd7a1ebb20185bd3/ruff-0.4.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:baa27d9d72a94574d250f42b7640b3bd2edc4c58ac8ac2778a8c82374bb27984", size = 8681129 },
-    { url = "https://files.pythonhosted.org/packages/e6/1c/66ed2617bfa589ff88490448bb49385bd776c7f39d09359e46cdcc78e537/ruff-0.4.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:f1ee41580bff1a651339eb3337c20c12f4037f6110a36ae4a2d864c52e5ef954", size = 9445917 },
-    { url = "https://files.pythonhosted.org/packages/36/8a/de76c13f9e1ce00bb03a70b371a3cd2cec86634263001b7afe8473f9da80/ruff-0.4.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0926cefb57fc5fced629603fbd1a23d458b25418681d96823992ba975f050c2b", size = 9143366 },
-    { url = "https://files.pythonhosted.org/packages/29/1d/d49911b2ad919575b0895043b97db81cce401635eb20cd8ebba949a038b9/ruff-0.4.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c6e37f2e3cd74496a74af9a4fa67b547ab3ca137688c484749189bf3a686ceb", size = 10109996 },
-    { url = "https://files.pythonhosted.org/packages/bd/38/0c172941d736433c494c5f3d3ce476c7a8060c70e2d8a2ab73fabd5e5869/ruff-0.4.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efd703a5975ac1998c2cc5e9494e13b28f31e66c616b0a76e206de2562e0843c", size = 8701793 },
-    { url = "https://files.pythonhosted.org/packages/28/fa/70236002bc002edca0a3d4ed2e45f3d3f499a45a3e9fd606c87f2c5822fe/ruff-0.4.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:b92f03b4aa9fa23e1799b40f15f8b95cdc418782a567d6c43def65e1bbb7f1cf", size = 8029779 },
-    { url = "https://files.pythonhosted.org/packages/c4/3f/11b0a93ae0e50bc323bfe74a70bbd2b84f6e1cd83b71967f9f34d9032d91/ruff-0.4.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1c859f294f8633889e7d77de228b203eb0e9a03071b72b5989d89a0cf98ee262", size = 7527013 },
-    { url = "https://files.pythonhosted.org/packages/50/19/1d25f4daf6518f615676cab90d205ef1e221500f95e4a79325e4cd2bf937/ruff-0.4.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:b34510141e393519a47f2d7b8216fec747ea1f2c81e85f076e9f2910588d4b64", size = 8292363 },
-    { url = "https://files.pythonhosted.org/packages/70/61/ad210ae4b48f15de5630d96eede38a6d98984130fb3b61e7000721848b71/ruff-0.4.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:6e68d248ed688b9d69fd4d18737edcbb79c98b251bba5a2b031ce2470224bdf9", size = 8757422 },
-    { url = "https://files.pythonhosted.org/packages/1c/30/b5f9fa73d7be01336750c8ca5bb3dc018a0ad29eb7e6d642785323bc67e5/ruff-0.4.1-py3-none-win32.whl", hash = "sha256:b90506f3d6d1f41f43f9b7b5ff845aeefabed6d2494307bc7b178360a8805252", size = 7600160 },
-    { url = "https://files.pythonhosted.org/packages/90/8f/07e0b4e24337ca92521472b8f8030f450c9765fe7bcd177ff248f708c028/ruff-0.4.1-py3-none-win_amd64.whl", hash = "sha256:c7d391e5936af5c9e252743d767c564670dc3889aff460d35c518ee76e4b26d7", size = 8451086 },
-    { url = "https://files.pythonhosted.org/packages/16/61/1843c9b453cd58b9c9e928388b609ebe70a63f3291bb10e7cfa24de69d03/ruff-0.4.1-py3-none-win_arm64.whl", hash = "sha256:a1eaf03d87e6a7cd5e661d36d8c6e874693cb9bc3049d110bc9a97b350680c43", size = 7964199 },
+version = "0.11.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/90/61/fb87430f040e4e577e784e325351186976516faef17d6fcd921fe28edfd7/ruff-0.11.2.tar.gz", hash = "sha256:ec47591497d5a1050175bdf4e1a4e6272cddff7da88a2ad595e1e326041d8d94", size = 3857511, upload-time = "2025-03-21T13:31:17.419Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/62/99/102578506f0f5fa29fd7e0df0a273864f79af044757aef73d1cae0afe6ad/ruff-0.11.2-py3-none-linux_armv6l.whl", hash = "sha256:c69e20ea49e973f3afec2c06376eb56045709f0212615c1adb0eda35e8a4e477", size = 10113146, upload-time = "2025-03-21T13:30:26.68Z" },
+    { url = "https://files.pythonhosted.org/packages/74/ad/5cd4ba58ab602a579997a8494b96f10f316e874d7c435bcc1a92e6da1b12/ruff-0.11.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:2c5424cc1c4eb1d8ecabe6d4f1b70470b4f24a0c0171356290b1953ad8f0e272", size = 10867092, upload-time = "2025-03-21T13:30:37.949Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/3e/d3f13619e1d152c7b600a38c1a035e833e794c6625c9a6cea6f63dbf3af4/ruff-0.11.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:ecf20854cc73f42171eedb66f006a43d0a21bfb98a2523a809931cda569552d9", size = 10224082, upload-time = "2025-03-21T13:30:39.962Z" },
+    { url = "https://files.pythonhosted.org/packages/90/06/f77b3d790d24a93f38e3806216f263974909888fd1e826717c3ec956bbcd/ruff-0.11.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c543bf65d5d27240321604cee0633a70c6c25c9a2f2492efa9f6d4b8e4199bb", size = 10394818, upload-time = "2025-03-21T13:30:42.551Z" },
+    { url = "https://files.pythonhosted.org/packages/99/7f/78aa431d3ddebfc2418cd95b786642557ba8b3cb578c075239da9ce97ff9/ruff-0.11.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20967168cc21195db5830b9224be0e964cc9c8ecf3b5a9e3ce19876e8d3a96e3", size = 9952251, upload-time = "2025-03-21T13:30:45.196Z" },
+    { url = "https://files.pythonhosted.org/packages/30/3e/f11186d1ddfaca438c3bbff73c6a2fdb5b60e6450cc466129c694b0ab7a2/ruff-0.11.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:955a9ce63483999d9f0b8f0b4a3ad669e53484232853054cc8b9d51ab4c5de74", size = 11563566, upload-time = "2025-03-21T13:30:47.516Z" },
+    { url = "https://files.pythonhosted.org/packages/22/6c/6ca91befbc0a6539ee133d9a9ce60b1a354db12c3c5d11cfdbf77140f851/ruff-0.11.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:86b3a27c38b8fce73bcd262b0de32e9a6801b76d52cdb3ae4c914515f0cef608", size = 12208721, upload-time = "2025-03-21T13:30:49.56Z" },
+    { url = "https://files.pythonhosted.org/packages/19/b0/24516a3b850d55b17c03fc399b681c6a549d06ce665915721dc5d6458a5c/ruff-0.11.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a3b66a03b248c9fcd9d64d445bafdf1589326bee6fc5c8e92d7562e58883e30f", size = 11662274, upload-time = "2025-03-21T13:30:52.055Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/65/76be06d28ecb7c6070280cef2bcb20c98fbf99ff60b1c57d2fb9b8771348/ruff-0.11.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0397c2672db015be5aa3d4dac54c69aa012429097ff219392c018e21f5085147", size = 13792284, upload-time = "2025-03-21T13:30:54.24Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/d2/4ceed7147e05852876f3b5f3fdc23f878ce2b7e0b90dd6e698bda3d20787/ruff-0.11.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:869bcf3f9abf6457fbe39b5a37333aa4eecc52a3b99c98827ccc371a8e5b6f1b", size = 11327861, upload-time = "2025-03-21T13:30:56.757Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/78/4935ecba13706fd60ebe0e3dc50371f2bdc3d9bc80e68adc32ff93914534/ruff-0.11.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:2a2b50ca35457ba785cd8c93ebbe529467594087b527a08d487cf0ee7b3087e9", size = 10276560, upload-time = "2025-03-21T13:30:58.881Z" },
+    { url = "https://files.pythonhosted.org/packages/81/7f/1b2435c3f5245d410bb5dc80f13ec796454c21fbda12b77d7588d5cf4e29/ruff-0.11.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7c69c74bf53ddcfbc22e6eb2f31211df7f65054bfc1f72288fc71e5f82db3eab", size = 9945091, upload-time = "2025-03-21T13:31:01.45Z" },
+    { url = "https://files.pythonhosted.org/packages/39/c4/692284c07e6bf2b31d82bb8c32f8840f9d0627d92983edaac991a2b66c0a/ruff-0.11.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6e8fb75e14560f7cf53b15bbc55baf5ecbe373dd5f3aab96ff7aa7777edd7630", size = 10977133, upload-time = "2025-03-21T13:31:04.013Z" },
+    { url = "https://files.pythonhosted.org/packages/94/cf/8ab81cb7dd7a3b0a3960c2769825038f3adcd75faf46dd6376086df8b128/ruff-0.11.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:842a472d7b4d6f5924e9297aa38149e5dcb1e628773b70e6387ae2c97a63c58f", size = 11378514, upload-time = "2025-03-21T13:31:06.166Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/3a/a647fa4f316482dacf2fd68e8a386327a33d6eabd8eb2f9a0c3d291ec549/ruff-0.11.2-py3-none-win32.whl", hash = "sha256:aca01ccd0eb5eb7156b324cfaa088586f06a86d9e5314b0eb330cb48415097cc", size = 10319835, upload-time = "2025-03-21T13:31:10.7Z" },
+    { url = "https://files.pythonhosted.org/packages/86/54/3c12d3af58012a5e2cd7ebdbe9983f4834af3f8cbea0e8a8c74fa1e23b2b/ruff-0.11.2-py3-none-win_amd64.whl", hash = "sha256:3170150172a8f994136c0c66f494edf199a0bbea7a409f649e4bc8f4d7084080", size = 11373713, upload-time = "2025-03-21T13:31:13.148Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/d4/dd813703af8a1e2ac33bf3feb27e8a5ad514c9f219df80c64d69807e7f71/ruff-0.11.2-py3-none-win_arm64.whl", hash = "sha256:52933095158ff328f4c77af3d74f0379e34fd52f175144cefc1b192e7ccd32b4", size = 10441990, upload-time = "2025-03-21T13:31:15.206Z" },
 ]
 
 [[package]]
@@ -2519,27 +3001,27 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "botocore" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/62/74/8d69dcb7a9efe8baa2046891735e5dfe433ad558ae23d9e3c14c633d1d58/s3transfer-0.14.0.tar.gz", hash = "sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125", size = 151547 }
+sdist = { url = "https://files.pythonhosted.org/packages/62/74/8d69dcb7a9efe8baa2046891735e5dfe433ad558ae23d9e3c14c633d1d58/s3transfer-0.14.0.tar.gz", hash = "sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125", size = 151547, upload-time = "2025-09-09T19:23:31.089Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712 },
+    { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" },
 ]
 
 [[package]]
 name = "setuptools"
 version = "80.9.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958 }
+sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486 },
+    { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" },
 ]
 
 [[package]]
 name = "six"
 version = "1.17.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031 }
+sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050 },
+    { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" },
 ]
 
 [[package]]
@@ -2549,9 +3031,9 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "mpmath" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921 }
+sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353 },
+    { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" },
 ]
 
 [[package]]
@@ -2562,17 +3044,19 @@ dependencies = [
     { name = "absl-py" },
     { name = "grpcio" },
     { name = "markdown" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "packaging" },
     { name = "pillow" },
-    { name = "protobuf" },
+    { name = "protobuf", version = "3.19.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "protobuf", version = "6.32.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
     { name = "setuptools" },
     { name = "tensorboard-data-server" },
     { name = "werkzeug" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl", hash = "sha256:9dc9f978cb84c0723acf9a345d96c184f0293d18f166bb8d59ee098e6cfaaba6", size = 5525680 },
+    { url = "https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl", hash = "sha256:9dc9f978cb84c0723acf9a345d96c184f0293d18f166bb8d59ee098e6cfaaba6", size = 5525680, upload-time = "2025-07-17T19:20:49.638Z" },
 ]
 
 [[package]]
@@ -2580,95 +3064,161 @@ name = "tensorboard-data-server"
 version = "0.7.2"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb", size = 2356 },
-    { url = "https://files.pythonhosted.org/packages/73/c6/825dab04195756cf8ff2e12698f22513b3db2f64925bdd41671bfb33aaa5/tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530", size = 6590363 },
+    { url = "https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb", size = 2356, upload-time = "2023-10-23T21:23:32.16Z" },
+    { url = "https://files.pythonhosted.org/packages/73/c6/825dab04195756cf8ff2e12698f22513b3db2f64925bdd41671bfb33aaa5/tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530", size = 6590363, upload-time = "2023-10-23T21:23:35.583Z" },
+]
+
+[[package]]
+name = "tensorflow"
+version = "2.7.4"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+dependencies = [
+    { name = "absl-py", marker = "python_full_version < '3.10'" },
+    { name = "astunparse", marker = "python_full_version < '3.10'" },
+    { name = "flatbuffers", version = "2.0.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "gast", version = "0.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "google-pasta", marker = "python_full_version < '3.10'" },
+    { name = "grpcio", marker = "python_full_version < '3.10'" },
+    { name = "h5py", marker = "python_full_version < '3.10'" },
+    { name = "keras", version = "2.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "keras-preprocessing", marker = "python_full_version < '3.10'" },
+    { name = "libclang", marker = "python_full_version < '3.10'" },
+    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "opt-einsum", marker = "python_full_version < '3.10'" },
+    { name = "protobuf", version = "3.19.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "six", marker = "python_full_version < '3.10'" },
+    { name = "tensorboard", marker = "python_full_version < '3.10'" },
+    { name = "tensorflow-estimator", marker = "python_full_version < '3.10'" },
+    { name = "tensorflow-io-gcs-filesystem", marker = "python_full_version < '3.10'" },
+    { name = "termcolor", marker = "python_full_version < '3.10'" },
+    { name = "typing-extensions", marker = "python_full_version < '3.10'" },
+    { name = "wheel", marker = "python_full_version < '3.10'" },
+    { name = "wrapt", marker = "python_full_version < '3.10'" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5e/31/d49a3dff9c4ca6e6c09c2c5fea95f58cf59cc3cd4f0d557069c7dccd6f57/tensorflow-2.7.4-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:c4597635dd71fc6809b7fffcb462524d73e2ade09da61844059e6a2fead71140", size = 496066688, upload-time = "2022-09-02T19:11:01.631Z" },
 ]
 
 [[package]]
 name = "tensorflow"
 version = "2.20.0"
 source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version >= '3.14'",
+    "python_full_version == '3.13.*'",
+    "python_full_version == '3.12.*'",
+    "python_full_version == '3.11.*'",
+    "python_full_version == '3.10.*'",
+]
 dependencies = [
-    { name = "absl-py" },
-    { name = "astunparse" },
-    { name = "flatbuffers" },
-    { name = "gast" },
-    { name = "google-pasta" },
-    { name = "grpcio" },
-    { name = "h5py" },
-    { name = "keras" },
-    { name = "libclang" },
-    { name = "ml-dtypes" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "absl-py", marker = "python_full_version >= '3.10'" },
+    { name = "astunparse", marker = "python_full_version >= '3.10'" },
+    { name = "flatbuffers", version = "25.9.23", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "gast", version = "0.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "google-pasta", marker = "python_full_version >= '3.10'" },
+    { name = "grpcio", marker = "python_full_version >= '3.10'" },
+    { name = "h5py", marker = "python_full_version >= '3.10'" },
+    { name = "keras", version = "3.11.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "libclang", marker = "python_full_version >= '3.10'" },
+    { name = "ml-dtypes", marker = "python_full_version >= '3.10'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "opt-einsum" },
-    { name = "packaging" },
-    { name = "protobuf" },
-    { name = "requests" },
-    { name = "setuptools" },
-    { name = "six" },
-    { name = "tensorboard" },
-    { name = "termcolor" },
-    { name = "typing-extensions" },
-    { name = "wrapt" },
+    { name = "opt-einsum", marker = "python_full_version >= '3.10'" },
+    { name = "packaging", marker = "python_full_version >= '3.10'" },
+    { name = "protobuf", version = "6.32.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "requests", marker = "python_full_version >= '3.10'" },
+    { name = "setuptools", marker = "python_full_version >= '3.10'" },
+    { name = "six", marker = "python_full_version >= '3.10'" },
+    { name = "tensorboard", marker = "python_full_version >= '3.10'" },
+    { name = "termcolor", marker = "python_full_version >= '3.10'" },
+    { name = "typing-extensions", marker = "python_full_version >= '3.10'" },
+    { name = "wrapt", marker = "python_full_version >= '3.10'" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ff/07/ea91ac67a9fd36d3372099f5a3e69860ded544f877f5f2117802388f4212/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02a0293d94f5c8b7125b66abf622cc4854a33ae9d618a0d41309f95e091bbaea", size = 259307122, upload-time = "2025-08-13T16:50:47.909Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/9e/0d57922cf46b9e91de636cd5b5e0d7a424ebe98f3245380a713f1f6c2a0b/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7abd7f3a010e0d354dc804182372779a722d474c4d8a3db8f4a3f5baef2a591e", size = 620425510, upload-time = "2025-08-13T16:51:02.608Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/b7/a3d455db88ab5b35ce53ab885ec0dd9f28d905a86a2250423048bc8cafa0/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e9568c8efcb05c0266be223e3269c62ebf7ad3498f156438311735f6fa5ced5", size = 259465882, upload-time = "2025-08-13T16:51:39.546Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/0c/7df285ee8a88139fab0b237003634d90690759fae9c18f55ddb7c04656ec/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:481499fd0f824583de8945be61d5e827898cdaa4f5ea1bc2cc28ca2ccff8229e", size = 620570129, upload-time = "2025-08-13T16:51:55.104Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/b4/f028a5de27d0fda10ba6145bc76e40c37ff6d2d1e95b601adb5ae17d635e/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bfbfb3dd0e22bffc45fe1e922390d27753e99261fab8a882e802cf98a0e078f", size = 259533109, upload-time = "2025-08-13T16:52:31.513Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/d1/6aa15085d672056d5f08b5f28b1c7ce01c4e12149a23b0c98e3c79d04441/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25265b0bc527e0d54b1e9cc60c44a24f44a809fe27666b905f0466471f9c52ec", size = 620682547, upload-time = "2025-08-13T16:52:46.396Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/4c/c1aa90c5cc92e9f7f9c78421e121ef25bae7d378f8d1d4cbad46c6308836/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47c88e05a07f1ead4977b4894b3ecd4d8075c40191065afc4fd9355c9db3d926", size = 259663776, upload-time = "2025-08-13T16:53:24.507Z" },
+    { url = "https://files.pythonhosted.org/packages/43/fb/8be8547c128613d82a2b006004026d86ed0bd672e913029a98153af4ffab/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fa3729b0126f75a99882b89fb7d536515721eda8014a63e259e780ba0a37372", size = 620815537, upload-time = "2025-08-13T16:53:42.577Z" },
+    { url = "https://files.pythonhosted.org/packages/83/ff/a26d49895586207b2704403366ef976dcaa6ed07514699dae9a4fc3fa1a9/tensorflow-2.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28bc33759249c98eabcee9debd24e74506bbe29ac139e050cf0c74aa9888ebdf", size = 259307564, upload-time = "2025-08-13T16:54:17.691Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/fe/f3d738dc7c93ed5f67f9ace8dd3ed66971dab7c5a47f2d1c504ef0d0cf1d/tensorflow-2.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0deb5c583dfc53b54fd158a194ce0087b406bb6518af400ca3809735e4548ec3", size = 620427169, upload-time = "2025-08-13T16:54:33.431Z" },
 ]
+
+[[package]]
+name = "tensorflow-estimator"
+version = "2.7.0"
+source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ff/07/ea91ac67a9fd36d3372099f5a3e69860ded544f877f5f2117802388f4212/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02a0293d94f5c8b7125b66abf622cc4854a33ae9d618a0d41309f95e091bbaea", size = 259307122 },
-    { url = "https://files.pythonhosted.org/packages/e5/9e/0d57922cf46b9e91de636cd5b5e0d7a424ebe98f3245380a713f1f6c2a0b/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7abd7f3a010e0d354dc804182372779a722d474c4d8a3db8f4a3f5baef2a591e", size = 620425510 },
-    { url = "https://files.pythonhosted.org/packages/f1/b7/a3d455db88ab5b35ce53ab885ec0dd9f28d905a86a2250423048bc8cafa0/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e9568c8efcb05c0266be223e3269c62ebf7ad3498f156438311735f6fa5ced5", size = 259465882 },
-    { url = "https://files.pythonhosted.org/packages/ff/0c/7df285ee8a88139fab0b237003634d90690759fae9c18f55ddb7c04656ec/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:481499fd0f824583de8945be61d5e827898cdaa4f5ea1bc2cc28ca2ccff8229e", size = 620570129 },
-    { url = "https://files.pythonhosted.org/packages/ec/b4/f028a5de27d0fda10ba6145bc76e40c37ff6d2d1e95b601adb5ae17d635e/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bfbfb3dd0e22bffc45fe1e922390d27753e99261fab8a882e802cf98a0e078f", size = 259533109 },
-    { url = "https://files.pythonhosted.org/packages/9c/d1/6aa15085d672056d5f08b5f28b1c7ce01c4e12149a23b0c98e3c79d04441/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25265b0bc527e0d54b1e9cc60c44a24f44a809fe27666b905f0466471f9c52ec", size = 620682547 },
-    { url = "https://files.pythonhosted.org/packages/ea/4c/c1aa90c5cc92e9f7f9c78421e121ef25bae7d378f8d1d4cbad46c6308836/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47c88e05a07f1ead4977b4894b3ecd4d8075c40191065afc4fd9355c9db3d926", size = 259663776 },
-    { url = "https://files.pythonhosted.org/packages/43/fb/8be8547c128613d82a2b006004026d86ed0bd672e913029a98153af4ffab/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fa3729b0126f75a99882b89fb7d536515721eda8014a63e259e780ba0a37372", size = 620815537 },
+    { url = "https://files.pythonhosted.org/packages/db/de/3a71ad41b87f9dd424e3aec3b0794a60f169fa7e9a9a1e3dd44290b86dd6/tensorflow_estimator-2.7.0-py2.py3-none-any.whl", hash = "sha256:325b5a224864379242b7b76c6987ca544239be82579d33e68ec7c2bda57abc9d", size = 463110, upload-time = "2021-10-29T23:02:47.14Z" },
+]
+
+[[package]]
+name = "tensorflow-io-gcs-filesystem"
+version = "0.37.1"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e2/19/9095c69e22c879cb3896321e676c69273a549a3148c4f62aa4bc5ebdb20f/tensorflow_io_gcs_filesystem-0.37.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8febbfcc67c61e542a5ac1a98c7c20a91a5e1afc2e14b1ef0cb7c28bc3b6aa70", size = 4842078, upload-time = "2024-07-01T23:44:18.977Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/48/47b7d25572961a48b1de3729b7a11e835b888e41e0203cca82df95d23b91/tensorflow_io_gcs_filesystem-0.37.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9679b36e3a80921876f31685ab6f7270f3411a4cc51bc2847e80d0e4b5291e27", size = 5085736, upload-time = "2024-07-01T23:44:21.034Z" },
+    { url = "https://files.pythonhosted.org/packages/de/bf/ba597d3884c77d05a78050f3c178933d69e3f80200a261df6eaa920656cd/tensorflow_io_gcs_filesystem-0.37.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e1f2796b57e799a8ca1b75bf47c2aaa437c968408cc1a402a9862929e104cda", size = 4842079, upload-time = "2024-07-01T23:44:26.825Z" },
+    { url = "https://files.pythonhosted.org/packages/66/7f/e36ae148c2f03d61ca1bff24bc13a0fef6d6825c966abef73fc6f880a23b/tensorflow_io_gcs_filesystem-0.37.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee7c8ee5fe2fd8cb6392669ef16e71841133041fee8a330eff519ad9b36e4556", size = 5085736, upload-time = "2024-07-01T23:44:28.618Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/46/962f47af08bd39fc9feb280d3192825431a91a078c856d17a78ae4884eb1/tensorflow_io_gcs_filesystem-0.37.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fbb33f1745f218464a59cecd9a18e32ca927b0f4d77abd8f8671b645cc1a182f", size = 4842077, upload-time = "2024-07-01T23:44:33.86Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/9b/790d290c232bce9b691391cf16e95a96e469669c56abfb1d9d0f35fa437c/tensorflow_io_gcs_filesystem-0.37.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:286389a203a5aee1a4fa2e53718c661091aa5fea797ff4fa6715ab8436b02e6c", size = 5085733, upload-time = "2024-07-01T23:44:36.663Z" },
+    { url = "https://files.pythonhosted.org/packages/66/5f/334a011caa1eb97689274d1141df8e6b7a25e389f0390bdcd90235de9783/tensorflow_io_gcs_filesystem-0.37.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:426de1173cb81fbd62becec2012fc00322a295326d90eb6c737fab636f182aed", size = 4842075, upload-time = "2024-07-01T23:44:42.094Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/cb/7dcee55fc5a7d7d8a862e12519322851cd5fe5b086f946fd71e4ae1ef281/tensorflow_io_gcs_filesystem-0.37.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0df00891669390078a003cedbdd3b8e645c718b111917535fa1d7725e95cdb95", size = 5087496, upload-time = "2024-07-01T23:44:43.797Z" },
 ]
 
 [[package]]
 name = "termcolor"
 version = "3.1.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/ca/6c/3d75c196ac07ac8749600b60b03f4f6094d54e132c4d94ebac6ee0e0add0/termcolor-3.1.0.tar.gz", hash = "sha256:6a6dd7fbee581909eeec6a756cff1d7f7c376063b14e4a298dc4980309e55970", size = 14324 }
+sdist = { url = "https://files.pythonhosted.org/packages/ca/6c/3d75c196ac07ac8749600b60b03f4f6094d54e132c4d94ebac6ee0e0add0/termcolor-3.1.0.tar.gz", hash = "sha256:6a6dd7fbee581909eeec6a756cff1d7f7c376063b14e4a298dc4980309e55970", size = 14324, upload-time = "2025-04-30T11:37:53.791Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/4f/bd/de8d508070629b6d84a30d01d57e4a65c69aa7f5abe7560b8fad3b50ea59/termcolor-3.1.0-py3-none-any.whl", hash = "sha256:591dd26b5c2ce03b9e43f391264626557873ce1d379019786f99b0c2bee140aa", size = 7684 },
+    { url = "https://files.pythonhosted.org/packages/4f/bd/de8d508070629b6d84a30d01d57e4a65c69aa7f5abe7560b8fad3b50ea59/termcolor-3.1.0-py3-none-any.whl", hash = "sha256:591dd26b5c2ce03b9e43f391264626557873ce1d379019786f99b0c2bee140aa", size = 7684, upload-time = "2025-04-30T11:37:52.382Z" },
 ]
 
 [[package]]
 name = "tomli"
 version = "2.2.1"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077 },
-    { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429 },
-    { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067 },
-    { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030 },
-    { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898 },
-    { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894 },
-    { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319 },
-    { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273 },
-    { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310 },
-    { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309 },
-    { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762 },
-    { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453 },
-    { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486 },
-    { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349 },
-    { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159 },
-    { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243 },
-    { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645 },
-    { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584 },
-    { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875 },
-    { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418 },
-    { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708 },
-    { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582 },
-    { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543 },
-    { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691 },
-    { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170 },
-    { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530 },
-    { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666 },
-    { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954 },
-    { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724 },
-    { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383 },
-    { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257 },
+sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" },
+    { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" },
+    { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" },
+    { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" },
+    { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" },
+    { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" },
+    { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" },
+    { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" },
+    { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" },
+    { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" },
+    { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" },
+    { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" },
 ]
 
 [[package]]
@@ -2679,7 +3229,8 @@ dependencies = [
     { name = "filelock" },
     { name = "fsspec" },
     { name = "jinja2" },
-    { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
+    { name = "networkx", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
     { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
@@ -2701,26 +3252,30 @@ dependencies = [
     { name = "typing-extensions" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/63/28/110f7274254f1b8476c561dada127173f994afa2b1ffc044efb773c15650/torch-2.8.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0be92c08b44009d4131d1ff7a8060d10bafdb7ddcb7359ef8d8c5169007ea905", size = 102052793 },
-    { url = "https://files.pythonhosted.org/packages/70/1c/58da560016f81c339ae14ab16c98153d51c941544ae568da3cb5b1ceb572/torch-2.8.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:89aa9ee820bb39d4d72b794345cccef106b574508dd17dbec457949678c76011", size = 888025420 },
-    { url = "https://files.pythonhosted.org/packages/70/87/f69752d0dd4ba8218c390f0438130c166fa264a33b7025adb5014b92192c/torch-2.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:e8e5bf982e87e2b59d932769938b698858c64cc53753894be25629bdf5cf2f46", size = 241363614 },
-    { url = "https://files.pythonhosted.org/packages/ef/d6/e6d4c57e61c2b2175d3aafbfb779926a2cfd7c32eeda7c543925dceec923/torch-2.8.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:a3f16a58a9a800f589b26d47ee15aca3acf065546137fc2af039876135f4c760", size = 73611154 },
-    { url = "https://files.pythonhosted.org/packages/8f/c4/3e7a3887eba14e815e614db70b3b529112d1513d9dae6f4d43e373360b7f/torch-2.8.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:220a06fd7af8b653c35d359dfe1aaf32f65aa85befa342629f716acb134b9710", size = 102073391 },
-    { url = "https://files.pythonhosted.org/packages/5a/63/4fdc45a0304536e75a5e1b1bbfb1b56dd0e2743c48ee83ca729f7ce44162/torch-2.8.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:c12fa219f51a933d5f80eeb3a7a5d0cbe9168c0a14bbb4055f1979431660879b", size = 888063640 },
-    { url = "https://files.pythonhosted.org/packages/84/57/2f64161769610cf6b1c5ed782bd8a780e18a3c9d48931319f2887fa9d0b1/torch-2.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:8c7ef765e27551b2fbfc0f41bcf270e1292d9bf79f8e0724848b1682be6e80aa", size = 241366752 },
-    { url = "https://files.pythonhosted.org/packages/a4/5e/05a5c46085d9b97e928f3f037081d3d2b87fb4b4195030fc099aaec5effc/torch-2.8.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:5ae0524688fb6707c57a530c2325e13bb0090b745ba7b4a2cd6a3ce262572916", size = 73621174 },
-    { url = "https://files.pythonhosted.org/packages/49/0c/2fd4df0d83a495bb5e54dca4474c4ec5f9c62db185421563deeb5dabf609/torch-2.8.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e2fab4153768d433f8ed9279c8133a114a034a61e77a3a104dcdf54388838705", size = 101906089 },
-    { url = "https://files.pythonhosted.org/packages/99/a8/6acf48d48838fb8fe480597d98a0668c2beb02ee4755cc136de92a0a956f/torch-2.8.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b2aca0939fb7e4d842561febbd4ffda67a8e958ff725c1c27e244e85e982173c", size = 887913624 },
-    { url = "https://files.pythonhosted.org/packages/af/8a/5c87f08e3abd825c7dfecef5a0f1d9aa5df5dd0e3fd1fa2f490a8e512402/torch-2.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:2f4ac52f0130275d7517b03a33d2493bab3693c83dcfadf4f81688ea82147d2e", size = 241326087 },
-    { url = "https://files.pythonhosted.org/packages/be/66/5c9a321b325aaecb92d4d1855421e3a055abd77903b7dab6575ca07796db/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:619c2869db3ada2c0105487ba21b5008defcc472d23f8b80ed91ac4a380283b0", size = 73630478 },
-    { url = "https://files.pythonhosted.org/packages/10/4e/469ced5a0603245d6a19a556e9053300033f9c5baccf43a3d25ba73e189e/torch-2.8.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:2b2f96814e0345f5a5aed9bf9734efa913678ed19caf6dc2cddb7930672d6128", size = 101936856 },
-    { url = "https://files.pythonhosted.org/packages/16/82/3948e54c01b2109238357c6f86242e6ecbf0c63a1af46906772902f82057/torch-2.8.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:65616ca8ec6f43245e1f5f296603e33923f4c30f93d65e103d9e50c25b35150b", size = 887922844 },
-    { url = "https://files.pythonhosted.org/packages/e3/54/941ea0a860f2717d86a811adf0c2cd01b3983bdd460d0803053c4e0b8649/torch-2.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:659df54119ae03e83a800addc125856effda88b016dfc54d9f65215c3975be16", size = 241330968 },
-    { url = "https://files.pythonhosted.org/packages/de/69/8b7b13bba430f5e21d77708b616f767683629fc4f8037564a177d20f90ed/torch-2.8.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:1a62a1ec4b0498930e2543535cf70b1bef8c777713de7ceb84cd79115f553767", size = 73915128 },
-    { url = "https://files.pythonhosted.org/packages/15/0e/8a800e093b7f7430dbaefa80075aee9158ec22e4c4fc3c1a66e4fb96cb4f/torch-2.8.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:83c13411a26fac3d101fe8035a6b0476ae606deb8688e904e796a3534c197def", size = 102020139 },
-    { url = "https://files.pythonhosted.org/packages/4a/15/5e488ca0bc6162c86a33b58642bc577c84ded17c7b72d97e49b5833e2d73/torch-2.8.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:8f0a9d617a66509ded240add3754e462430a6c1fc5589f86c17b433dd808f97a", size = 887990692 },
-    { url = "https://files.pythonhosted.org/packages/b4/a8/6a04e4b54472fc5dba7ca2341ab219e529f3c07b6941059fbf18dccac31f/torch-2.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:a7242b86f42be98ac674b88a4988643b9bc6145437ec8f048fea23f72feb5eca", size = 241603453 },
-    { url = "https://files.pythonhosted.org/packages/04/6e/650bb7f28f771af0cb791b02348db8b7f5f64f40f6829ee82aa6ce99aabe/torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:7b677e17f5a3e69fdef7eb3b9da72622f8d322692930297e4ccb52fefc6c8211", size = 73632395 },
+    { url = "https://files.pythonhosted.org/packages/63/28/110f7274254f1b8476c561dada127173f994afa2b1ffc044efb773c15650/torch-2.8.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0be92c08b44009d4131d1ff7a8060d10bafdb7ddcb7359ef8d8c5169007ea905", size = 102052793, upload-time = "2025-08-06T14:53:15.852Z" },
+    { url = "https://files.pythonhosted.org/packages/70/1c/58da560016f81c339ae14ab16c98153d51c941544ae568da3cb5b1ceb572/torch-2.8.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:89aa9ee820bb39d4d72b794345cccef106b574508dd17dbec457949678c76011", size = 888025420, upload-time = "2025-08-06T14:54:18.014Z" },
+    { url = "https://files.pythonhosted.org/packages/70/87/f69752d0dd4ba8218c390f0438130c166fa264a33b7025adb5014b92192c/torch-2.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:e8e5bf982e87e2b59d932769938b698858c64cc53753894be25629bdf5cf2f46", size = 241363614, upload-time = "2025-08-06T14:53:31.496Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/d6/e6d4c57e61c2b2175d3aafbfb779926a2cfd7c32eeda7c543925dceec923/torch-2.8.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:a3f16a58a9a800f589b26d47ee15aca3acf065546137fc2af039876135f4c760", size = 73611154, upload-time = "2025-08-06T14:53:10.919Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/c4/3e7a3887eba14e815e614db70b3b529112d1513d9dae6f4d43e373360b7f/torch-2.8.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:220a06fd7af8b653c35d359dfe1aaf32f65aa85befa342629f716acb134b9710", size = 102073391, upload-time = "2025-08-06T14:53:20.937Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/63/4fdc45a0304536e75a5e1b1bbfb1b56dd0e2743c48ee83ca729f7ce44162/torch-2.8.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:c12fa219f51a933d5f80eeb3a7a5d0cbe9168c0a14bbb4055f1979431660879b", size = 888063640, upload-time = "2025-08-06T14:55:05.325Z" },
+    { url = "https://files.pythonhosted.org/packages/84/57/2f64161769610cf6b1c5ed782bd8a780e18a3c9d48931319f2887fa9d0b1/torch-2.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:8c7ef765e27551b2fbfc0f41bcf270e1292d9bf79f8e0724848b1682be6e80aa", size = 241366752, upload-time = "2025-08-06T14:53:38.692Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/5e/05a5c46085d9b97e928f3f037081d3d2b87fb4b4195030fc099aaec5effc/torch-2.8.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:5ae0524688fb6707c57a530c2325e13bb0090b745ba7b4a2cd6a3ce262572916", size = 73621174, upload-time = "2025-08-06T14:53:25.44Z" },
+    { url = "https://files.pythonhosted.org/packages/49/0c/2fd4df0d83a495bb5e54dca4474c4ec5f9c62db185421563deeb5dabf609/torch-2.8.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e2fab4153768d433f8ed9279c8133a114a034a61e77a3a104dcdf54388838705", size = 101906089, upload-time = "2025-08-06T14:53:52.631Z" },
+    { url = "https://files.pythonhosted.org/packages/99/a8/6acf48d48838fb8fe480597d98a0668c2beb02ee4755cc136de92a0a956f/torch-2.8.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b2aca0939fb7e4d842561febbd4ffda67a8e958ff725c1c27e244e85e982173c", size = 887913624, upload-time = "2025-08-06T14:56:44.33Z" },
+    { url = "https://files.pythonhosted.org/packages/af/8a/5c87f08e3abd825c7dfecef5a0f1d9aa5df5dd0e3fd1fa2f490a8e512402/torch-2.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:2f4ac52f0130275d7517b03a33d2493bab3693c83dcfadf4f81688ea82147d2e", size = 241326087, upload-time = "2025-08-06T14:53:46.503Z" },
+    { url = "https://files.pythonhosted.org/packages/be/66/5c9a321b325aaecb92d4d1855421e3a055abd77903b7dab6575ca07796db/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:619c2869db3ada2c0105487ba21b5008defcc472d23f8b80ed91ac4a380283b0", size = 73630478, upload-time = "2025-08-06T14:53:57.144Z" },
+    { url = "https://files.pythonhosted.org/packages/10/4e/469ced5a0603245d6a19a556e9053300033f9c5baccf43a3d25ba73e189e/torch-2.8.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:2b2f96814e0345f5a5aed9bf9734efa913678ed19caf6dc2cddb7930672d6128", size = 101936856, upload-time = "2025-08-06T14:54:01.526Z" },
+    { url = "https://files.pythonhosted.org/packages/16/82/3948e54c01b2109238357c6f86242e6ecbf0c63a1af46906772902f82057/torch-2.8.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:65616ca8ec6f43245e1f5f296603e33923f4c30f93d65e103d9e50c25b35150b", size = 887922844, upload-time = "2025-08-06T14:55:50.78Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/54/941ea0a860f2717d86a811adf0c2cd01b3983bdd460d0803053c4e0b8649/torch-2.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:659df54119ae03e83a800addc125856effda88b016dfc54d9f65215c3975be16", size = 241330968, upload-time = "2025-08-06T14:54:45.293Z" },
+    { url = "https://files.pythonhosted.org/packages/de/69/8b7b13bba430f5e21d77708b616f767683629fc4f8037564a177d20f90ed/torch-2.8.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:1a62a1ec4b0498930e2543535cf70b1bef8c777713de7ceb84cd79115f553767", size = 73915128, upload-time = "2025-08-06T14:54:34.769Z" },
+    { url = "https://files.pythonhosted.org/packages/15/0e/8a800e093b7f7430dbaefa80075aee9158ec22e4c4fc3c1a66e4fb96cb4f/torch-2.8.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:83c13411a26fac3d101fe8035a6b0476ae606deb8688e904e796a3534c197def", size = 102020139, upload-time = "2025-08-06T14:54:39.047Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/15/5e488ca0bc6162c86a33b58642bc577c84ded17c7b72d97e49b5833e2d73/torch-2.8.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:8f0a9d617a66509ded240add3754e462430a6c1fc5589f86c17b433dd808f97a", size = 887990692, upload-time = "2025-08-06T14:56:18.286Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/a8/6a04e4b54472fc5dba7ca2341ab219e529f3c07b6941059fbf18dccac31f/torch-2.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:a7242b86f42be98ac674b88a4988643b9bc6145437ec8f048fea23f72feb5eca", size = 241603453, upload-time = "2025-08-06T14:55:22.945Z" },
+    { url = "https://files.pythonhosted.org/packages/04/6e/650bb7f28f771af0cb791b02348db8b7f5f64f40f6829ee82aa6ce99aabe/torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:7b677e17f5a3e69fdef7eb3b9da72622f8d322692930297e4ccb52fefc6c8211", size = 73632395, upload-time = "2025-08-06T14:55:28.645Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/b0/a321f27270049baa12f5c3fb0d6ceea005634787e3af9a8d75dce8306b0a/torch-2.8.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:da6afa31c13b669d4ba49d8a2169f0db2c3ec6bec4af898aa714f401d4c38904", size = 102059214, upload-time = "2025-08-06T14:55:33.433Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/dd/1630cb51b10d3d2e97db95e5a84c32def81fc26b005bce6fc880b0e6db81/torch-2.8.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:06fcee8000e5c62a9f3e52a688b9c5abb7c6228d0e56e3452983416025c41381", size = 888024302, upload-time = "2025-08-06T14:57:28.23Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/dc/1f1f621afe15e3c496e1e8f94f8903f75f87e7d642d5a985e92210cc208d/torch-2.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:5128fe752a355d9308e56af1ad28b15266fe2da5948660fad44de9e3a9e36e8c", size = 241249338, upload-time = "2025-08-06T14:57:05.669Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/95/ae26263aceb3d57b821179f827d0e321373ed49423e603dd5906ab14a730/torch-2.8.0-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:e9f071f5b52a9f6970dc8a919694b27a91ae9dc08898b2b988abbef5eddfd1ae", size = 73610795, upload-time = "2025-08-06T14:57:11.513Z" },
 ]
 
 [[package]]
@@ -2730,9 +3285,9 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "colorama", marker = "sys_platform == 'win32'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737 }
+sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540 },
+    { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" },
 ]
 
 [[package]]
@@ -2740,23 +3295,25 @@ name = "triton"
 version = "3.4.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
+    { name = "importlib-metadata", marker = "python_full_version < '3.10'" },
     { name = "setuptools" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/62/ee/0ee5f64a87eeda19bbad9bc54ae5ca5b98186ed00055281fd40fb4beb10e/triton-3.4.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7ff2785de9bc02f500e085420273bb5cc9c9bb767584a4aa28d6e360cec70128", size = 155430069 },
-    { url = "https://files.pythonhosted.org/packages/7d/39/43325b3b651d50187e591eefa22e236b2981afcebaefd4f2fc0ea99df191/triton-3.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b70f5e6a41e52e48cfc087436c8a28c17ff98db369447bcaff3b887a3ab4467", size = 155531138 },
-    { url = "https://files.pythonhosted.org/packages/d0/66/b1eb52839f563623d185f0927eb3530ee4d5ffe9d377cdaf5346b306689e/triton-3.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31c1d84a5c0ec2c0f8e8a072d7fd150cab84a9c239eaddc6706c081bfae4eb04", size = 155560068 },
-    { url = "https://files.pythonhosted.org/packages/30/7b/0a685684ed5322d2af0bddefed7906674f67974aa88b0fae6e82e3b766f6/triton-3.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00be2964616f4c619193cb0d1b29a99bd4b001d7dc333816073f92cf2a8ccdeb", size = 155569223 },
-    { url = "https://files.pythonhosted.org/packages/20/63/8cb444ad5cdb25d999b7d647abac25af0ee37d292afc009940c05b82dda0/triton-3.4.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7936b18a3499ed62059414d7df563e6c163c5e16c3773678a3ee3d417865035d", size = 155659780 },
+    { url = "https://files.pythonhosted.org/packages/62/ee/0ee5f64a87eeda19bbad9bc54ae5ca5b98186ed00055281fd40fb4beb10e/triton-3.4.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7ff2785de9bc02f500e085420273bb5cc9c9bb767584a4aa28d6e360cec70128", size = 155430069, upload-time = "2025-07-30T19:58:21.715Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/39/43325b3b651d50187e591eefa22e236b2981afcebaefd4f2fc0ea99df191/triton-3.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b70f5e6a41e52e48cfc087436c8a28c17ff98db369447bcaff3b887a3ab4467", size = 155531138, upload-time = "2025-07-30T19:58:29.908Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/66/b1eb52839f563623d185f0927eb3530ee4d5ffe9d377cdaf5346b306689e/triton-3.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31c1d84a5c0ec2c0f8e8a072d7fd150cab84a9c239eaddc6706c081bfae4eb04", size = 155560068, upload-time = "2025-07-30T19:58:37.081Z" },
+    { url = "https://files.pythonhosted.org/packages/30/7b/0a685684ed5322d2af0bddefed7906674f67974aa88b0fae6e82e3b766f6/triton-3.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00be2964616f4c619193cb0d1b29a99bd4b001d7dc333816073f92cf2a8ccdeb", size = 155569223, upload-time = "2025-07-30T19:58:44.017Z" },
+    { url = "https://files.pythonhosted.org/packages/20/63/8cb444ad5cdb25d999b7d647abac25af0ee37d292afc009940c05b82dda0/triton-3.4.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7936b18a3499ed62059414d7df563e6c163c5e16c3773678a3ee3d417865035d", size = 155659780, upload-time = "2025-07-30T19:58:51.171Z" },
+    { url = "https://files.pythonhosted.org/packages/12/34/1251beb5a3cb93f3950ebe68732752014646003ef6eb11eb5f1a37ca78cd/triton-3.4.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98e5c1442eaeabae2e2452ae765801bd53cd4ce873cab0d1bdd59a32ab2d9397", size = 155430799, upload-time = "2025-07-30T19:58:57.664Z" },
 ]
 
 [[package]]
 name = "typing-extensions"
 version = "4.15.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391 }
+sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614 },
+    { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" },
 ]
 
 [[package]]
@@ -2766,27 +3323,46 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949 }
+sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611 },
+    { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" },
 ]
 
 [[package]]
 name = "tzdata"
 version = "2025.2"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380 }
+sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" },
+]
+
+[[package]]
+name = "urllib3"
+version = "1.26.20"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e4/e8/6ff5e6bc22095cfc59b6ea711b687e2b7ed4bdb373f7eeec370a97d7392f/urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32", size = 307380, upload-time = "2024-08-29T15:43:11.37Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839 },
+    { url = "https://files.pythonhosted.org/packages/33/cf/8435d5a7159e2a9c83a95896ed596f68cf798005fe107cc655b5c5c14704/urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e", size = 144225, upload-time = "2024-08-29T15:43:08.921Z" },
 ]
 
 [[package]]
 name = "urllib3"
 version = "2.5.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185 }
+resolution-markers = [
+    "python_full_version >= '3.14'",
+    "python_full_version == '3.13.*'",
+    "python_full_version == '3.12.*'",
+    "python_full_version == '3.11.*'",
+    "python_full_version == '3.10.*'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795 },
+    { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" },
 ]
 
 [[package]]
@@ -2796,169 +3372,188 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "markupsafe" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925 }
+sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925, upload-time = "2024-11-08T15:52:18.093Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498 },
+    { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" },
 ]
 
 [[package]]
 name = "wheel"
 version = "0.45.1"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/8a/98/2d9906746cdc6a6ef809ae6338005b3f21bb568bea3165cfc6a243fdc25c/wheel-0.45.1.tar.gz", hash = "sha256:661e1abd9198507b1409a20c02106d9670b2576e916d58f520316666abca6729", size = 107545 }
+sdist = { url = "https://files.pythonhosted.org/packages/8a/98/2d9906746cdc6a6ef809ae6338005b3f21bb568bea3165cfc6a243fdc25c/wheel-0.45.1.tar.gz", hash = "sha256:661e1abd9198507b1409a20c02106d9670b2576e916d58f520316666abca6729", size = 107545, upload-time = "2024-11-23T00:18:23.513Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/0b/2c/87f3254fd8ffd29e4c02732eee68a83a1d3c346ae39bc6822dcbcb697f2b/wheel-0.45.1-py3-none-any.whl", hash = "sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248", size = 72494 },
+    { url = "https://files.pythonhosted.org/packages/0b/2c/87f3254fd8ffd29e4c02732eee68a83a1d3c346ae39bc6822dcbcb697f2b/wheel-0.45.1-py3-none-any.whl", hash = "sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248", size = 72494, upload-time = "2024-11-23T00:18:21.207Z" },
 ]
 
 [[package]]
 name = "wrapt"
 version = "1.17.3"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/de/17/9f8f86755c191d6779d7ddead1a53c7a8aa18bccb7cea8e7e72dfa6a8a09/wrapt-1.17.3-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f9b2601381be482f70e5d1051a5965c25fb3625455a2bf520b5a077b22afb775", size = 81975 },
-    { url = "https://files.pythonhosted.org/packages/f2/15/dd576273491f9f43dd09fce517f6c2ce6eb4fe21681726068db0d0467096/wrapt-1.17.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:343e44b2a8e60e06a7e0d29c1671a0d9951f59174f3709962b5143f60a2a98bd", size = 83149 },
-    { url = "https://files.pythonhosted.org/packages/0c/c4/5eb4ce0d4814521fee7aa806264bf7a114e748ad05110441cd5b8a5c744b/wrapt-1.17.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:33486899acd2d7d3066156b03465b949da3fd41a5da6e394ec49d271baefcf05", size = 82209 },
-    { url = "https://files.pythonhosted.org/packages/31/4b/819e9e0eb5c8dc86f60dfc42aa4e2c0d6c3db8732bce93cc752e604bb5f5/wrapt-1.17.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e6f40a8aa5a92f150bdb3e1c44b7e98fb7113955b2e5394122fa5532fec4b418", size = 81551 },
-    { url = "https://files.pythonhosted.org/packages/5d/8f/a32a99fc03e4b37e31b57cb9cefc65050ea08147a8ce12f288616b05ef54/wrapt-1.17.3-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b32888aad8b6e68f83a8fdccbf3165f5469702a7544472bdf41f582970ed3311", size = 82376 },
-    { url = "https://files.pythonhosted.org/packages/31/57/4930cb8d9d70d59c27ee1332a318c20291749b4fba31f113c2f8ac49a72e/wrapt-1.17.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cccf4f81371f257440c88faed6b74f1053eef90807b77e31ca057b2db74edb1", size = 83604 },
-    { url = "https://files.pythonhosted.org/packages/a8/f3/1afd48de81d63dd66e01b263a6fbb86e1b5053b419b9b33d13e1f6d0f7d0/wrapt-1.17.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8a210b158a34164de8bb68b0e7780041a903d7b00c87e906fb69928bf7890d5", size = 82782 },
-    { url = "https://files.pythonhosted.org/packages/1e/d7/4ad5327612173b144998232f98a85bb24b60c352afb73bc48e3e0d2bdc4e/wrapt-1.17.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:79573c24a46ce11aab457b472efd8d125e5a51da2d1d24387666cd85f54c05b2", size = 82076 },
-    { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036 },
-    { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156 },
-    { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102 },
-    { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732 },
-    { url = "https://files.pythonhosted.org/packages/0c/37/6faf15cfa41bf1f3dba80cd3f5ccc6622dfccb660ab26ed79f0178c7497f/wrapt-1.17.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277", size = 88072 },
-    { url = "https://files.pythonhosted.org/packages/78/f2/efe19ada4a38e4e15b6dff39c3e3f3f73f5decf901f66e6f72fe79623a06/wrapt-1.17.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d", size = 88214 },
-    { url = "https://files.pythonhosted.org/packages/40/90/ca86701e9de1622b16e09689fc24b76f69b06bb0150990f6f4e8b0eeb576/wrapt-1.17.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa", size = 87105 },
-    { url = "https://files.pythonhosted.org/packages/fd/e0/d10bd257c9a3e15cbf5523025252cc14d77468e8ed644aafb2d6f54cb95d/wrapt-1.17.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050", size = 87766 },
-    { url = "https://files.pythonhosted.org/packages/c3/f7/c983d2762bcce2326c317c26a6a1e7016f7eb039c27cdf5c4e30f4160f31/wrapt-1.17.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b", size = 87163 },
-    { url = "https://files.pythonhosted.org/packages/e4/0f/f673f75d489c7f22d17fe0193e84b41540d962f75fce579cf6873167c29b/wrapt-1.17.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa", size = 87963 },
-    { url = "https://files.pythonhosted.org/packages/df/61/515ad6caca68995da2fac7a6af97faab8f78ebe3bf4f761e1b77efbc47b5/wrapt-1.17.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7", size = 86945 },
-    { url = "https://files.pythonhosted.org/packages/d3/bd/4e70162ce398462a467bc09e768bee112f1412e563620adc353de9055d33/wrapt-1.17.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4", size = 86857 },
-    { url = "https://files.pythonhosted.org/packages/64/0e/f4472f2fdde2d4617975144311f8800ef73677a159be7fe61fa50997d6c0/wrapt-1.17.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e", size = 108571 },
-    { url = "https://files.pythonhosted.org/packages/cc/01/9b85a99996b0a97c8a17484684f206cbb6ba73c1ce6890ac668bcf3838fb/wrapt-1.17.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f", size = 113094 },
-    { url = "https://files.pythonhosted.org/packages/25/02/78926c1efddcc7b3aa0bc3d6b33a822f7d898059f7cd9ace8c8318e559ef/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056", size = 110659 },
-    { url = "https://files.pythonhosted.org/packages/dc/ee/c414501ad518ac3e6fe184753632fe5e5ecacdcf0effc23f31c1e4f7bfcf/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804", size = 106946 },
-    { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591 },
+sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/de/17/9f8f86755c191d6779d7ddead1a53c7a8aa18bccb7cea8e7e72dfa6a8a09/wrapt-1.17.3-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f9b2601381be482f70e5d1051a5965c25fb3625455a2bf520b5a077b22afb775", size = 81975, upload-time = "2025-08-12T05:52:30.109Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/15/dd576273491f9f43dd09fce517f6c2ce6eb4fe21681726068db0d0467096/wrapt-1.17.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:343e44b2a8e60e06a7e0d29c1671a0d9951f59174f3709962b5143f60a2a98bd", size = 83149, upload-time = "2025-08-12T05:52:09.316Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/c4/5eb4ce0d4814521fee7aa806264bf7a114e748ad05110441cd5b8a5c744b/wrapt-1.17.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:33486899acd2d7d3066156b03465b949da3fd41a5da6e394ec49d271baefcf05", size = 82209, upload-time = "2025-08-12T05:52:10.331Z" },
+    { url = "https://files.pythonhosted.org/packages/31/4b/819e9e0eb5c8dc86f60dfc42aa4e2c0d6c3db8732bce93cc752e604bb5f5/wrapt-1.17.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e6f40a8aa5a92f150bdb3e1c44b7e98fb7113955b2e5394122fa5532fec4b418", size = 81551, upload-time = "2025-08-12T05:52:31.137Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/8f/a32a99fc03e4b37e31b57cb9cefc65050ea08147a8ce12f288616b05ef54/wrapt-1.17.3-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b32888aad8b6e68f83a8fdccbf3165f5469702a7544472bdf41f582970ed3311", size = 82376, upload-time = "2025-08-12T05:52:32.134Z" },
+    { url = "https://files.pythonhosted.org/packages/31/57/4930cb8d9d70d59c27ee1332a318c20291749b4fba31f113c2f8ac49a72e/wrapt-1.17.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cccf4f81371f257440c88faed6b74f1053eef90807b77e31ca057b2db74edb1", size = 83604, upload-time = "2025-08-12T05:52:11.663Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/f3/1afd48de81d63dd66e01b263a6fbb86e1b5053b419b9b33d13e1f6d0f7d0/wrapt-1.17.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8a210b158a34164de8bb68b0e7780041a903d7b00c87e906fb69928bf7890d5", size = 82782, upload-time = "2025-08-12T05:52:12.626Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/d7/4ad5327612173b144998232f98a85bb24b60c352afb73bc48e3e0d2bdc4e/wrapt-1.17.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:79573c24a46ce11aab457b472efd8d125e5a51da2d1d24387666cd85f54c05b2", size = 82076, upload-time = "2025-08-12T05:52:33.168Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/37/6faf15cfa41bf1f3dba80cd3f5ccc6622dfccb660ab26ed79f0178c7497f/wrapt-1.17.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277", size = 88072, upload-time = "2025-08-12T05:52:37.53Z" },
+    { url = "https://files.pythonhosted.org/packages/78/f2/efe19ada4a38e4e15b6dff39c3e3f3f73f5decf901f66e6f72fe79623a06/wrapt-1.17.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d", size = 88214, upload-time = "2025-08-12T05:52:15.886Z" },
+    { url = "https://files.pythonhosted.org/packages/40/90/ca86701e9de1622b16e09689fc24b76f69b06bb0150990f6f4e8b0eeb576/wrapt-1.17.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa", size = 87105, upload-time = "2025-08-12T05:52:17.914Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/e0/d10bd257c9a3e15cbf5523025252cc14d77468e8ed644aafb2d6f54cb95d/wrapt-1.17.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050", size = 87766, upload-time = "2025-08-12T05:52:39.243Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/f7/c983d2762bcce2326c317c26a6a1e7016f7eb039c27cdf5c4e30f4160f31/wrapt-1.17.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b", size = 87163, upload-time = "2025-08-12T05:52:40.965Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/0f/f673f75d489c7f22d17fe0193e84b41540d962f75fce579cf6873167c29b/wrapt-1.17.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa", size = 87963, upload-time = "2025-08-12T05:52:20.326Z" },
+    { url = "https://files.pythonhosted.org/packages/df/61/515ad6caca68995da2fac7a6af97faab8f78ebe3bf4f761e1b77efbc47b5/wrapt-1.17.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7", size = 86945, upload-time = "2025-08-12T05:52:21.581Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/bd/4e70162ce398462a467bc09e768bee112f1412e563620adc353de9055d33/wrapt-1.17.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4", size = 86857, upload-time = "2025-08-12T05:52:43.043Z" },
+    { url = "https://files.pythonhosted.org/packages/64/0e/f4472f2fdde2d4617975144311f8800ef73677a159be7fe61fa50997d6c0/wrapt-1.17.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e", size = 108571, upload-time = "2025-08-12T05:52:44.521Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/01/9b85a99996b0a97c8a17484684f206cbb6ba73c1ce6890ac668bcf3838fb/wrapt-1.17.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f", size = 113094, upload-time = "2025-08-12T05:52:22.618Z" },
+    { url = "https://files.pythonhosted.org/packages/25/02/78926c1efddcc7b3aa0bc3d6b33a822f7d898059f7cd9ace8c8318e559ef/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056", size = 110659, upload-time = "2025-08-12T05:52:24.057Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/ee/c414501ad518ac3e6fe184753632fe5e5ecacdcf0effc23f31c1e4f7bfcf/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804", size = 106946, upload-time = "2025-08-12T05:52:45.976Z" },
+    { url = "https://files.pythonhosted.org/packages/43/46/dd0791943613885f62619f18ee6107e6133237a6b6ed8a9ecfac339d0b4f/wrapt-1.17.3-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7e18f01b0c3e4a07fe6dfdb00e29049ba17eadbc5e7609a2a3a4af83ab7d710a", size = 81745, upload-time = "2025-08-12T05:52:49.62Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/ec/bb2d19bd1a614cc4f438abac13ae26c57186197920432d2a915183b15a8b/wrapt-1.17.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f5f51a6466667a5a356e6381d362d259125b57f059103dd9fdc8c0cf1d14139", size = 82833, upload-time = "2025-08-12T05:52:27.738Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/eb/66579aea6ad36f07617fedca8e282e49c7c9bab64c63b446cfe4f7f47a49/wrapt-1.17.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:59923aa12d0157f6b82d686c3fd8e1166fa8cdfb3e17b42ce3b6147ff81528df", size = 81889, upload-time = "2025-08-12T05:52:29.023Z" },
+    { url = "https://files.pythonhosted.org/packages/04/9c/a56b5ac0e2473bdc3fb11b22dd69ff423154d63861cf77911cdde5e38fd2/wrapt-1.17.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:46acc57b331e0b3bcb3e1ca3b421d65637915cfcd65eb783cb2f78a511193f9b", size = 81344, upload-time = "2025-08-12T05:52:50.869Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" },
 ]
 
 [[package]]
 name = "xxhash"
 version = "3.6.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/02/84/30869e01909fb37a6cc7e18688ee8bf1e42d57e7e0777636bd47524c43c7/xxhash-3.6.0.tar.gz", hash = "sha256:f0162a78b13a0d7617b2845b90c763339d1f1d82bb04a4b07f4ab535cc5e05d6", size = 85160 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/34/ee/f9f1d656ad168681bb0f6b092372c1e533c4416b8069b1896a175c46e484/xxhash-3.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:87ff03d7e35c61435976554477a7f4cd1704c3596a89a8300d5ce7fc83874a71", size = 32845 },
-    { url = "https://files.pythonhosted.org/packages/a3/b1/93508d9460b292c74a09b83d16750c52a0ead89c51eea9951cb97a60d959/xxhash-3.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f572dfd3d0e2eb1a57511831cf6341242f5a9f8298a45862d085f5b93394a27d", size = 30807 },
-    { url = "https://files.pythonhosted.org/packages/07/55/28c93a3662f2d200c70704efe74aab9640e824f8ce330d8d3943bf7c9b3c/xxhash-3.6.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:89952ea539566b9fed2bbd94e589672794b4286f342254fad28b149f9615fef8", size = 193786 },
-    { url = "https://files.pythonhosted.org/packages/c1/96/fec0be9bb4b8f5d9c57d76380a366f31a1781fb802f76fc7cda6c84893c7/xxhash-3.6.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48e6f2ffb07a50b52465a1032c3cf1f4a5683f944acaca8a134a2f23674c2058", size = 212830 },
-    { url = "https://files.pythonhosted.org/packages/c4/a0/c706845ba77b9611f81fd2e93fad9859346b026e8445e76f8c6fd057cc6d/xxhash-3.6.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b5b848ad6c16d308c3ac7ad4ba6bede80ed5df2ba8ed382f8932df63158dd4b2", size = 211606 },
-    { url = "https://files.pythonhosted.org/packages/67/1e/164126a2999e5045f04a69257eea946c0dc3e86541b400d4385d646b53d7/xxhash-3.6.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a034590a727b44dd8ac5914236a7b8504144447a9682586c3327e935f33ec8cc", size = 444872 },
-    { url = "https://files.pythonhosted.org/packages/2d/4b/55ab404c56cd70a2cf5ecfe484838865d0fea5627365c6c8ca156bd09c8f/xxhash-3.6.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a8f1972e75ebdd161d7896743122834fe87378160c20e97f8b09166213bf8cc", size = 193217 },
-    { url = "https://files.pythonhosted.org/packages/45/e6/52abf06bac316db33aa269091ae7311bd53cfc6f4b120ae77bac1b348091/xxhash-3.6.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ee34327b187f002a596d7b167ebc59a1b729e963ce645964bbc050d2f1b73d07", size = 210139 },
-    { url = "https://files.pythonhosted.org/packages/34/37/db94d490b8691236d356bc249c08819cbcef9273a1a30acf1254ff9ce157/xxhash-3.6.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:339f518c3c7a850dd033ab416ea25a692759dc7478a71131fe8869010d2b75e4", size = 197669 },
-    { url = "https://files.pythonhosted.org/packages/b7/36/c4f219ef4a17a4f7a64ed3569bc2b5a9c8311abdb22249ac96093625b1a4/xxhash-3.6.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:bf48889c9630542d4709192578aebbd836177c9f7a4a2778a7d6340107c65f06", size = 210018 },
-    { url = "https://files.pythonhosted.org/packages/fd/06/bfac889a374fc2fc439a69223d1750eed2e18a7db8514737ab630534fa08/xxhash-3.6.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5576b002a56207f640636056b4160a378fe36a58db73ae5c27a7ec8db35f71d4", size = 413058 },
-    { url = "https://files.pythonhosted.org/packages/c9/d1/555d8447e0dd32ad0930a249a522bb2e289f0d08b6b16204cfa42c1f5a0c/xxhash-3.6.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af1f3278bd02814d6dedc5dec397993b549d6f16c19379721e5a1d31e132c49b", size = 190628 },
-    { url = "https://files.pythonhosted.org/packages/d1/15/8751330b5186cedc4ed4b597989882ea05e0408b53fa47bcb46a6125bfc6/xxhash-3.6.0-cp310-cp310-win32.whl", hash = "sha256:aed058764db109dc9052720da65fafe84873b05eb8b07e5e653597951af57c3b", size = 30577 },
-    { url = "https://files.pythonhosted.org/packages/bb/cc/53f87e8b5871a6eb2ff7e89c48c66093bda2be52315a8161ddc54ea550c4/xxhash-3.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:e82da5670f2d0d98950317f82a0e4a0197150ff19a6df2ba40399c2a3b9ae5fb", size = 31487 },
-    { url = "https://files.pythonhosted.org/packages/9f/00/60f9ea3bb697667a14314d7269956f58bf56bb73864f8f8d52a3c2535e9a/xxhash-3.6.0-cp310-cp310-win_arm64.whl", hash = "sha256:4a082ffff8c6ac07707fb6b671caf7c6e020c75226c561830b73d862060f281d", size = 27863 },
-    { url = "https://files.pythonhosted.org/packages/17/d4/cc2f0400e9154df4b9964249da78ebd72f318e35ccc425e9f403c392f22a/xxhash-3.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b47bbd8cf2d72797f3c2772eaaac0ded3d3af26481a26d7d7d41dc2d3c46b04a", size = 32844 },
-    { url = "https://files.pythonhosted.org/packages/5e/ec/1cc11cd13e26ea8bc3cb4af4eaadd8d46d5014aebb67be3f71fb0b68802a/xxhash-3.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2b6821e94346f96db75abaa6e255706fb06ebd530899ed76d32cd99f20dc52fa", size = 30809 },
-    { url = "https://files.pythonhosted.org/packages/04/5f/19fe357ea348d98ca22f456f75a30ac0916b51c753e1f8b2e0e6fb884cce/xxhash-3.6.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d0a9751f71a1a65ce3584e9cae4467651c7e70c9d31017fa57574583a4540248", size = 194665 },
-    { url = "https://files.pythonhosted.org/packages/90/3b/d1f1a8f5442a5fd8beedae110c5af7604dc37349a8e16519c13c19a9a2de/xxhash-3.6.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b29ee68625ab37b04c0b40c3fafdf24d2f75ccd778333cfb698f65f6c463f62", size = 213550 },
-    { url = "https://files.pythonhosted.org/packages/c4/ef/3a9b05eb527457d5db13a135a2ae1a26c80fecd624d20f3e8dcc4cb170f3/xxhash-3.6.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6812c25fe0d6c36a46ccb002f40f27ac903bf18af9f6dd8f9669cb4d176ab18f", size = 212384 },
-    { url = "https://files.pythonhosted.org/packages/0f/18/ccc194ee698c6c623acbf0f8c2969811a8a4b6185af5e824cd27b9e4fd3e/xxhash-3.6.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4ccbff013972390b51a18ef1255ef5ac125c92dc9143b2d1909f59abc765540e", size = 445749 },
-    { url = "https://files.pythonhosted.org/packages/a5/86/cf2c0321dc3940a7aa73076f4fd677a0fb3e405cb297ead7d864fd90847e/xxhash-3.6.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:297b7fbf86c82c550e12e8fb71968b3f033d27b874276ba3624ea868c11165a8", size = 193880 },
-    { url = "https://files.pythonhosted.org/packages/82/fb/96213c8560e6f948a1ecc9a7613f8032b19ee45f747f4fca4eb31bb6d6ed/xxhash-3.6.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dea26ae1eb293db089798d3973a5fc928a18fdd97cc8801226fae705b02b14b0", size = 210912 },
-    { url = "https://files.pythonhosted.org/packages/40/aa/4395e669b0606a096d6788f40dbdf2b819d6773aa290c19e6e83cbfc312f/xxhash-3.6.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7a0b169aafb98f4284f73635a8e93f0735f9cbde17bd5ec332480484241aaa77", size = 198654 },
-    { url = "https://files.pythonhosted.org/packages/67/74/b044fcd6b3d89e9b1b665924d85d3f400636c23590226feb1eb09e1176ce/xxhash-3.6.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:08d45aef063a4531b785cd72de4887766d01dc8f362a515693df349fdb825e0c", size = 210867 },
-    { url = "https://files.pythonhosted.org/packages/bc/fd/3ce73bf753b08cb19daee1eb14aa0d7fe331f8da9c02dd95316ddfe5275e/xxhash-3.6.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:929142361a48ee07f09121fe9e96a84950e8d4df3bb298ca5d88061969f34d7b", size = 414012 },
-    { url = "https://files.pythonhosted.org/packages/ba/b3/5a4241309217c5c876f156b10778f3ab3af7ba7e3259e6d5f5c7d0129eb2/xxhash-3.6.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:51312c768403d8540487dbbfb557454cfc55589bbde6424456951f7fcd4facb3", size = 191409 },
-    { url = "https://files.pythonhosted.org/packages/c0/01/99bfbc15fb9abb9a72b088c1d95219fc4782b7d01fc835bd5744d66dd0b8/xxhash-3.6.0-cp311-cp311-win32.whl", hash = "sha256:d1927a69feddc24c987b337ce81ac15c4720955b667fe9b588e02254b80446fd", size = 30574 },
-    { url = "https://files.pythonhosted.org/packages/65/79/9d24d7f53819fe301b231044ea362ce64e86c74f6e8c8e51320de248b3e5/xxhash-3.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:26734cdc2d4ffe449b41d186bbeac416f704a482ed835d375a5c0cb02bc63fef", size = 31481 },
-    { url = "https://files.pythonhosted.org/packages/30/4e/15cd0e3e8772071344eab2961ce83f6e485111fed8beb491a3f1ce100270/xxhash-3.6.0-cp311-cp311-win_arm64.whl", hash = "sha256:d72f67ef8bf36e05f5b6c65e8524f265bd61071471cd4cf1d36743ebeeeb06b7", size = 27861 },
-    { url = "https://files.pythonhosted.org/packages/9a/07/d9412f3d7d462347e4511181dea65e47e0d0e16e26fbee2ea86a2aefb657/xxhash-3.6.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:01362c4331775398e7bb34e3ab403bc9ee9f7c497bc7dee6272114055277dd3c", size = 32744 },
-    { url = "https://files.pythonhosted.org/packages/79/35/0429ee11d035fc33abe32dca1b2b69e8c18d236547b9a9b72c1929189b9a/xxhash-3.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b7b2df81a23f8cb99656378e72501b2cb41b1827c0f5a86f87d6b06b69f9f204", size = 30816 },
-    { url = "https://files.pythonhosted.org/packages/b7/f2/57eb99aa0f7d98624c0932c5b9a170e1806406cdbcdb510546634a1359e0/xxhash-3.6.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:dc94790144e66b14f67b10ac8ed75b39ca47536bf8800eb7c24b50271ea0c490", size = 194035 },
-    { url = "https://files.pythonhosted.org/packages/4c/ed/6224ba353690d73af7a3f1c7cdb1fc1b002e38f783cb991ae338e1eb3d79/xxhash-3.6.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93f107c673bccf0d592cdba077dedaf52fe7f42dcd7676eba1f6d6f0c3efffd2", size = 212914 },
-    { url = "https://files.pythonhosted.org/packages/38/86/fb6b6130d8dd6b8942cc17ab4d90e223653a89aa32ad2776f8af7064ed13/xxhash-3.6.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2aa5ee3444c25b69813663c9f8067dcfaa2e126dc55e8dddf40f4d1c25d7effa", size = 212163 },
-    { url = "https://files.pythonhosted.org/packages/ee/dc/e84875682b0593e884ad73b2d40767b5790d417bde603cceb6878901d647/xxhash-3.6.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7f99123f0e1194fa59cc69ad46dbae2e07becec5df50a0509a808f90a0f03f0", size = 445411 },
-    { url = "https://files.pythonhosted.org/packages/11/4f/426f91b96701ec2f37bb2b8cec664eff4f658a11f3fa9d94f0a887ea6d2b/xxhash-3.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49e03e6fe2cac4a1bc64952dd250cf0dbc5ef4ebb7b8d96bce82e2de163c82a2", size = 193883 },
-    { url = "https://files.pythonhosted.org/packages/53/5a/ddbb83eee8e28b778eacfc5a85c969673e4023cdeedcfcef61f36731610b/xxhash-3.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bd17fede52a17a4f9a7bc4472a5867cb0b160deeb431795c0e4abe158bc784e9", size = 210392 },
-    { url = "https://files.pythonhosted.org/packages/1e/c2/ff69efd07c8c074ccdf0a4f36fcdd3d27363665bcdf4ba399abebe643465/xxhash-3.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6fb5f5476bef678f69db04f2bd1efbed3030d2aba305b0fc1773645f187d6a4e", size = 197898 },
-    { url = "https://files.pythonhosted.org/packages/58/ca/faa05ac19b3b622c7c9317ac3e23954187516298a091eb02c976d0d3dd45/xxhash-3.6.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:843b52f6d88071f87eba1631b684fcb4b2068cd2180a0224122fe4ef011a9374", size = 210655 },
-    { url = "https://files.pythonhosted.org/packages/d4/7a/06aa7482345480cc0cb597f5c875b11a82c3953f534394f620b0be2f700c/xxhash-3.6.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7d14a6cfaf03b1b6f5f9790f76880601ccc7896aff7ab9cd8978a939c1eb7e0d", size = 414001 },
-    { url = "https://files.pythonhosted.org/packages/23/07/63ffb386cd47029aa2916b3d2f454e6cc5b9f5c5ada3790377d5430084e7/xxhash-3.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:418daf3db71e1413cfe211c2f9a528456936645c17f46b5204705581a45390ae", size = 191431 },
-    { url = "https://files.pythonhosted.org/packages/0f/93/14fde614cadb4ddf5e7cebf8918b7e8fac5ae7861c1875964f17e678205c/xxhash-3.6.0-cp312-cp312-win32.whl", hash = "sha256:50fc255f39428a27299c20e280d6193d8b63b8ef8028995323bf834a026b4fbb", size = 30617 },
-    { url = "https://files.pythonhosted.org/packages/13/5d/0d125536cbe7565a83d06e43783389ecae0c0f2ed037b48ede185de477c0/xxhash-3.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:c0f2ab8c715630565ab8991b536ecded9416d615538be8ecddce43ccf26cbc7c", size = 31534 },
-    { url = "https://files.pythonhosted.org/packages/54/85/6ec269b0952ec7e36ba019125982cf11d91256a778c7c3f98a4c5043d283/xxhash-3.6.0-cp312-cp312-win_arm64.whl", hash = "sha256:eae5c13f3bc455a3bbb68bdc513912dc7356de7e2280363ea235f71f54064829", size = 27876 },
-    { url = "https://files.pythonhosted.org/packages/33/76/35d05267ac82f53ae9b0e554da7c5e281ee61f3cad44c743f0fcd354f211/xxhash-3.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:599e64ba7f67472481ceb6ee80fa3bd828fd61ba59fb11475572cc5ee52b89ec", size = 32738 },
-    { url = "https://files.pythonhosted.org/packages/31/a8/3fbce1cd96534a95e35d5120637bf29b0d7f5d8fa2f6374e31b4156dd419/xxhash-3.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7d8b8aaa30fca4f16f0c84a5c8d7ddee0e25250ec2796c973775373257dde8f1", size = 30821 },
-    { url = "https://files.pythonhosted.org/packages/0c/ea/d387530ca7ecfa183cb358027f1833297c6ac6098223fd14f9782cd0015c/xxhash-3.6.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d597acf8506d6e7101a4a44a5e428977a51c0fadbbfd3c39650cca9253f6e5a6", size = 194127 },
-    { url = "https://files.pythonhosted.org/packages/ba/0c/71435dcb99874b09a43b8d7c54071e600a7481e42b3e3ce1eb5226a5711a/xxhash-3.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:858dc935963a33bc33490128edc1c12b0c14d9c7ebaa4e387a7869ecc4f3e263", size = 212975 },
-    { url = "https://files.pythonhosted.org/packages/84/7a/c2b3d071e4bb4a90b7057228a99b10d51744878f4a8a6dd643c8bd897620/xxhash-3.6.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba284920194615cb8edf73bf52236ce2e1664ccd4a38fdb543506413529cc546", size = 212241 },
-    { url = "https://files.pythonhosted.org/packages/81/5f/640b6eac0128e215f177df99eadcd0f1b7c42c274ab6a394a05059694c5a/xxhash-3.6.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b54219177f6c6674d5378bd862c6aedf64725f70dd29c472eaae154df1a2e89", size = 445471 },
-    { url = "https://files.pythonhosted.org/packages/5e/1e/3c3d3ef071b051cc3abbe3721ffb8365033a172613c04af2da89d5548a87/xxhash-3.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42c36dd7dbad2f5238950c377fcbf6811b1cdb1c444fab447960030cea60504d", size = 193936 },
-    { url = "https://files.pythonhosted.org/packages/2c/bd/4a5f68381939219abfe1c22a9e3a5854a4f6f6f3c4983a87d255f21f2e5d/xxhash-3.6.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f22927652cba98c44639ffdc7aaf35828dccf679b10b31c4ad72a5b530a18eb7", size = 210440 },
-    { url = "https://files.pythonhosted.org/packages/eb/37/b80fe3d5cfb9faff01a02121a0f4d565eb7237e9e5fc66e73017e74dcd36/xxhash-3.6.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b45fad44d9c5c119e9c6fbf2e1c656a46dc68e280275007bbfd3d572b21426db", size = 197990 },
-    { url = "https://files.pythonhosted.org/packages/d7/fd/2c0a00c97b9e18f72e1f240ad4e8f8a90fd9d408289ba9c7c495ed7dc05c/xxhash-3.6.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:6f2580ffab1a8b68ef2b901cde7e55fa8da5e4be0977c68f78fc80f3c143de42", size = 210689 },
-    { url = "https://files.pythonhosted.org/packages/93/86/5dd8076a926b9a95db3206aba20d89a7fc14dd5aac16e5c4de4b56033140/xxhash-3.6.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40c391dd3cd041ebc3ffe6f2c862f402e306eb571422e0aa918d8070ba31da11", size = 414068 },
-    { url = "https://files.pythonhosted.org/packages/af/3c/0bb129170ee8f3650f08e993baee550a09593462a5cddd8e44d0011102b1/xxhash-3.6.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f205badabde7aafd1a31e8ca2a3e5a763107a71c397c4481d6a804eb5063d8bd", size = 191495 },
-    { url = "https://files.pythonhosted.org/packages/e9/3a/6797e0114c21d1725e2577508e24006fd7ff1d8c0c502d3b52e45c1771d8/xxhash-3.6.0-cp313-cp313-win32.whl", hash = "sha256:2577b276e060b73b73a53042ea5bd5203d3e6347ce0d09f98500f418a9fcf799", size = 30620 },
-    { url = "https://files.pythonhosted.org/packages/86/15/9bc32671e9a38b413a76d24722a2bf8784a132c043063a8f5152d390b0f9/xxhash-3.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:757320d45d2fbcce8f30c42a6b2f47862967aea7bf458b9625b4bbe7ee390392", size = 31542 },
-    { url = "https://files.pythonhosted.org/packages/39/c5/cc01e4f6188656e56112d6a8e0dfe298a16934b8c47a247236549a3f7695/xxhash-3.6.0-cp313-cp313-win_arm64.whl", hash = "sha256:457b8f85dec5825eed7b69c11ae86834a018b8e3df5e77783c999663da2f96d6", size = 27880 },
-    { url = "https://files.pythonhosted.org/packages/f3/30/25e5321c8732759e930c555176d37e24ab84365482d257c3b16362235212/xxhash-3.6.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a42e633d75cdad6d625434e3468126c73f13f7584545a9cf34e883aa1710e702", size = 32956 },
-    { url = "https://files.pythonhosted.org/packages/9f/3c/0573299560d7d9f8ab1838f1efc021a280b5ae5ae2e849034ef3dee18810/xxhash-3.6.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:568a6d743219e717b07b4e03b0a828ce593833e498c3b64752e0f5df6bfe84db", size = 31072 },
-    { url = "https://files.pythonhosted.org/packages/7a/1c/52d83a06e417cd9d4137722693424885cc9878249beb3a7c829e74bf7ce9/xxhash-3.6.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bec91b562d8012dae276af8025a55811b875baace6af510412a5e58e3121bc54", size = 196409 },
-    { url = "https://files.pythonhosted.org/packages/e3/8e/c6d158d12a79bbd0b878f8355432075fc82759e356ab5a111463422a239b/xxhash-3.6.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78e7f2f4c521c30ad5e786fdd6bae89d47a32672a80195467b5de0480aa97b1f", size = 215736 },
-    { url = "https://files.pythonhosted.org/packages/bc/68/c4c80614716345d55071a396cf03d06e34b5f4917a467faf43083c995155/xxhash-3.6.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ed0df1b11a79856df5ffcab572cbd6b9627034c1c748c5566fa79df9048a7c5", size = 214833 },
-    { url = "https://files.pythonhosted.org/packages/7e/e9/ae27c8ffec8b953efa84c7c4a6c6802c263d587b9fc0d6e7cea64e08c3af/xxhash-3.6.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e4edbfc7d420925b0dd5e792478ed393d6e75ff8fc219a6546fb446b6a417b1", size = 448348 },
-    { url = "https://files.pythonhosted.org/packages/d7/6b/33e21afb1b5b3f46b74b6bd1913639066af218d704cc0941404ca717fc57/xxhash-3.6.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fba27a198363a7ef87f8c0f6b171ec36b674fe9053742c58dd7e3201c1ab30ee", size = 196070 },
-    { url = "https://files.pythonhosted.org/packages/96/b6/fcabd337bc5fa624e7203aa0fa7d0c49eed22f72e93229431752bddc83d9/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:794fe9145fe60191c6532fa95063765529770edcdd67b3d537793e8004cabbfd", size = 212907 },
-    { url = "https://files.pythonhosted.org/packages/4b/d3/9ee6160e644d660fcf176c5825e61411c7f62648728f69c79ba237250143/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:6105ef7e62b5ac73a837778efc331a591d8442f8ef5c7e102376506cb4ae2729", size = 200839 },
-    { url = "https://files.pythonhosted.org/packages/0d/98/e8de5baa5109394baf5118f5e72ab21a86387c4f89b0e77ef3e2f6b0327b/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:f01375c0e55395b814a679b3eea205db7919ac2af213f4a6682e01220e5fe292", size = 213304 },
-    { url = "https://files.pythonhosted.org/packages/7b/1d/71056535dec5c3177eeb53e38e3d367dd1d16e024e63b1cee208d572a033/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:d706dca2d24d834a4661619dcacf51a75c16d65985718d6a7d73c1eeeb903ddf", size = 416930 },
-    { url = "https://files.pythonhosted.org/packages/dc/6c/5cbde9de2cd967c322e651c65c543700b19e7ae3e0aae8ece3469bf9683d/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5f059d9faeacd49c0215d66f4056e1326c80503f51a1532ca336a385edadd033", size = 193787 },
-    { url = "https://files.pythonhosted.org/packages/19/fa/0172e350361d61febcea941b0cc541d6e6c8d65d153e85f850a7b256ff8a/xxhash-3.6.0-cp313-cp313t-win32.whl", hash = "sha256:1244460adc3a9be84731d72b8e80625788e5815b68da3da8b83f78115a40a7ec", size = 30916 },
-    { url = "https://files.pythonhosted.org/packages/ad/e6/e8cf858a2b19d6d45820f072eff1bea413910592ff17157cabc5f1227a16/xxhash-3.6.0-cp313-cp313t-win_amd64.whl", hash = "sha256:b1e420ef35c503869c4064f4a2f2b08ad6431ab7b229a05cce39d74268bca6b8", size = 31799 },
-    { url = "https://files.pythonhosted.org/packages/56/15/064b197e855bfb7b343210e82490ae672f8bc7cdf3ddb02e92f64304ee8a/xxhash-3.6.0-cp313-cp313t-win_arm64.whl", hash = "sha256:ec44b73a4220623235f67a996c862049f375df3b1052d9899f40a6382c32d746", size = 28044 },
-    { url = "https://files.pythonhosted.org/packages/7e/5e/0138bc4484ea9b897864d59fce9be9086030825bc778b76cb5a33a906d37/xxhash-3.6.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a40a3d35b204b7cc7643cbcf8c9976d818cb47befcfac8bbefec8038ac363f3e", size = 32754 },
-    { url = "https://files.pythonhosted.org/packages/18/d7/5dac2eb2ec75fd771957a13e5dda560efb2176d5203f39502a5fc571f899/xxhash-3.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a54844be970d3fc22630b32d515e79a90d0a3ddb2644d8d7402e3c4c8da61405", size = 30846 },
-    { url = "https://files.pythonhosted.org/packages/fe/71/8bc5be2bb00deb5682e92e8da955ebe5fa982da13a69da5a40a4c8db12fb/xxhash-3.6.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:016e9190af8f0a4e3741343777710e3d5717427f175adfdc3e72508f59e2a7f3", size = 194343 },
-    { url = "https://files.pythonhosted.org/packages/e7/3b/52badfb2aecec2c377ddf1ae75f55db3ba2d321c5e164f14461c90837ef3/xxhash-3.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f6f72232f849eb9d0141e2ebe2677ece15adfd0fa599bc058aad83c714bb2c6", size = 213074 },
-    { url = "https://files.pythonhosted.org/packages/a2/2b/ae46b4e9b92e537fa30d03dbc19cdae57ed407e9c26d163895e968e3de85/xxhash-3.6.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63275a8aba7865e44b1813d2177e0f5ea7eadad3dd063a21f7cf9afdc7054063", size = 212388 },
-    { url = "https://files.pythonhosted.org/packages/f5/80/49f88d3afc724b4ac7fbd664c8452d6db51b49915be48c6982659e0e7942/xxhash-3.6.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cd01fa2aa00d8b017c97eb46b9a794fbdca53fc14f845f5a328c71254b0abb7", size = 445614 },
-    { url = "https://files.pythonhosted.org/packages/ed/ba/603ce3961e339413543d8cd44f21f2c80e2a7c5cfe692a7b1f2cccf58f3c/xxhash-3.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0226aa89035b62b6a86d3c68df4d7c1f47a342b8683da2b60cedcddb46c4d95b", size = 194024 },
-    { url = "https://files.pythonhosted.org/packages/78/d1/8e225ff7113bf81545cfdcd79eef124a7b7064a0bba53605ff39590b95c2/xxhash-3.6.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c6e193e9f56e4ca4923c61238cdaced324f0feac782544eb4c6d55ad5cc99ddd", size = 210541 },
-    { url = "https://files.pythonhosted.org/packages/6f/58/0f89d149f0bad89def1a8dd38feb50ccdeb643d9797ec84707091d4cb494/xxhash-3.6.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9176dcaddf4ca963d4deb93866d739a343c01c969231dbe21680e13a5d1a5bf0", size = 198305 },
-    { url = "https://files.pythonhosted.org/packages/11/38/5eab81580703c4df93feb5f32ff8fa7fe1e2c51c1f183ee4e48d4bb9d3d7/xxhash-3.6.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c1ce4009c97a752e682b897aa99aef84191077a9433eb237774689f14f8ec152", size = 210848 },
-    { url = "https://files.pythonhosted.org/packages/5e/6b/953dc4b05c3ce678abca756416e4c130d2382f877a9c30a20d08ee6a77c0/xxhash-3.6.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:8cb2f4f679b01513b7adbb9b1b2f0f9cdc31b70007eaf9d59d0878809f385b11", size = 414142 },
-    { url = "https://files.pythonhosted.org/packages/08/a9/238ec0d4e81a10eb5026d4a6972677cbc898ba6c8b9dbaec12ae001b1b35/xxhash-3.6.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:653a91d7c2ab54a92c19ccf43508b6a555440b9be1bc8be553376778be7f20b5", size = 191547 },
-    { url = "https://files.pythonhosted.org/packages/f1/ee/3cf8589e06c2164ac77c3bf0aa127012801128f1feebf2a079272da5737c/xxhash-3.6.0-cp314-cp314-win32.whl", hash = "sha256:a756fe893389483ee8c394d06b5ab765d96e68fbbfe6fde7aa17e11f5720559f", size = 31214 },
-    { url = "https://files.pythonhosted.org/packages/02/5d/a19552fbc6ad4cb54ff953c3908bbc095f4a921bc569433d791f755186f1/xxhash-3.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:39be8e4e142550ef69629c9cd71b88c90e9a5db703fecbcf265546d9536ca4ad", size = 32290 },
-    { url = "https://files.pythonhosted.org/packages/b1/11/dafa0643bc30442c887b55baf8e73353a344ee89c1901b5a5c54a6c17d39/xxhash-3.6.0-cp314-cp314-win_arm64.whl", hash = "sha256:25915e6000338999236f1eb68a02a32c3275ac338628a7eaa5a269c401995679", size = 28795 },
-    { url = "https://files.pythonhosted.org/packages/2c/db/0e99732ed7f64182aef4a6fb145e1a295558deec2a746265dcdec12d191e/xxhash-3.6.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c5294f596a9017ca5a3e3f8884c00b91ab2ad2933cf288f4923c3fd4346cf3d4", size = 32955 },
-    { url = "https://files.pythonhosted.org/packages/55/f4/2a7c3c68e564a099becfa44bb3d398810cc0ff6749b0d3cb8ccb93f23c14/xxhash-3.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1cf9dcc4ab9cff01dfbba78544297a3a01dafd60f3bde4e2bfd016cf7e4ddc67", size = 31072 },
-    { url = "https://files.pythonhosted.org/packages/c6/d9/72a29cddc7250e8a5819dad5d466facb5dc4c802ce120645630149127e73/xxhash-3.6.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:01262da8798422d0685f7cef03b2bd3f4f46511b02830861df548d7def4402ad", size = 196579 },
-    { url = "https://files.pythonhosted.org/packages/63/93/b21590e1e381040e2ca305a884d89e1c345b347404f7780f07f2cdd47ef4/xxhash-3.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51a73fb7cb3a3ead9f7a8b583ffd9b8038e277cdb8cb87cf890e88b3456afa0b", size = 215854 },
-    { url = "https://files.pythonhosted.org/packages/ce/b8/edab8a7d4fa14e924b29be877d54155dcbd8b80be85ea00d2be3413a9ed4/xxhash-3.6.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b9c6df83594f7df8f7f708ce5ebeacfc69f72c9fbaaababf6cf4758eaada0c9b", size = 214965 },
-    { url = "https://files.pythonhosted.org/packages/27/67/dfa980ac7f0d509d54ea0d5a486d2bb4b80c3f1bb22b66e6a05d3efaf6c0/xxhash-3.6.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:627f0af069b0ea56f312fd5189001c24578868643203bca1abbc2c52d3a6f3ca", size = 448484 },
-    { url = "https://files.pythonhosted.org/packages/8c/63/8ffc2cc97e811c0ca5d00ab36604b3ea6f4254f20b7bc658ca825ce6c954/xxhash-3.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa912c62f842dfd013c5f21a642c9c10cd9f4c4e943e0af83618b4a404d9091a", size = 196162 },
-    { url = "https://files.pythonhosted.org/packages/4b/77/07f0e7a3edd11a6097e990f6e5b815b6592459cb16dae990d967693e6ea9/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b465afd7909db30168ab62afe40b2fcf79eedc0b89a6c0ab3123515dc0df8b99", size = 213007 },
-    { url = "https://files.pythonhosted.org/packages/ae/d8/bc5fa0d152837117eb0bef6f83f956c509332ce133c91c63ce07ee7c4873/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:a881851cf38b0a70e7c4d3ce81fc7afd86fbc2a024f4cfb2a97cf49ce04b75d3", size = 200956 },
-    { url = "https://files.pythonhosted.org/packages/26/a5/d749334130de9411783873e9b98ecc46688dad5db64ca6e04b02acc8b473/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:9b3222c686a919a0f3253cfc12bb118b8b103506612253b5baeaac10d8027cf6", size = 213401 },
-    { url = "https://files.pythonhosted.org/packages/89/72/abed959c956a4bfc72b58c0384bb7940663c678127538634d896b1195c10/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:c5aa639bc113e9286137cec8fadc20e9cd732b2cc385c0b7fa673b84fc1f2a93", size = 417083 },
-    { url = "https://files.pythonhosted.org/packages/0c/b3/62fd2b586283b7d7d665fb98e266decadf31f058f1cf6c478741f68af0cb/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5c1343d49ac102799905e115aee590183c3921d475356cb24b4de29a4bc56518", size = 193913 },
-    { url = "https://files.pythonhosted.org/packages/9a/9a/c19c42c5b3f5a4aad748a6d5b4f23df3bed7ee5445accc65a0fb3ff03953/xxhash-3.6.0-cp314-cp314t-win32.whl", hash = "sha256:5851f033c3030dd95c086b4a36a2683c2ff4a799b23af60977188b057e467119", size = 31586 },
-    { url = "https://files.pythonhosted.org/packages/03/d6/4cc450345be9924fd5dc8c590ceda1db5b43a0a889587b0ae81a95511360/xxhash-3.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0444e7967dac37569052d2409b00a8860c2135cff05502df4da80267d384849f", size = 32526 },
-    { url = "https://files.pythonhosted.org/packages/0f/c9/7243eb3f9eaabd1a88a5a5acadf06df2d83b100c62684b7425c6a11bcaa8/xxhash-3.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:bb79b1e63f6fd84ec778a4b1916dfe0a7c3fdb986c06addd5db3a0d413819d95", size = 28898 },
-    { url = "https://files.pythonhosted.org/packages/93/1e/8aec23647a34a249f62e2398c42955acd9b4c6ed5cf08cbea94dc46f78d2/xxhash-3.6.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0f7b7e2ec26c1666ad5fc9dbfa426a6a3367ceaf79db5dd76264659d509d73b0", size = 30662 },
-    { url = "https://files.pythonhosted.org/packages/b8/0b/b14510b38ba91caf43006209db846a696ceea6a847a0c9ba0a5b1adc53d6/xxhash-3.6.0-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5dc1e14d14fa0f5789ec29a7062004b5933964bb9b02aae6622b8f530dc40296", size = 41056 },
-    { url = "https://files.pythonhosted.org/packages/50/55/15a7b8a56590e66ccd374bbfa3f9ffc45b810886c8c3b614e3f90bd2367c/xxhash-3.6.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:881b47fc47e051b37d94d13e7455131054b56749b91b508b0907eb07900d1c13", size = 36251 },
-    { url = "https://files.pythonhosted.org/packages/62/b2/5ac99a041a29e58e95f907876b04f7067a0242cb85b5f39e726153981503/xxhash-3.6.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6dc31591899f5e5666f04cc2e529e69b4072827085c1ef15294d91a004bc1bd", size = 32481 },
-    { url = "https://files.pythonhosted.org/packages/7b/d9/8d95e906764a386a3d3b596f3c68bb63687dfca806373509f51ce8eea81f/xxhash-3.6.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:15e0dac10eb9309508bfc41f7f9deaa7755c69e35af835db9cb10751adebc35d", size = 31565 },
+sdist = { url = "https://files.pythonhosted.org/packages/02/84/30869e01909fb37a6cc7e18688ee8bf1e42d57e7e0777636bd47524c43c7/xxhash-3.6.0.tar.gz", hash = "sha256:f0162a78b13a0d7617b2845b90c763339d1f1d82bb04a4b07f4ab535cc5e05d6", size = 85160, upload-time = "2025-10-02T14:37:08.097Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/34/ee/f9f1d656ad168681bb0f6b092372c1e533c4416b8069b1896a175c46e484/xxhash-3.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:87ff03d7e35c61435976554477a7f4cd1704c3596a89a8300d5ce7fc83874a71", size = 32845, upload-time = "2025-10-02T14:33:51.573Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/b1/93508d9460b292c74a09b83d16750c52a0ead89c51eea9951cb97a60d959/xxhash-3.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f572dfd3d0e2eb1a57511831cf6341242f5a9f8298a45862d085f5b93394a27d", size = 30807, upload-time = "2025-10-02T14:33:52.964Z" },
+    { url = "https://files.pythonhosted.org/packages/07/55/28c93a3662f2d200c70704efe74aab9640e824f8ce330d8d3943bf7c9b3c/xxhash-3.6.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:89952ea539566b9fed2bbd94e589672794b4286f342254fad28b149f9615fef8", size = 193786, upload-time = "2025-10-02T14:33:54.272Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/96/fec0be9bb4b8f5d9c57d76380a366f31a1781fb802f76fc7cda6c84893c7/xxhash-3.6.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48e6f2ffb07a50b52465a1032c3cf1f4a5683f944acaca8a134a2f23674c2058", size = 212830, upload-time = "2025-10-02T14:33:55.706Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/a0/c706845ba77b9611f81fd2e93fad9859346b026e8445e76f8c6fd057cc6d/xxhash-3.6.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b5b848ad6c16d308c3ac7ad4ba6bede80ed5df2ba8ed382f8932df63158dd4b2", size = 211606, upload-time = "2025-10-02T14:33:57.133Z" },
+    { url = "https://files.pythonhosted.org/packages/67/1e/164126a2999e5045f04a69257eea946c0dc3e86541b400d4385d646b53d7/xxhash-3.6.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a034590a727b44dd8ac5914236a7b8504144447a9682586c3327e935f33ec8cc", size = 444872, upload-time = "2025-10-02T14:33:58.446Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/4b/55ab404c56cd70a2cf5ecfe484838865d0fea5627365c6c8ca156bd09c8f/xxhash-3.6.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a8f1972e75ebdd161d7896743122834fe87378160c20e97f8b09166213bf8cc", size = 193217, upload-time = "2025-10-02T14:33:59.724Z" },
+    { url = "https://files.pythonhosted.org/packages/45/e6/52abf06bac316db33aa269091ae7311bd53cfc6f4b120ae77bac1b348091/xxhash-3.6.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ee34327b187f002a596d7b167ebc59a1b729e963ce645964bbc050d2f1b73d07", size = 210139, upload-time = "2025-10-02T14:34:02.041Z" },
+    { url = "https://files.pythonhosted.org/packages/34/37/db94d490b8691236d356bc249c08819cbcef9273a1a30acf1254ff9ce157/xxhash-3.6.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:339f518c3c7a850dd033ab416ea25a692759dc7478a71131fe8869010d2b75e4", size = 197669, upload-time = "2025-10-02T14:34:03.664Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/36/c4f219ef4a17a4f7a64ed3569bc2b5a9c8311abdb22249ac96093625b1a4/xxhash-3.6.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:bf48889c9630542d4709192578aebbd836177c9f7a4a2778a7d6340107c65f06", size = 210018, upload-time = "2025-10-02T14:34:05.325Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/06/bfac889a374fc2fc439a69223d1750eed2e18a7db8514737ab630534fa08/xxhash-3.6.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5576b002a56207f640636056b4160a378fe36a58db73ae5c27a7ec8db35f71d4", size = 413058, upload-time = "2025-10-02T14:34:06.925Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/d1/555d8447e0dd32ad0930a249a522bb2e289f0d08b6b16204cfa42c1f5a0c/xxhash-3.6.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af1f3278bd02814d6dedc5dec397993b549d6f16c19379721e5a1d31e132c49b", size = 190628, upload-time = "2025-10-02T14:34:08.669Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/15/8751330b5186cedc4ed4b597989882ea05e0408b53fa47bcb46a6125bfc6/xxhash-3.6.0-cp310-cp310-win32.whl", hash = "sha256:aed058764db109dc9052720da65fafe84873b05eb8b07e5e653597951af57c3b", size = 30577, upload-time = "2025-10-02T14:34:10.234Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/cc/53f87e8b5871a6eb2ff7e89c48c66093bda2be52315a8161ddc54ea550c4/xxhash-3.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:e82da5670f2d0d98950317f82a0e4a0197150ff19a6df2ba40399c2a3b9ae5fb", size = 31487, upload-time = "2025-10-02T14:34:11.618Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/00/60f9ea3bb697667a14314d7269956f58bf56bb73864f8f8d52a3c2535e9a/xxhash-3.6.0-cp310-cp310-win_arm64.whl", hash = "sha256:4a082ffff8c6ac07707fb6b671caf7c6e020c75226c561830b73d862060f281d", size = 27863, upload-time = "2025-10-02T14:34:12.619Z" },
+    { url = "https://files.pythonhosted.org/packages/17/d4/cc2f0400e9154df4b9964249da78ebd72f318e35ccc425e9f403c392f22a/xxhash-3.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b47bbd8cf2d72797f3c2772eaaac0ded3d3af26481a26d7d7d41dc2d3c46b04a", size = 32844, upload-time = "2025-10-02T14:34:14.037Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/ec/1cc11cd13e26ea8bc3cb4af4eaadd8d46d5014aebb67be3f71fb0b68802a/xxhash-3.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2b6821e94346f96db75abaa6e255706fb06ebd530899ed76d32cd99f20dc52fa", size = 30809, upload-time = "2025-10-02T14:34:15.484Z" },
+    { url = "https://files.pythonhosted.org/packages/04/5f/19fe357ea348d98ca22f456f75a30ac0916b51c753e1f8b2e0e6fb884cce/xxhash-3.6.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d0a9751f71a1a65ce3584e9cae4467651c7e70c9d31017fa57574583a4540248", size = 194665, upload-time = "2025-10-02T14:34:16.541Z" },
+    { url = "https://files.pythonhosted.org/packages/90/3b/d1f1a8f5442a5fd8beedae110c5af7604dc37349a8e16519c13c19a9a2de/xxhash-3.6.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b29ee68625ab37b04c0b40c3fafdf24d2f75ccd778333cfb698f65f6c463f62", size = 213550, upload-time = "2025-10-02T14:34:17.878Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/ef/3a9b05eb527457d5db13a135a2ae1a26c80fecd624d20f3e8dcc4cb170f3/xxhash-3.6.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6812c25fe0d6c36a46ccb002f40f27ac903bf18af9f6dd8f9669cb4d176ab18f", size = 212384, upload-time = "2025-10-02T14:34:19.182Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/18/ccc194ee698c6c623acbf0f8c2969811a8a4b6185af5e824cd27b9e4fd3e/xxhash-3.6.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4ccbff013972390b51a18ef1255ef5ac125c92dc9143b2d1909f59abc765540e", size = 445749, upload-time = "2025-10-02T14:34:20.659Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/86/cf2c0321dc3940a7aa73076f4fd677a0fb3e405cb297ead7d864fd90847e/xxhash-3.6.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:297b7fbf86c82c550e12e8fb71968b3f033d27b874276ba3624ea868c11165a8", size = 193880, upload-time = "2025-10-02T14:34:22.431Z" },
+    { url = "https://files.pythonhosted.org/packages/82/fb/96213c8560e6f948a1ecc9a7613f8032b19ee45f747f4fca4eb31bb6d6ed/xxhash-3.6.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dea26ae1eb293db089798d3973a5fc928a18fdd97cc8801226fae705b02b14b0", size = 210912, upload-time = "2025-10-02T14:34:23.937Z" },
+    { url = "https://files.pythonhosted.org/packages/40/aa/4395e669b0606a096d6788f40dbdf2b819d6773aa290c19e6e83cbfc312f/xxhash-3.6.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7a0b169aafb98f4284f73635a8e93f0735f9cbde17bd5ec332480484241aaa77", size = 198654, upload-time = "2025-10-02T14:34:25.644Z" },
+    { url = "https://files.pythonhosted.org/packages/67/74/b044fcd6b3d89e9b1b665924d85d3f400636c23590226feb1eb09e1176ce/xxhash-3.6.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:08d45aef063a4531b785cd72de4887766d01dc8f362a515693df349fdb825e0c", size = 210867, upload-time = "2025-10-02T14:34:27.203Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/fd/3ce73bf753b08cb19daee1eb14aa0d7fe331f8da9c02dd95316ddfe5275e/xxhash-3.6.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:929142361a48ee07f09121fe9e96a84950e8d4df3bb298ca5d88061969f34d7b", size = 414012, upload-time = "2025-10-02T14:34:28.409Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/b3/5a4241309217c5c876f156b10778f3ab3af7ba7e3259e6d5f5c7d0129eb2/xxhash-3.6.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:51312c768403d8540487dbbfb557454cfc55589bbde6424456951f7fcd4facb3", size = 191409, upload-time = "2025-10-02T14:34:29.696Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/01/99bfbc15fb9abb9a72b088c1d95219fc4782b7d01fc835bd5744d66dd0b8/xxhash-3.6.0-cp311-cp311-win32.whl", hash = "sha256:d1927a69feddc24c987b337ce81ac15c4720955b667fe9b588e02254b80446fd", size = 30574, upload-time = "2025-10-02T14:34:31.028Z" },
+    { url = "https://files.pythonhosted.org/packages/65/79/9d24d7f53819fe301b231044ea362ce64e86c74f6e8c8e51320de248b3e5/xxhash-3.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:26734cdc2d4ffe449b41d186bbeac416f704a482ed835d375a5c0cb02bc63fef", size = 31481, upload-time = "2025-10-02T14:34:32.062Z" },
+    { url = "https://files.pythonhosted.org/packages/30/4e/15cd0e3e8772071344eab2961ce83f6e485111fed8beb491a3f1ce100270/xxhash-3.6.0-cp311-cp311-win_arm64.whl", hash = "sha256:d72f67ef8bf36e05f5b6c65e8524f265bd61071471cd4cf1d36743ebeeeb06b7", size = 27861, upload-time = "2025-10-02T14:34:33.555Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/07/d9412f3d7d462347e4511181dea65e47e0d0e16e26fbee2ea86a2aefb657/xxhash-3.6.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:01362c4331775398e7bb34e3ab403bc9ee9f7c497bc7dee6272114055277dd3c", size = 32744, upload-time = "2025-10-02T14:34:34.622Z" },
+    { url = "https://files.pythonhosted.org/packages/79/35/0429ee11d035fc33abe32dca1b2b69e8c18d236547b9a9b72c1929189b9a/xxhash-3.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b7b2df81a23f8cb99656378e72501b2cb41b1827c0f5a86f87d6b06b69f9f204", size = 30816, upload-time = "2025-10-02T14:34:36.043Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/f2/57eb99aa0f7d98624c0932c5b9a170e1806406cdbcdb510546634a1359e0/xxhash-3.6.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:dc94790144e66b14f67b10ac8ed75b39ca47536bf8800eb7c24b50271ea0c490", size = 194035, upload-time = "2025-10-02T14:34:37.354Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/ed/6224ba353690d73af7a3f1c7cdb1fc1b002e38f783cb991ae338e1eb3d79/xxhash-3.6.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93f107c673bccf0d592cdba077dedaf52fe7f42dcd7676eba1f6d6f0c3efffd2", size = 212914, upload-time = "2025-10-02T14:34:38.6Z" },
+    { url = "https://files.pythonhosted.org/packages/38/86/fb6b6130d8dd6b8942cc17ab4d90e223653a89aa32ad2776f8af7064ed13/xxhash-3.6.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2aa5ee3444c25b69813663c9f8067dcfaa2e126dc55e8dddf40f4d1c25d7effa", size = 212163, upload-time = "2025-10-02T14:34:39.872Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/dc/e84875682b0593e884ad73b2d40767b5790d417bde603cceb6878901d647/xxhash-3.6.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7f99123f0e1194fa59cc69ad46dbae2e07becec5df50a0509a808f90a0f03f0", size = 445411, upload-time = "2025-10-02T14:34:41.569Z" },
+    { url = "https://files.pythonhosted.org/packages/11/4f/426f91b96701ec2f37bb2b8cec664eff4f658a11f3fa9d94f0a887ea6d2b/xxhash-3.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49e03e6fe2cac4a1bc64952dd250cf0dbc5ef4ebb7b8d96bce82e2de163c82a2", size = 193883, upload-time = "2025-10-02T14:34:43.249Z" },
+    { url = "https://files.pythonhosted.org/packages/53/5a/ddbb83eee8e28b778eacfc5a85c969673e4023cdeedcfcef61f36731610b/xxhash-3.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bd17fede52a17a4f9a7bc4472a5867cb0b160deeb431795c0e4abe158bc784e9", size = 210392, upload-time = "2025-10-02T14:34:45.042Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/c2/ff69efd07c8c074ccdf0a4f36fcdd3d27363665bcdf4ba399abebe643465/xxhash-3.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6fb5f5476bef678f69db04f2bd1efbed3030d2aba305b0fc1773645f187d6a4e", size = 197898, upload-time = "2025-10-02T14:34:46.302Z" },
+    { url = "https://files.pythonhosted.org/packages/58/ca/faa05ac19b3b622c7c9317ac3e23954187516298a091eb02c976d0d3dd45/xxhash-3.6.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:843b52f6d88071f87eba1631b684fcb4b2068cd2180a0224122fe4ef011a9374", size = 210655, upload-time = "2025-10-02T14:34:47.571Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/7a/06aa7482345480cc0cb597f5c875b11a82c3953f534394f620b0be2f700c/xxhash-3.6.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7d14a6cfaf03b1b6f5f9790f76880601ccc7896aff7ab9cd8978a939c1eb7e0d", size = 414001, upload-time = "2025-10-02T14:34:49.273Z" },
+    { url = "https://files.pythonhosted.org/packages/23/07/63ffb386cd47029aa2916b3d2f454e6cc5b9f5c5ada3790377d5430084e7/xxhash-3.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:418daf3db71e1413cfe211c2f9a528456936645c17f46b5204705581a45390ae", size = 191431, upload-time = "2025-10-02T14:34:50.798Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/93/14fde614cadb4ddf5e7cebf8918b7e8fac5ae7861c1875964f17e678205c/xxhash-3.6.0-cp312-cp312-win32.whl", hash = "sha256:50fc255f39428a27299c20e280d6193d8b63b8ef8028995323bf834a026b4fbb", size = 30617, upload-time = "2025-10-02T14:34:51.954Z" },
+    { url = "https://files.pythonhosted.org/packages/13/5d/0d125536cbe7565a83d06e43783389ecae0c0f2ed037b48ede185de477c0/xxhash-3.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:c0f2ab8c715630565ab8991b536ecded9416d615538be8ecddce43ccf26cbc7c", size = 31534, upload-time = "2025-10-02T14:34:53.276Z" },
+    { url = "https://files.pythonhosted.org/packages/54/85/6ec269b0952ec7e36ba019125982cf11d91256a778c7c3f98a4c5043d283/xxhash-3.6.0-cp312-cp312-win_arm64.whl", hash = "sha256:eae5c13f3bc455a3bbb68bdc513912dc7356de7e2280363ea235f71f54064829", size = 27876, upload-time = "2025-10-02T14:34:54.371Z" },
+    { url = "https://files.pythonhosted.org/packages/33/76/35d05267ac82f53ae9b0e554da7c5e281ee61f3cad44c743f0fcd354f211/xxhash-3.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:599e64ba7f67472481ceb6ee80fa3bd828fd61ba59fb11475572cc5ee52b89ec", size = 32738, upload-time = "2025-10-02T14:34:55.839Z" },
+    { url = "https://files.pythonhosted.org/packages/31/a8/3fbce1cd96534a95e35d5120637bf29b0d7f5d8fa2f6374e31b4156dd419/xxhash-3.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7d8b8aaa30fca4f16f0c84a5c8d7ddee0e25250ec2796c973775373257dde8f1", size = 30821, upload-time = "2025-10-02T14:34:57.219Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/ea/d387530ca7ecfa183cb358027f1833297c6ac6098223fd14f9782cd0015c/xxhash-3.6.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d597acf8506d6e7101a4a44a5e428977a51c0fadbbfd3c39650cca9253f6e5a6", size = 194127, upload-time = "2025-10-02T14:34:59.21Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/0c/71435dcb99874b09a43b8d7c54071e600a7481e42b3e3ce1eb5226a5711a/xxhash-3.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:858dc935963a33bc33490128edc1c12b0c14d9c7ebaa4e387a7869ecc4f3e263", size = 212975, upload-time = "2025-10-02T14:35:00.816Z" },
+    { url = "https://files.pythonhosted.org/packages/84/7a/c2b3d071e4bb4a90b7057228a99b10d51744878f4a8a6dd643c8bd897620/xxhash-3.6.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba284920194615cb8edf73bf52236ce2e1664ccd4a38fdb543506413529cc546", size = 212241, upload-time = "2025-10-02T14:35:02.207Z" },
+    { url = "https://files.pythonhosted.org/packages/81/5f/640b6eac0128e215f177df99eadcd0f1b7c42c274ab6a394a05059694c5a/xxhash-3.6.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b54219177f6c6674d5378bd862c6aedf64725f70dd29c472eaae154df1a2e89", size = 445471, upload-time = "2025-10-02T14:35:03.61Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/1e/3c3d3ef071b051cc3abbe3721ffb8365033a172613c04af2da89d5548a87/xxhash-3.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42c36dd7dbad2f5238950c377fcbf6811b1cdb1c444fab447960030cea60504d", size = 193936, upload-time = "2025-10-02T14:35:05.013Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/bd/4a5f68381939219abfe1c22a9e3a5854a4f6f6f3c4983a87d255f21f2e5d/xxhash-3.6.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f22927652cba98c44639ffdc7aaf35828dccf679b10b31c4ad72a5b530a18eb7", size = 210440, upload-time = "2025-10-02T14:35:06.239Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/37/b80fe3d5cfb9faff01a02121a0f4d565eb7237e9e5fc66e73017e74dcd36/xxhash-3.6.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b45fad44d9c5c119e9c6fbf2e1c656a46dc68e280275007bbfd3d572b21426db", size = 197990, upload-time = "2025-10-02T14:35:07.735Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/fd/2c0a00c97b9e18f72e1f240ad4e8f8a90fd9d408289ba9c7c495ed7dc05c/xxhash-3.6.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:6f2580ffab1a8b68ef2b901cde7e55fa8da5e4be0977c68f78fc80f3c143de42", size = 210689, upload-time = "2025-10-02T14:35:09.438Z" },
+    { url = "https://files.pythonhosted.org/packages/93/86/5dd8076a926b9a95db3206aba20d89a7fc14dd5aac16e5c4de4b56033140/xxhash-3.6.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40c391dd3cd041ebc3ffe6f2c862f402e306eb571422e0aa918d8070ba31da11", size = 414068, upload-time = "2025-10-02T14:35:11.162Z" },
+    { url = "https://files.pythonhosted.org/packages/af/3c/0bb129170ee8f3650f08e993baee550a09593462a5cddd8e44d0011102b1/xxhash-3.6.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f205badabde7aafd1a31e8ca2a3e5a763107a71c397c4481d6a804eb5063d8bd", size = 191495, upload-time = "2025-10-02T14:35:12.971Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/3a/6797e0114c21d1725e2577508e24006fd7ff1d8c0c502d3b52e45c1771d8/xxhash-3.6.0-cp313-cp313-win32.whl", hash = "sha256:2577b276e060b73b73a53042ea5bd5203d3e6347ce0d09f98500f418a9fcf799", size = 30620, upload-time = "2025-10-02T14:35:14.129Z" },
+    { url = "https://files.pythonhosted.org/packages/86/15/9bc32671e9a38b413a76d24722a2bf8784a132c043063a8f5152d390b0f9/xxhash-3.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:757320d45d2fbcce8f30c42a6b2f47862967aea7bf458b9625b4bbe7ee390392", size = 31542, upload-time = "2025-10-02T14:35:15.21Z" },
+    { url = "https://files.pythonhosted.org/packages/39/c5/cc01e4f6188656e56112d6a8e0dfe298a16934b8c47a247236549a3f7695/xxhash-3.6.0-cp313-cp313-win_arm64.whl", hash = "sha256:457b8f85dec5825eed7b69c11ae86834a018b8e3df5e77783c999663da2f96d6", size = 27880, upload-time = "2025-10-02T14:35:16.315Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/30/25e5321c8732759e930c555176d37e24ab84365482d257c3b16362235212/xxhash-3.6.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a42e633d75cdad6d625434e3468126c73f13f7584545a9cf34e883aa1710e702", size = 32956, upload-time = "2025-10-02T14:35:17.413Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/3c/0573299560d7d9f8ab1838f1efc021a280b5ae5ae2e849034ef3dee18810/xxhash-3.6.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:568a6d743219e717b07b4e03b0a828ce593833e498c3b64752e0f5df6bfe84db", size = 31072, upload-time = "2025-10-02T14:35:18.844Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/1c/52d83a06e417cd9d4137722693424885cc9878249beb3a7c829e74bf7ce9/xxhash-3.6.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bec91b562d8012dae276af8025a55811b875baace6af510412a5e58e3121bc54", size = 196409, upload-time = "2025-10-02T14:35:20.31Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/8e/c6d158d12a79bbd0b878f8355432075fc82759e356ab5a111463422a239b/xxhash-3.6.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78e7f2f4c521c30ad5e786fdd6bae89d47a32672a80195467b5de0480aa97b1f", size = 215736, upload-time = "2025-10-02T14:35:21.616Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/68/c4c80614716345d55071a396cf03d06e34b5f4917a467faf43083c995155/xxhash-3.6.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ed0df1b11a79856df5ffcab572cbd6b9627034c1c748c5566fa79df9048a7c5", size = 214833, upload-time = "2025-10-02T14:35:23.32Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/e9/ae27c8ffec8b953efa84c7c4a6c6802c263d587b9fc0d6e7cea64e08c3af/xxhash-3.6.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e4edbfc7d420925b0dd5e792478ed393d6e75ff8fc219a6546fb446b6a417b1", size = 448348, upload-time = "2025-10-02T14:35:25.111Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/6b/33e21afb1b5b3f46b74b6bd1913639066af218d704cc0941404ca717fc57/xxhash-3.6.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fba27a198363a7ef87f8c0f6b171ec36b674fe9053742c58dd7e3201c1ab30ee", size = 196070, upload-time = "2025-10-02T14:35:26.586Z" },
+    { url = "https://files.pythonhosted.org/packages/96/b6/fcabd337bc5fa624e7203aa0fa7d0c49eed22f72e93229431752bddc83d9/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:794fe9145fe60191c6532fa95063765529770edcdd67b3d537793e8004cabbfd", size = 212907, upload-time = "2025-10-02T14:35:28.087Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/d3/9ee6160e644d660fcf176c5825e61411c7f62648728f69c79ba237250143/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:6105ef7e62b5ac73a837778efc331a591d8442f8ef5c7e102376506cb4ae2729", size = 200839, upload-time = "2025-10-02T14:35:29.857Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/98/e8de5baa5109394baf5118f5e72ab21a86387c4f89b0e77ef3e2f6b0327b/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:f01375c0e55395b814a679b3eea205db7919ac2af213f4a6682e01220e5fe292", size = 213304, upload-time = "2025-10-02T14:35:31.222Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/1d/71056535dec5c3177eeb53e38e3d367dd1d16e024e63b1cee208d572a033/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:d706dca2d24d834a4661619dcacf51a75c16d65985718d6a7d73c1eeeb903ddf", size = 416930, upload-time = "2025-10-02T14:35:32.517Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/6c/5cbde9de2cd967c322e651c65c543700b19e7ae3e0aae8ece3469bf9683d/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5f059d9faeacd49c0215d66f4056e1326c80503f51a1532ca336a385edadd033", size = 193787, upload-time = "2025-10-02T14:35:33.827Z" },
+    { url = "https://files.pythonhosted.org/packages/19/fa/0172e350361d61febcea941b0cc541d6e6c8d65d153e85f850a7b256ff8a/xxhash-3.6.0-cp313-cp313t-win32.whl", hash = "sha256:1244460adc3a9be84731d72b8e80625788e5815b68da3da8b83f78115a40a7ec", size = 30916, upload-time = "2025-10-02T14:35:35.107Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/e6/e8cf858a2b19d6d45820f072eff1bea413910592ff17157cabc5f1227a16/xxhash-3.6.0-cp313-cp313t-win_amd64.whl", hash = "sha256:b1e420ef35c503869c4064f4a2f2b08ad6431ab7b229a05cce39d74268bca6b8", size = 31799, upload-time = "2025-10-02T14:35:36.165Z" },
+    { url = "https://files.pythonhosted.org/packages/56/15/064b197e855bfb7b343210e82490ae672f8bc7cdf3ddb02e92f64304ee8a/xxhash-3.6.0-cp313-cp313t-win_arm64.whl", hash = "sha256:ec44b73a4220623235f67a996c862049f375df3b1052d9899f40a6382c32d746", size = 28044, upload-time = "2025-10-02T14:35:37.195Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/5e/0138bc4484ea9b897864d59fce9be9086030825bc778b76cb5a33a906d37/xxhash-3.6.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a40a3d35b204b7cc7643cbcf8c9976d818cb47befcfac8bbefec8038ac363f3e", size = 32754, upload-time = "2025-10-02T14:35:38.245Z" },
+    { url = "https://files.pythonhosted.org/packages/18/d7/5dac2eb2ec75fd771957a13e5dda560efb2176d5203f39502a5fc571f899/xxhash-3.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a54844be970d3fc22630b32d515e79a90d0a3ddb2644d8d7402e3c4c8da61405", size = 30846, upload-time = "2025-10-02T14:35:39.6Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/71/8bc5be2bb00deb5682e92e8da955ebe5fa982da13a69da5a40a4c8db12fb/xxhash-3.6.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:016e9190af8f0a4e3741343777710e3d5717427f175adfdc3e72508f59e2a7f3", size = 194343, upload-time = "2025-10-02T14:35:40.69Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/3b/52badfb2aecec2c377ddf1ae75f55db3ba2d321c5e164f14461c90837ef3/xxhash-3.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f6f72232f849eb9d0141e2ebe2677ece15adfd0fa599bc058aad83c714bb2c6", size = 213074, upload-time = "2025-10-02T14:35:42.29Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/2b/ae46b4e9b92e537fa30d03dbc19cdae57ed407e9c26d163895e968e3de85/xxhash-3.6.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63275a8aba7865e44b1813d2177e0f5ea7eadad3dd063a21f7cf9afdc7054063", size = 212388, upload-time = "2025-10-02T14:35:43.929Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/80/49f88d3afc724b4ac7fbd664c8452d6db51b49915be48c6982659e0e7942/xxhash-3.6.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cd01fa2aa00d8b017c97eb46b9a794fbdca53fc14f845f5a328c71254b0abb7", size = 445614, upload-time = "2025-10-02T14:35:45.216Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/ba/603ce3961e339413543d8cd44f21f2c80e2a7c5cfe692a7b1f2cccf58f3c/xxhash-3.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0226aa89035b62b6a86d3c68df4d7c1f47a342b8683da2b60cedcddb46c4d95b", size = 194024, upload-time = "2025-10-02T14:35:46.959Z" },
+    { url = "https://files.pythonhosted.org/packages/78/d1/8e225ff7113bf81545cfdcd79eef124a7b7064a0bba53605ff39590b95c2/xxhash-3.6.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c6e193e9f56e4ca4923c61238cdaced324f0feac782544eb4c6d55ad5cc99ddd", size = 210541, upload-time = "2025-10-02T14:35:48.301Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/58/0f89d149f0bad89def1a8dd38feb50ccdeb643d9797ec84707091d4cb494/xxhash-3.6.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9176dcaddf4ca963d4deb93866d739a343c01c969231dbe21680e13a5d1a5bf0", size = 198305, upload-time = "2025-10-02T14:35:49.584Z" },
+    { url = "https://files.pythonhosted.org/packages/11/38/5eab81580703c4df93feb5f32ff8fa7fe1e2c51c1f183ee4e48d4bb9d3d7/xxhash-3.6.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c1ce4009c97a752e682b897aa99aef84191077a9433eb237774689f14f8ec152", size = 210848, upload-time = "2025-10-02T14:35:50.877Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/6b/953dc4b05c3ce678abca756416e4c130d2382f877a9c30a20d08ee6a77c0/xxhash-3.6.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:8cb2f4f679b01513b7adbb9b1b2f0f9cdc31b70007eaf9d59d0878809f385b11", size = 414142, upload-time = "2025-10-02T14:35:52.15Z" },
+    { url = "https://files.pythonhosted.org/packages/08/a9/238ec0d4e81a10eb5026d4a6972677cbc898ba6c8b9dbaec12ae001b1b35/xxhash-3.6.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:653a91d7c2ab54a92c19ccf43508b6a555440b9be1bc8be553376778be7f20b5", size = 191547, upload-time = "2025-10-02T14:35:53.547Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/ee/3cf8589e06c2164ac77c3bf0aa127012801128f1feebf2a079272da5737c/xxhash-3.6.0-cp314-cp314-win32.whl", hash = "sha256:a756fe893389483ee8c394d06b5ab765d96e68fbbfe6fde7aa17e11f5720559f", size = 31214, upload-time = "2025-10-02T14:35:54.746Z" },
+    { url = "https://files.pythonhosted.org/packages/02/5d/a19552fbc6ad4cb54ff953c3908bbc095f4a921bc569433d791f755186f1/xxhash-3.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:39be8e4e142550ef69629c9cd71b88c90e9a5db703fecbcf265546d9536ca4ad", size = 32290, upload-time = "2025-10-02T14:35:55.791Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/11/dafa0643bc30442c887b55baf8e73353a344ee89c1901b5a5c54a6c17d39/xxhash-3.6.0-cp314-cp314-win_arm64.whl", hash = "sha256:25915e6000338999236f1eb68a02a32c3275ac338628a7eaa5a269c401995679", size = 28795, upload-time = "2025-10-02T14:35:57.162Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/db/0e99732ed7f64182aef4a6fb145e1a295558deec2a746265dcdec12d191e/xxhash-3.6.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c5294f596a9017ca5a3e3f8884c00b91ab2ad2933cf288f4923c3fd4346cf3d4", size = 32955, upload-time = "2025-10-02T14:35:58.267Z" },
+    { url = "https://files.pythonhosted.org/packages/55/f4/2a7c3c68e564a099becfa44bb3d398810cc0ff6749b0d3cb8ccb93f23c14/xxhash-3.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1cf9dcc4ab9cff01dfbba78544297a3a01dafd60f3bde4e2bfd016cf7e4ddc67", size = 31072, upload-time = "2025-10-02T14:35:59.382Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/d9/72a29cddc7250e8a5819dad5d466facb5dc4c802ce120645630149127e73/xxhash-3.6.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:01262da8798422d0685f7cef03b2bd3f4f46511b02830861df548d7def4402ad", size = 196579, upload-time = "2025-10-02T14:36:00.838Z" },
+    { url = "https://files.pythonhosted.org/packages/63/93/b21590e1e381040e2ca305a884d89e1c345b347404f7780f07f2cdd47ef4/xxhash-3.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51a73fb7cb3a3ead9f7a8b583ffd9b8038e277cdb8cb87cf890e88b3456afa0b", size = 215854, upload-time = "2025-10-02T14:36:02.207Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/b8/edab8a7d4fa14e924b29be877d54155dcbd8b80be85ea00d2be3413a9ed4/xxhash-3.6.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b9c6df83594f7df8f7f708ce5ebeacfc69f72c9fbaaababf6cf4758eaada0c9b", size = 214965, upload-time = "2025-10-02T14:36:03.507Z" },
+    { url = "https://files.pythonhosted.org/packages/27/67/dfa980ac7f0d509d54ea0d5a486d2bb4b80c3f1bb22b66e6a05d3efaf6c0/xxhash-3.6.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:627f0af069b0ea56f312fd5189001c24578868643203bca1abbc2c52d3a6f3ca", size = 448484, upload-time = "2025-10-02T14:36:04.828Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/63/8ffc2cc97e811c0ca5d00ab36604b3ea6f4254f20b7bc658ca825ce6c954/xxhash-3.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa912c62f842dfd013c5f21a642c9c10cd9f4c4e943e0af83618b4a404d9091a", size = 196162, upload-time = "2025-10-02T14:36:06.182Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/77/07f0e7a3edd11a6097e990f6e5b815b6592459cb16dae990d967693e6ea9/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b465afd7909db30168ab62afe40b2fcf79eedc0b89a6c0ab3123515dc0df8b99", size = 213007, upload-time = "2025-10-02T14:36:07.733Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/d8/bc5fa0d152837117eb0bef6f83f956c509332ce133c91c63ce07ee7c4873/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:a881851cf38b0a70e7c4d3ce81fc7afd86fbc2a024f4cfb2a97cf49ce04b75d3", size = 200956, upload-time = "2025-10-02T14:36:09.106Z" },
+    { url = "https://files.pythonhosted.org/packages/26/a5/d749334130de9411783873e9b98ecc46688dad5db64ca6e04b02acc8b473/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:9b3222c686a919a0f3253cfc12bb118b8b103506612253b5baeaac10d8027cf6", size = 213401, upload-time = "2025-10-02T14:36:10.585Z" },
+    { url = "https://files.pythonhosted.org/packages/89/72/abed959c956a4bfc72b58c0384bb7940663c678127538634d896b1195c10/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:c5aa639bc113e9286137cec8fadc20e9cd732b2cc385c0b7fa673b84fc1f2a93", size = 417083, upload-time = "2025-10-02T14:36:12.276Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/b3/62fd2b586283b7d7d665fb98e266decadf31f058f1cf6c478741f68af0cb/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5c1343d49ac102799905e115aee590183c3921d475356cb24b4de29a4bc56518", size = 193913, upload-time = "2025-10-02T14:36:14.025Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/9a/c19c42c5b3f5a4aad748a6d5b4f23df3bed7ee5445accc65a0fb3ff03953/xxhash-3.6.0-cp314-cp314t-win32.whl", hash = "sha256:5851f033c3030dd95c086b4a36a2683c2ff4a799b23af60977188b057e467119", size = 31586, upload-time = "2025-10-02T14:36:15.603Z" },
+    { url = "https://files.pythonhosted.org/packages/03/d6/4cc450345be9924fd5dc8c590ceda1db5b43a0a889587b0ae81a95511360/xxhash-3.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0444e7967dac37569052d2409b00a8860c2135cff05502df4da80267d384849f", size = 32526, upload-time = "2025-10-02T14:36:16.708Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/c9/7243eb3f9eaabd1a88a5a5acadf06df2d83b100c62684b7425c6a11bcaa8/xxhash-3.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:bb79b1e63f6fd84ec778a4b1916dfe0a7c3fdb986c06addd5db3a0d413819d95", size = 28898, upload-time = "2025-10-02T14:36:17.843Z" },
+    { url = "https://files.pythonhosted.org/packages/03/ff/1b4bb3f397552116c1df6266c1b83a21aeeb26061ab1f462984b499a3870/xxhash-3.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cc604dc06027dbeb8281aeac5899c35fcfe7c77b25212833709f0bff4ce74d2a", size = 32844, upload-time = "2025-10-02T14:36:39.157Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/db/27146d0bee4346a9a31f7b498a81fc02747f6f1e6c52a2e7989504278051/xxhash-3.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:277175a73900ad43a8caeb8b99b9604f21fe8d7c842f2f9061a364a7e220ddb7", size = 30806, upload-time = "2025-10-02T14:36:40.621Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/2b/4896188df564908817a75de19bf7f2384b99a75af2d528f9c49326f76458/xxhash-3.6.0-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cfbc5b91397c8c2972fdac13fb3e4ed2f7f8ccac85cd2c644887557780a9b6e2", size = 193448, upload-time = "2025-10-02T14:36:41.797Z" },
+    { url = "https://files.pythonhosted.org/packages/51/c5/be8953f62e772340319a826ce1e07489935600089756cf83b628cd36ebe3/xxhash-3.6.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2762bfff264c4e73c0e507274b40634ff465e025f0eaf050897e88ec8367575d", size = 212547, upload-time = "2025-10-02T14:36:43.581Z" },
+    { url = "https://files.pythonhosted.org/packages/51/1a/1e9f0b911d1cf00dd537c074ae3fae15b535a7f0d9e7edd42a9d2c4f78ce/xxhash-3.6.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f171a900d59d51511209f7476933c34a0c2c711078d3c80e74e0fe4f38680ec", size = 211309, upload-time = "2025-10-02T14:36:45.307Z" },
+    { url = "https://files.pythonhosted.org/packages/63/88/b284c6a128d88dc47f201957f926e707db79fb7415a87072e15c0e490de0/xxhash-3.6.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:780b90c313348f030b811efc37b0fa1431163cb8db8064cf88a7936b6ce5f222", size = 444480, upload-time = "2025-10-02T14:36:47.226Z" },
+    { url = "https://files.pythonhosted.org/packages/87/e4/798293a2bf9e4fac5f6d53ce59cba4739930778dfc6c7c73f40044ab0e6e/xxhash-3.6.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b242455eccdfcd1fa4134c431a30737d2b4f045770f8fe84356b3469d4b919", size = 192957, upload-time = "2025-10-02T14:36:48.968Z" },
+    { url = "https://files.pythonhosted.org/packages/78/55/bfd0d7db447a927897469048b953caececa3532e743b940dd1f5c1032d24/xxhash-3.6.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a75ffc1bd5def584129774c158e108e5d768e10b75813f2b32650bb041066ed6", size = 209850, upload-time = "2025-10-02T14:36:50.258Z" },
+    { url = "https://files.pythonhosted.org/packages/31/06/d08ef9a792bfebfd2fb2bcbf04a541ad283bef74749ead6f089a0809d288/xxhash-3.6.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1fc1ed882d1e8df932a66e2999429ba6cc4d5172914c904ab193381fba825360", size = 197342, upload-time = "2025-10-02T14:36:51.651Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/1a/aebf90797c94e9ca407c28e23f54d71f7149d91a93406a08a09e44d06994/xxhash-3.6.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:44e342e8cc11b4e79dae5c57f2fb6360c3c20cc57d32049af8f567f5b4bcb5f4", size = 209757, upload-time = "2025-10-02T14:36:53.009Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/80/799eec3d0a144dc3edf8c19b4f139c27fb923c50b34352796089ca206429/xxhash-3.6.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:c2f9ccd5c4be370939a2e17602fbc49995299203da72a3429db013d44d590e86", size = 412773, upload-time = "2025-10-02T14:36:54.691Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/f9/09df7545699de09219a205123b8463ce9ea83f48acc7aeeba0269507f9d3/xxhash-3.6.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:02ea4cb627c76f48cd9fb37cf7ab22bd51e57e1b519807234b473faebe526796", size = 190357, upload-time = "2025-10-02T14:36:56.363Z" },
+    { url = "https://files.pythonhosted.org/packages/07/40/2f8327f94e64a3f34d6ce3347c55207c322abbc80ae486ea45df4c62e7b3/xxhash-3.6.0-cp39-cp39-win32.whl", hash = "sha256:6551880383f0e6971dc23e512c9ccc986147ce7bfa1cd2e4b520b876c53e9f3d", size = 30585, upload-time = "2025-10-02T14:36:57.664Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/c8/2ecbc6799be9c02e8bf7b5a66cd94832b6ac13d59808746f0d402481c6ad/xxhash-3.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:7c35c4cdc65f2a29f34425c446f2f5cdcd0e3c34158931e1cc927ece925ab802", size = 31512, upload-time = "2025-10-02T14:36:58.837Z" },
+    { url = "https://files.pythonhosted.org/packages/19/94/1d5459a9c587c94d7b8bcc710bd08bbfa145cbd814ebde41b48494362a21/xxhash-3.6.0-cp39-cp39-win_arm64.whl", hash = "sha256:ffc578717a347baf25be8397cb10d2528802d24f94cfc005c0e44fef44b5cdd6", size = 27878, upload-time = "2025-10-02T14:37:00.201Z" },
+    { url = "https://files.pythonhosted.org/packages/93/1e/8aec23647a34a249f62e2398c42955acd9b4c6ed5cf08cbea94dc46f78d2/xxhash-3.6.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0f7b7e2ec26c1666ad5fc9dbfa426a6a3367ceaf79db5dd76264659d509d73b0", size = 30662, upload-time = "2025-10-02T14:37:01.743Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/0b/b14510b38ba91caf43006209db846a696ceea6a847a0c9ba0a5b1adc53d6/xxhash-3.6.0-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5dc1e14d14fa0f5789ec29a7062004b5933964bb9b02aae6622b8f530dc40296", size = 41056, upload-time = "2025-10-02T14:37:02.879Z" },
+    { url = "https://files.pythonhosted.org/packages/50/55/15a7b8a56590e66ccd374bbfa3f9ffc45b810886c8c3b614e3f90bd2367c/xxhash-3.6.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:881b47fc47e051b37d94d13e7455131054b56749b91b508b0907eb07900d1c13", size = 36251, upload-time = "2025-10-02T14:37:04.44Z" },
+    { url = "https://files.pythonhosted.org/packages/62/b2/5ac99a041a29e58e95f907876b04f7067a0242cb85b5f39e726153981503/xxhash-3.6.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6dc31591899f5e5666f04cc2e529e69b4072827085c1ef15294d91a004bc1bd", size = 32481, upload-time = "2025-10-02T14:37:05.869Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/d9/8d95e906764a386a3d3b596f3c68bb63687dfca806373509f51ce8eea81f/xxhash-3.6.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:15e0dac10eb9309508bfc41f7f9deaa7755c69e35af835db9cb10751adebc35d", size = 31565, upload-time = "2025-10-02T14:37:06.966Z" },
 ]
 
 [[package]]
@@ -2966,96 +3561,122 @@ name = "yarl"
 version = "1.20.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "idna" },
-    { name = "multidict" },
-    { name = "propcache" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/3c/fb/efaa23fa4e45537b827620f04cf8f3cd658b76642205162e072703a5b963/yarl-1.20.1.tar.gz", hash = "sha256:d017a4997ee50c91fd5466cef416231bb82177b93b029906cefc542ce14c35ac", size = 186428 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/cb/65/7fed0d774abf47487c64be14e9223749468922817b5e8792b8a64792a1bb/yarl-1.20.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6032e6da6abd41e4acda34d75a816012717000fa6839f37124a47fcefc49bec4", size = 132910 },
-    { url = "https://files.pythonhosted.org/packages/8a/7b/988f55a52da99df9e56dc733b8e4e5a6ae2090081dc2754fc8fd34e60aa0/yarl-1.20.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2c7b34d804b8cf9b214f05015c4fee2ebe7ed05cf581e7192c06555c71f4446a", size = 90644 },
-    { url = "https://files.pythonhosted.org/packages/f7/de/30d98f03e95d30c7e3cc093759982d038c8833ec2451001d45ef4854edc1/yarl-1.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c869f2651cc77465f6cd01d938d91a11d9ea5d798738c1dc077f3de0b5e5fed", size = 89322 },
-    { url = "https://files.pythonhosted.org/packages/e0/7a/f2f314f5ebfe9200724b0b748de2186b927acb334cf964fd312eb86fc286/yarl-1.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62915e6688eb4d180d93840cda4110995ad50c459bf931b8b3775b37c264af1e", size = 323786 },
-    { url = "https://files.pythonhosted.org/packages/15/3f/718d26f189db96d993d14b984ce91de52e76309d0fd1d4296f34039856aa/yarl-1.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41ebd28167bc6af8abb97fec1a399f412eec5fd61a3ccbe2305a18b84fb4ca73", size = 319627 },
-    { url = "https://files.pythonhosted.org/packages/a5/76/8fcfbf5fa2369157b9898962a4a7d96764b287b085b5b3d9ffae69cdefd1/yarl-1.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21242b4288a6d56f04ea193adde174b7e347ac46ce6bc84989ff7c1b1ecea84e", size = 339149 },
-    { url = "https://files.pythonhosted.org/packages/3c/95/d7fc301cc4661785967acc04f54a4a42d5124905e27db27bb578aac49b5c/yarl-1.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bea21cdae6c7eb02ba02a475f37463abfe0a01f5d7200121b03e605d6a0439f8", size = 333327 },
-    { url = "https://files.pythonhosted.org/packages/65/94/e21269718349582eee81efc5c1c08ee71c816bfc1585b77d0ec3f58089eb/yarl-1.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f8a891e4a22a89f5dde7862994485e19db246b70bb288d3ce73a34422e55b23", size = 326054 },
-    { url = "https://files.pythonhosted.org/packages/32/ae/8616d1f07853704523519f6131d21f092e567c5af93de7e3e94b38d7f065/yarl-1.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd803820d44c8853a109a34e3660e5a61beae12970da479cf44aa2954019bf70", size = 315035 },
-    { url = "https://files.pythonhosted.org/packages/48/aa/0ace06280861ef055855333707db5e49c6e3a08840a7ce62682259d0a6c0/yarl-1.20.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b982fa7f74c80d5c0c7b5b38f908971e513380a10fecea528091405f519b9ebb", size = 338962 },
-    { url = "https://files.pythonhosted.org/packages/20/52/1e9d0e6916f45a8fb50e6844f01cb34692455f1acd548606cbda8134cd1e/yarl-1.20.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:33f29ecfe0330c570d997bcf1afd304377f2e48f61447f37e846a6058a4d33b2", size = 335399 },
-    { url = "https://files.pythonhosted.org/packages/f2/65/60452df742952c630e82f394cd409de10610481d9043aa14c61bf846b7b1/yarl-1.20.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:835ab2cfc74d5eb4a6a528c57f05688099da41cf4957cf08cad38647e4a83b30", size = 338649 },
-    { url = "https://files.pythonhosted.org/packages/7b/f5/6cd4ff38dcde57a70f23719a838665ee17079640c77087404c3d34da6727/yarl-1.20.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:46b5e0ccf1943a9a6e766b2c2b8c732c55b34e28be57d8daa2b3c1d1d4009309", size = 358563 },
-    { url = "https://files.pythonhosted.org/packages/d1/90/c42eefd79d0d8222cb3227bdd51b640c0c1d0aa33fe4cc86c36eccba77d3/yarl-1.20.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:df47c55f7d74127d1b11251fe6397d84afdde0d53b90bedb46a23c0e534f9d24", size = 357609 },
-    { url = "https://files.pythonhosted.org/packages/03/c8/cea6b232cb4617514232e0f8a718153a95b5d82b5290711b201545825532/yarl-1.20.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:76d12524d05841276b0e22573f28d5fbcb67589836772ae9244d90dd7d66aa13", size = 350224 },
-    { url = "https://files.pythonhosted.org/packages/ce/a3/eaa0ab9712f1f3d01faf43cf6f1f7210ce4ea4a7e9b28b489a2261ca8db9/yarl-1.20.1-cp310-cp310-win32.whl", hash = "sha256:6c4fbf6b02d70e512d7ade4b1f998f237137f1417ab07ec06358ea04f69134f8", size = 81753 },
-    { url = "https://files.pythonhosted.org/packages/8f/34/e4abde70a9256465fe31c88ed02c3f8502b7b5dead693a4f350a06413f28/yarl-1.20.1-cp310-cp310-win_amd64.whl", hash = "sha256:aef6c4d69554d44b7f9d923245f8ad9a707d971e6209d51279196d8e8fe1ae16", size = 86817 },
-    { url = "https://files.pythonhosted.org/packages/b1/18/893b50efc2350e47a874c5c2d67e55a0ea5df91186b2a6f5ac52eff887cd/yarl-1.20.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:47ee6188fea634bdfaeb2cc420f5b3b17332e6225ce88149a17c413c77ff269e", size = 133833 },
-    { url = "https://files.pythonhosted.org/packages/89/ed/b8773448030e6fc47fa797f099ab9eab151a43a25717f9ac043844ad5ea3/yarl-1.20.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d0f6500f69e8402d513e5eedb77a4e1818691e8f45e6b687147963514d84b44b", size = 91070 },
-    { url = "https://files.pythonhosted.org/packages/e3/e3/409bd17b1e42619bf69f60e4f031ce1ccb29bd7380117a55529e76933464/yarl-1.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a8900a42fcdaad568de58887c7b2f602962356908eedb7628eaf6021a6e435b", size = 89818 },
-    { url = "https://files.pythonhosted.org/packages/f8/77/64d8431a4d77c856eb2d82aa3de2ad6741365245a29b3a9543cd598ed8c5/yarl-1.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bad6d131fda8ef508b36be3ece16d0902e80b88ea7200f030a0f6c11d9e508d4", size = 347003 },
-    { url = "https://files.pythonhosted.org/packages/8d/d2/0c7e4def093dcef0bd9fa22d4d24b023788b0a33b8d0088b51aa51e21e99/yarl-1.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:df018d92fe22aaebb679a7f89fe0c0f368ec497e3dda6cb81a567610f04501f1", size = 336537 },
-    { url = "https://files.pythonhosted.org/packages/f0/f3/fc514f4b2cf02cb59d10cbfe228691d25929ce8f72a38db07d3febc3f706/yarl-1.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f969afbb0a9b63c18d0feecf0db09d164b7a44a053e78a7d05f5df163e43833", size = 362358 },
-    { url = "https://files.pythonhosted.org/packages/ea/6d/a313ac8d8391381ff9006ac05f1d4331cee3b1efaa833a53d12253733255/yarl-1.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:812303eb4aa98e302886ccda58d6b099e3576b1b9276161469c25803a8db277d", size = 357362 },
-    { url = "https://files.pythonhosted.org/packages/00/70/8f78a95d6935a70263d46caa3dd18e1f223cf2f2ff2037baa01a22bc5b22/yarl-1.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98c4a7d166635147924aa0bf9bfe8d8abad6fffa6102de9c99ea04a1376f91e8", size = 348979 },
-    { url = "https://files.pythonhosted.org/packages/cb/05/42773027968968f4f15143553970ee36ead27038d627f457cc44bbbeecf3/yarl-1.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12e768f966538e81e6e7550f9086a6236b16e26cd964cf4df35349970f3551cf", size = 337274 },
-    { url = "https://files.pythonhosted.org/packages/05/be/665634aa196954156741ea591d2f946f1b78ceee8bb8f28488bf28c0dd62/yarl-1.20.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe41919b9d899661c5c28a8b4b0acf704510b88f27f0934ac7a7bebdd8938d5e", size = 363294 },
-    { url = "https://files.pythonhosted.org/packages/eb/90/73448401d36fa4e210ece5579895731f190d5119c4b66b43b52182e88cd5/yarl-1.20.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:8601bc010d1d7780592f3fc1bdc6c72e2b6466ea34569778422943e1a1f3c389", size = 358169 },
-    { url = "https://files.pythonhosted.org/packages/c3/b0/fce922d46dc1eb43c811f1889f7daa6001b27a4005587e94878570300881/yarl-1.20.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:daadbdc1f2a9033a2399c42646fbd46da7992e868a5fe9513860122d7fe7a73f", size = 362776 },
-    { url = "https://files.pythonhosted.org/packages/f1/0d/b172628fce039dae8977fd22caeff3eeebffd52e86060413f5673767c427/yarl-1.20.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:03aa1e041727cb438ca762628109ef1333498b122e4c76dd858d186a37cec845", size = 381341 },
-    { url = "https://files.pythonhosted.org/packages/6b/9b/5b886d7671f4580209e855974fe1cecec409aa4a89ea58b8f0560dc529b1/yarl-1.20.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:642980ef5e0fa1de5fa96d905c7e00cb2c47cb468bfcac5a18c58e27dbf8d8d1", size = 379988 },
-    { url = "https://files.pythonhosted.org/packages/73/be/75ef5fd0fcd8f083a5d13f78fd3f009528132a1f2a1d7c925c39fa20aa79/yarl-1.20.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:86971e2795584fe8c002356d3b97ef6c61862720eeff03db2a7c86b678d85b3e", size = 371113 },
-    { url = "https://files.pythonhosted.org/packages/50/4f/62faab3b479dfdcb741fe9e3f0323e2a7d5cd1ab2edc73221d57ad4834b2/yarl-1.20.1-cp311-cp311-win32.whl", hash = "sha256:597f40615b8d25812f14562699e287f0dcc035d25eb74da72cae043bb884d773", size = 81485 },
-    { url = "https://files.pythonhosted.org/packages/f0/09/d9c7942f8f05c32ec72cd5c8e041c8b29b5807328b68b4801ff2511d4d5e/yarl-1.20.1-cp311-cp311-win_amd64.whl", hash = "sha256:26ef53a9e726e61e9cd1cda6b478f17e350fb5800b4bd1cd9fe81c4d91cfeb2e", size = 86686 },
-    { url = "https://files.pythonhosted.org/packages/5f/9a/cb7fad7d73c69f296eda6815e4a2c7ed53fc70c2f136479a91c8e5fbdb6d/yarl-1.20.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdcc4cd244e58593a4379fe60fdee5ac0331f8eb70320a24d591a3be197b94a9", size = 133667 },
-    { url = "https://files.pythonhosted.org/packages/67/38/688577a1cb1e656e3971fb66a3492501c5a5df56d99722e57c98249e5b8a/yarl-1.20.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b29a2c385a5f5b9c7d9347e5812b6f7ab267193c62d282a540b4fc528c8a9d2a", size = 91025 },
-    { url = "https://files.pythonhosted.org/packages/50/ec/72991ae51febeb11a42813fc259f0d4c8e0507f2b74b5514618d8b640365/yarl-1.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1112ae8154186dfe2de4732197f59c05a83dc814849a5ced892b708033f40dc2", size = 89709 },
-    { url = "https://files.pythonhosted.org/packages/99/da/4d798025490e89426e9f976702e5f9482005c548c579bdae792a4c37769e/yarl-1.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90bbd29c4fe234233f7fa2b9b121fb63c321830e5d05b45153a2ca68f7d310ee", size = 352287 },
-    { url = "https://files.pythonhosted.org/packages/1a/26/54a15c6a567aac1c61b18aa0f4b8aa2e285a52d547d1be8bf48abe2b3991/yarl-1.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:680e19c7ce3710ac4cd964e90dad99bf9b5029372ba0c7cbfcd55e54d90ea819", size = 345429 },
-    { url = "https://files.pythonhosted.org/packages/d6/95/9dcf2386cb875b234353b93ec43e40219e14900e046bf6ac118f94b1e353/yarl-1.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a979218c1fdb4246a05efc2cc23859d47c89af463a90b99b7c56094daf25a16", size = 365429 },
-    { url = "https://files.pythonhosted.org/packages/91/b2/33a8750f6a4bc224242a635f5f2cff6d6ad5ba651f6edcccf721992c21a0/yarl-1.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255b468adf57b4a7b65d8aad5b5138dce6a0752c139965711bdcb81bc370e1b6", size = 363862 },
-    { url = "https://files.pythonhosted.org/packages/98/28/3ab7acc5b51f4434b181b0cee8f1f4b77a65919700a355fb3617f9488874/yarl-1.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a97d67108e79cfe22e2b430d80d7571ae57d19f17cda8bb967057ca8a7bf5bfd", size = 355616 },
-    { url = "https://files.pythonhosted.org/packages/36/a3/f666894aa947a371724ec7cd2e5daa78ee8a777b21509b4252dd7bd15e29/yarl-1.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8570d998db4ddbfb9a590b185a0a33dbf8aafb831d07a5257b4ec9948df9cb0a", size = 339954 },
-    { url = "https://files.pythonhosted.org/packages/f1/81/5f466427e09773c04219d3450d7a1256138a010b6c9f0af2d48565e9ad13/yarl-1.20.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97c75596019baae7c71ccf1d8cc4738bc08134060d0adfcbe5642f778d1dca38", size = 365575 },
-    { url = "https://files.pythonhosted.org/packages/2e/e3/e4b0ad8403e97e6c9972dd587388940a032f030ebec196ab81a3b8e94d31/yarl-1.20.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1c48912653e63aef91ff988c5432832692ac5a1d8f0fb8a33091520b5bbe19ef", size = 365061 },
-    { url = "https://files.pythonhosted.org/packages/ac/99/b8a142e79eb86c926f9f06452eb13ecb1bb5713bd01dc0038faf5452e544/yarl-1.20.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4c3ae28f3ae1563c50f3d37f064ddb1511ecc1d5584e88c6b7c63cf7702a6d5f", size = 364142 },
-    { url = "https://files.pythonhosted.org/packages/34/f2/08ed34a4a506d82a1a3e5bab99ccd930a040f9b6449e9fd050320e45845c/yarl-1.20.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e9642f27036283550f5f57dc6156c51084b458570b9d0d96100c8bebb186a8", size = 381894 },
-    { url = "https://files.pythonhosted.org/packages/92/f8/9a3fbf0968eac704f681726eff595dce9b49c8a25cd92bf83df209668285/yarl-1.20.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2c26b0c49220d5799f7b22c6838409ee9bc58ee5c95361a4d7831f03cc225b5a", size = 383378 },
-    { url = "https://files.pythonhosted.org/packages/af/85/9363f77bdfa1e4d690957cd39d192c4cacd1c58965df0470a4905253b54f/yarl-1.20.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564ab3d517e3d01c408c67f2e5247aad4019dcf1969982aba3974b4093279004", size = 374069 },
-    { url = "https://files.pythonhosted.org/packages/35/99/9918c8739ba271dcd935400cff8b32e3cd319eaf02fcd023d5dcd487a7c8/yarl-1.20.1-cp312-cp312-win32.whl", hash = "sha256:daea0d313868da1cf2fac6b2d3a25c6e3a9e879483244be38c8e6a41f1d876a5", size = 81249 },
-    { url = "https://files.pythonhosted.org/packages/eb/83/5d9092950565481b413b31a23e75dd3418ff0a277d6e0abf3729d4d1ce25/yarl-1.20.1-cp312-cp312-win_amd64.whl", hash = "sha256:48ea7d7f9be0487339828a4de0360d7ce0efc06524a48e1810f945c45b813698", size = 86710 },
-    { url = "https://files.pythonhosted.org/packages/8a/e1/2411b6d7f769a07687acee88a062af5833cf1966b7266f3d8dfb3d3dc7d3/yarl-1.20.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:0b5ff0fbb7c9f1b1b5ab53330acbfc5247893069e7716840c8e7d5bb7355038a", size = 131811 },
-    { url = "https://files.pythonhosted.org/packages/b2/27/584394e1cb76fb771371770eccad35de400e7b434ce3142c2dd27392c968/yarl-1.20.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:14f326acd845c2b2e2eb38fb1346c94f7f3b01a4f5c788f8144f9b630bfff9a3", size = 90078 },
-    { url = "https://files.pythonhosted.org/packages/bf/9a/3246ae92d4049099f52d9b0fe3486e3b500e29b7ea872d0f152966fc209d/yarl-1.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f60e4ad5db23f0b96e49c018596707c3ae89f5d0bd97f0ad3684bcbad899f1e7", size = 88748 },
-    { url = "https://files.pythonhosted.org/packages/a3/25/35afe384e31115a1a801fbcf84012d7a066d89035befae7c5d4284df1e03/yarl-1.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:49bdd1b8e00ce57e68ba51916e4bb04461746e794e7c4d4bbc42ba2f18297691", size = 349595 },
-    { url = "https://files.pythonhosted.org/packages/28/2d/8aca6cb2cabc8f12efcb82749b9cefecbccfc7b0384e56cd71058ccee433/yarl-1.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:66252d780b45189975abfed839616e8fd2dbacbdc262105ad7742c6ae58f3e31", size = 342616 },
-    { url = "https://files.pythonhosted.org/packages/0b/e9/1312633d16b31acf0098d30440ca855e3492d66623dafb8e25b03d00c3da/yarl-1.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59174e7332f5d153d8f7452a102b103e2e74035ad085f404df2e40e663a22b28", size = 361324 },
-    { url = "https://files.pythonhosted.org/packages/bc/a0/688cc99463f12f7669eec7c8acc71ef56a1521b99eab7cd3abb75af887b0/yarl-1.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e3968ec7d92a0c0f9ac34d5ecfd03869ec0cab0697c91a45db3fbbd95fe1b653", size = 359676 },
-    { url = "https://files.pythonhosted.org/packages/af/44/46407d7f7a56e9a85a4c207724c9f2c545c060380718eea9088f222ba697/yarl-1.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1a4fbb50e14396ba3d375f68bfe02215d8e7bc3ec49da8341fe3157f59d2ff5", size = 352614 },
-    { url = "https://files.pythonhosted.org/packages/b1/91/31163295e82b8d5485d31d9cf7754d973d41915cadce070491778d9c9825/yarl-1.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11a62c839c3a8eac2410e951301309426f368388ff2f33799052787035793b02", size = 336766 },
-    { url = "https://files.pythonhosted.org/packages/b4/8e/c41a5bc482121f51c083c4c2bcd16b9e01e1cf8729e380273a952513a21f/yarl-1.20.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:041eaa14f73ff5a8986b4388ac6bb43a77f2ea09bf1913df7a35d4646db69e53", size = 364615 },
-    { url = "https://files.pythonhosted.org/packages/e3/5b/61a3b054238d33d70ea06ebba7e58597891b71c699e247df35cc984ab393/yarl-1.20.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:377fae2fef158e8fd9d60b4c8751387b8d1fb121d3d0b8e9b0be07d1b41e83dc", size = 360982 },
-    { url = "https://files.pythonhosted.org/packages/df/a3/6a72fb83f8d478cb201d14927bc8040af901811a88e0ff2da7842dd0ed19/yarl-1.20.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1c92f4390e407513f619d49319023664643d3339bd5e5a56a3bebe01bc67ec04", size = 369792 },
-    { url = "https://files.pythonhosted.org/packages/7c/af/4cc3c36dfc7c077f8dedb561eb21f69e1e9f2456b91b593882b0b18c19dc/yarl-1.20.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d25ddcf954df1754ab0f86bb696af765c5bfaba39b74095f27eececa049ef9a4", size = 382049 },
-    { url = "https://files.pythonhosted.org/packages/19/3a/e54e2c4752160115183a66dc9ee75a153f81f3ab2ba4bf79c3c53b33de34/yarl-1.20.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:909313577e9619dcff8c31a0ea2aa0a2a828341d92673015456b3ae492e7317b", size = 384774 },
-    { url = "https://files.pythonhosted.org/packages/9c/20/200ae86dabfca89060ec6447649f219b4cbd94531e425e50d57e5f5ac330/yarl-1.20.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:793fd0580cb9664548c6b83c63b43c477212c0260891ddf86809e1c06c8b08f1", size = 374252 },
-    { url = "https://files.pythonhosted.org/packages/83/75/11ee332f2f516b3d094e89448da73d557687f7d137d5a0f48c40ff211487/yarl-1.20.1-cp313-cp313-win32.whl", hash = "sha256:468f6e40285de5a5b3c44981ca3a319a4b208ccc07d526b20b12aeedcfa654b7", size = 81198 },
-    { url = "https://files.pythonhosted.org/packages/ba/ba/39b1ecbf51620b40ab402b0fc817f0ff750f6d92712b44689c2c215be89d/yarl-1.20.1-cp313-cp313-win_amd64.whl", hash = "sha256:495b4ef2fea40596bfc0affe3837411d6aa3371abcf31aac0ccc4bdd64d4ef5c", size = 86346 },
-    { url = "https://files.pythonhosted.org/packages/43/c7/669c52519dca4c95153c8ad96dd123c79f354a376346b198f438e56ffeb4/yarl-1.20.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:f60233b98423aab21d249a30eb27c389c14929f47be8430efa7dbd91493a729d", size = 138826 },
-    { url = "https://files.pythonhosted.org/packages/6a/42/fc0053719b44f6ad04a75d7f05e0e9674d45ef62f2d9ad2c1163e5c05827/yarl-1.20.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6f3eff4cc3f03d650d8755c6eefc844edde99d641d0dcf4da3ab27141a5f8ddf", size = 93217 },
-    { url = "https://files.pythonhosted.org/packages/4f/7f/fa59c4c27e2a076bba0d959386e26eba77eb52ea4a0aac48e3515c186b4c/yarl-1.20.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:69ff8439d8ba832d6bed88af2c2b3445977eba9a4588b787b32945871c2444e3", size = 92700 },
-    { url = "https://files.pythonhosted.org/packages/2f/d4/062b2f48e7c93481e88eff97a6312dca15ea200e959f23e96d8ab898c5b8/yarl-1.20.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cf34efa60eb81dd2645a2e13e00bb98b76c35ab5061a3989c7a70f78c85006d", size = 347644 },
-    { url = "https://files.pythonhosted.org/packages/89/47/78b7f40d13c8f62b499cc702fdf69e090455518ae544c00a3bf4afc9fc77/yarl-1.20.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8e0fe9364ad0fddab2688ce72cb7a8e61ea42eff3c7caeeb83874a5d479c896c", size = 323452 },
-    { url = "https://files.pythonhosted.org/packages/eb/2b/490d3b2dc66f52987d4ee0d3090a147ea67732ce6b4d61e362c1846d0d32/yarl-1.20.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f64fbf81878ba914562c672024089e3401974a39767747691c65080a67b18c1", size = 346378 },
-    { url = "https://files.pythonhosted.org/packages/66/ad/775da9c8a94ce925d1537f939a4f17d782efef1f973039d821cbe4bcc211/yarl-1.20.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6342d643bf9a1de97e512e45e4b9560a043347e779a173250824f8b254bd5ce", size = 353261 },
-    { url = "https://files.pythonhosted.org/packages/4b/23/0ed0922b47a4f5c6eb9065d5ff1e459747226ddce5c6a4c111e728c9f701/yarl-1.20.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56dac5f452ed25eef0f6e3c6a066c6ab68971d96a9fb441791cad0efba6140d3", size = 335987 },
-    { url = "https://files.pythonhosted.org/packages/3e/49/bc728a7fe7d0e9336e2b78f0958a2d6b288ba89f25a1762407a222bf53c3/yarl-1.20.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7d7f497126d65e2cad8dc5f97d34c27b19199b6414a40cb36b52f41b79014be", size = 329361 },
-    { url = "https://files.pythonhosted.org/packages/93/8f/b811b9d1f617c83c907e7082a76e2b92b655400e61730cd61a1f67178393/yarl-1.20.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:67e708dfb8e78d8a19169818eeb5c7a80717562de9051bf2413aca8e3696bf16", size = 346460 },
-    { url = "https://files.pythonhosted.org/packages/70/fd/af94f04f275f95da2c3b8b5e1d49e3e79f1ed8b6ceb0f1664cbd902773ff/yarl-1.20.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:595c07bc79af2494365cc96ddeb772f76272364ef7c80fb892ef9d0649586513", size = 334486 },
-    { url = "https://files.pythonhosted.org/packages/84/65/04c62e82704e7dd0a9b3f61dbaa8447f8507655fd16c51da0637b39b2910/yarl-1.20.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7bdd2f80f4a7df852ab9ab49484a4dee8030023aa536df41f2d922fd57bf023f", size = 342219 },
-    { url = "https://files.pythonhosted.org/packages/91/95/459ca62eb958381b342d94ab9a4b6aec1ddec1f7057c487e926f03c06d30/yarl-1.20.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c03bfebc4ae8d862f853a9757199677ab74ec25424d0ebd68a0027e9c639a390", size = 350693 },
-    { url = "https://files.pythonhosted.org/packages/a6/00/d393e82dd955ad20617abc546a8f1aee40534d599ff555ea053d0ec9bf03/yarl-1.20.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:344d1103e9c1523f32a5ed704d576172d2cabed3122ea90b1d4e11fe17c66458", size = 355803 },
-    { url = "https://files.pythonhosted.org/packages/9e/ed/c5fb04869b99b717985e244fd93029c7a8e8febdfcffa06093e32d7d44e7/yarl-1.20.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:88cab98aa4e13e1ade8c141daeedd300a4603b7132819c484841bb7af3edce9e", size = 341709 },
-    { url = "https://files.pythonhosted.org/packages/24/fd/725b8e73ac2a50e78a4534ac43c6addf5c1c2d65380dd48a9169cc6739a9/yarl-1.20.1-cp313-cp313t-win32.whl", hash = "sha256:b121ff6a7cbd4abc28985b6028235491941b9fe8fe226e6fdc539c977ea1739d", size = 86591 },
-    { url = "https://files.pythonhosted.org/packages/94/c3/b2e9f38bc3e11191981d57ea08cab2166e74ea770024a646617c9cddd9f6/yarl-1.20.1-cp313-cp313t-win_amd64.whl", hash = "sha256:541d050a355bbbc27e55d906bc91cb6fe42f96c01413dd0f4ed5a5240513874f", size = 93003 },
-    { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542 },
+    { name = "idna", marker = "python_full_version >= '3.10'" },
+    { name = "multidict", marker = "python_full_version >= '3.10'" },
+    { name = "propcache", marker = "python_full_version >= '3.10'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3c/fb/efaa23fa4e45537b827620f04cf8f3cd658b76642205162e072703a5b963/yarl-1.20.1.tar.gz", hash = "sha256:d017a4997ee50c91fd5466cef416231bb82177b93b029906cefc542ce14c35ac", size = 186428, upload-time = "2025-06-10T00:46:09.923Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cb/65/7fed0d774abf47487c64be14e9223749468922817b5e8792b8a64792a1bb/yarl-1.20.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6032e6da6abd41e4acda34d75a816012717000fa6839f37124a47fcefc49bec4", size = 132910, upload-time = "2025-06-10T00:42:31.108Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/7b/988f55a52da99df9e56dc733b8e4e5a6ae2090081dc2754fc8fd34e60aa0/yarl-1.20.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2c7b34d804b8cf9b214f05015c4fee2ebe7ed05cf581e7192c06555c71f4446a", size = 90644, upload-time = "2025-06-10T00:42:33.851Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/de/30d98f03e95d30c7e3cc093759982d038c8833ec2451001d45ef4854edc1/yarl-1.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c869f2651cc77465f6cd01d938d91a11d9ea5d798738c1dc077f3de0b5e5fed", size = 89322, upload-time = "2025-06-10T00:42:35.688Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/7a/f2f314f5ebfe9200724b0b748de2186b927acb334cf964fd312eb86fc286/yarl-1.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62915e6688eb4d180d93840cda4110995ad50c459bf931b8b3775b37c264af1e", size = 323786, upload-time = "2025-06-10T00:42:37.817Z" },
+    { url = "https://files.pythonhosted.org/packages/15/3f/718d26f189db96d993d14b984ce91de52e76309d0fd1d4296f34039856aa/yarl-1.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41ebd28167bc6af8abb97fec1a399f412eec5fd61a3ccbe2305a18b84fb4ca73", size = 319627, upload-time = "2025-06-10T00:42:39.937Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/76/8fcfbf5fa2369157b9898962a4a7d96764b287b085b5b3d9ffae69cdefd1/yarl-1.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21242b4288a6d56f04ea193adde174b7e347ac46ce6bc84989ff7c1b1ecea84e", size = 339149, upload-time = "2025-06-10T00:42:42.627Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/95/d7fc301cc4661785967acc04f54a4a42d5124905e27db27bb578aac49b5c/yarl-1.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bea21cdae6c7eb02ba02a475f37463abfe0a01f5d7200121b03e605d6a0439f8", size = 333327, upload-time = "2025-06-10T00:42:44.842Z" },
+    { url = "https://files.pythonhosted.org/packages/65/94/e21269718349582eee81efc5c1c08ee71c816bfc1585b77d0ec3f58089eb/yarl-1.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f8a891e4a22a89f5dde7862994485e19db246b70bb288d3ce73a34422e55b23", size = 326054, upload-time = "2025-06-10T00:42:47.149Z" },
+    { url = "https://files.pythonhosted.org/packages/32/ae/8616d1f07853704523519f6131d21f092e567c5af93de7e3e94b38d7f065/yarl-1.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd803820d44c8853a109a34e3660e5a61beae12970da479cf44aa2954019bf70", size = 315035, upload-time = "2025-06-10T00:42:48.852Z" },
+    { url = "https://files.pythonhosted.org/packages/48/aa/0ace06280861ef055855333707db5e49c6e3a08840a7ce62682259d0a6c0/yarl-1.20.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b982fa7f74c80d5c0c7b5b38f908971e513380a10fecea528091405f519b9ebb", size = 338962, upload-time = "2025-06-10T00:42:51.024Z" },
+    { url = "https://files.pythonhosted.org/packages/20/52/1e9d0e6916f45a8fb50e6844f01cb34692455f1acd548606cbda8134cd1e/yarl-1.20.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:33f29ecfe0330c570d997bcf1afd304377f2e48f61447f37e846a6058a4d33b2", size = 335399, upload-time = "2025-06-10T00:42:53.007Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/65/60452df742952c630e82f394cd409de10610481d9043aa14c61bf846b7b1/yarl-1.20.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:835ab2cfc74d5eb4a6a528c57f05688099da41cf4957cf08cad38647e4a83b30", size = 338649, upload-time = "2025-06-10T00:42:54.964Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/f5/6cd4ff38dcde57a70f23719a838665ee17079640c77087404c3d34da6727/yarl-1.20.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:46b5e0ccf1943a9a6e766b2c2b8c732c55b34e28be57d8daa2b3c1d1d4009309", size = 358563, upload-time = "2025-06-10T00:42:57.28Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/90/c42eefd79d0d8222cb3227bdd51b640c0c1d0aa33fe4cc86c36eccba77d3/yarl-1.20.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:df47c55f7d74127d1b11251fe6397d84afdde0d53b90bedb46a23c0e534f9d24", size = 357609, upload-time = "2025-06-10T00:42:59.055Z" },
+    { url = "https://files.pythonhosted.org/packages/03/c8/cea6b232cb4617514232e0f8a718153a95b5d82b5290711b201545825532/yarl-1.20.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:76d12524d05841276b0e22573f28d5fbcb67589836772ae9244d90dd7d66aa13", size = 350224, upload-time = "2025-06-10T00:43:01.248Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/a3/eaa0ab9712f1f3d01faf43cf6f1f7210ce4ea4a7e9b28b489a2261ca8db9/yarl-1.20.1-cp310-cp310-win32.whl", hash = "sha256:6c4fbf6b02d70e512d7ade4b1f998f237137f1417ab07ec06358ea04f69134f8", size = 81753, upload-time = "2025-06-10T00:43:03.486Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/34/e4abde70a9256465fe31c88ed02c3f8502b7b5dead693a4f350a06413f28/yarl-1.20.1-cp310-cp310-win_amd64.whl", hash = "sha256:aef6c4d69554d44b7f9d923245f8ad9a707d971e6209d51279196d8e8fe1ae16", size = 86817, upload-time = "2025-06-10T00:43:05.231Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/18/893b50efc2350e47a874c5c2d67e55a0ea5df91186b2a6f5ac52eff887cd/yarl-1.20.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:47ee6188fea634bdfaeb2cc420f5b3b17332e6225ce88149a17c413c77ff269e", size = 133833, upload-time = "2025-06-10T00:43:07.393Z" },
+    { url = "https://files.pythonhosted.org/packages/89/ed/b8773448030e6fc47fa797f099ab9eab151a43a25717f9ac043844ad5ea3/yarl-1.20.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d0f6500f69e8402d513e5eedb77a4e1818691e8f45e6b687147963514d84b44b", size = 91070, upload-time = "2025-06-10T00:43:09.538Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/e3/409bd17b1e42619bf69f60e4f031ce1ccb29bd7380117a55529e76933464/yarl-1.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a8900a42fcdaad568de58887c7b2f602962356908eedb7628eaf6021a6e435b", size = 89818, upload-time = "2025-06-10T00:43:11.575Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/77/64d8431a4d77c856eb2d82aa3de2ad6741365245a29b3a9543cd598ed8c5/yarl-1.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bad6d131fda8ef508b36be3ece16d0902e80b88ea7200f030a0f6c11d9e508d4", size = 347003, upload-time = "2025-06-10T00:43:14.088Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/d2/0c7e4def093dcef0bd9fa22d4d24b023788b0a33b8d0088b51aa51e21e99/yarl-1.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:df018d92fe22aaebb679a7f89fe0c0f368ec497e3dda6cb81a567610f04501f1", size = 336537, upload-time = "2025-06-10T00:43:16.431Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/f3/fc514f4b2cf02cb59d10cbfe228691d25929ce8f72a38db07d3febc3f706/yarl-1.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f969afbb0a9b63c18d0feecf0db09d164b7a44a053e78a7d05f5df163e43833", size = 362358, upload-time = "2025-06-10T00:43:18.704Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/6d/a313ac8d8391381ff9006ac05f1d4331cee3b1efaa833a53d12253733255/yarl-1.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:812303eb4aa98e302886ccda58d6b099e3576b1b9276161469c25803a8db277d", size = 357362, upload-time = "2025-06-10T00:43:20.888Z" },
+    { url = "https://files.pythonhosted.org/packages/00/70/8f78a95d6935a70263d46caa3dd18e1f223cf2f2ff2037baa01a22bc5b22/yarl-1.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98c4a7d166635147924aa0bf9bfe8d8abad6fffa6102de9c99ea04a1376f91e8", size = 348979, upload-time = "2025-06-10T00:43:23.169Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/05/42773027968968f4f15143553970ee36ead27038d627f457cc44bbbeecf3/yarl-1.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12e768f966538e81e6e7550f9086a6236b16e26cd964cf4df35349970f3551cf", size = 337274, upload-time = "2025-06-10T00:43:27.111Z" },
+    { url = "https://files.pythonhosted.org/packages/05/be/665634aa196954156741ea591d2f946f1b78ceee8bb8f28488bf28c0dd62/yarl-1.20.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe41919b9d899661c5c28a8b4b0acf704510b88f27f0934ac7a7bebdd8938d5e", size = 363294, upload-time = "2025-06-10T00:43:28.96Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/90/73448401d36fa4e210ece5579895731f190d5119c4b66b43b52182e88cd5/yarl-1.20.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:8601bc010d1d7780592f3fc1bdc6c72e2b6466ea34569778422943e1a1f3c389", size = 358169, upload-time = "2025-06-10T00:43:30.701Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/b0/fce922d46dc1eb43c811f1889f7daa6001b27a4005587e94878570300881/yarl-1.20.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:daadbdc1f2a9033a2399c42646fbd46da7992e868a5fe9513860122d7fe7a73f", size = 362776, upload-time = "2025-06-10T00:43:32.51Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/0d/b172628fce039dae8977fd22caeff3eeebffd52e86060413f5673767c427/yarl-1.20.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:03aa1e041727cb438ca762628109ef1333498b122e4c76dd858d186a37cec845", size = 381341, upload-time = "2025-06-10T00:43:34.543Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/9b/5b886d7671f4580209e855974fe1cecec409aa4a89ea58b8f0560dc529b1/yarl-1.20.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:642980ef5e0fa1de5fa96d905c7e00cb2c47cb468bfcac5a18c58e27dbf8d8d1", size = 379988, upload-time = "2025-06-10T00:43:36.489Z" },
+    { url = "https://files.pythonhosted.org/packages/73/be/75ef5fd0fcd8f083a5d13f78fd3f009528132a1f2a1d7c925c39fa20aa79/yarl-1.20.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:86971e2795584fe8c002356d3b97ef6c61862720eeff03db2a7c86b678d85b3e", size = 371113, upload-time = "2025-06-10T00:43:38.592Z" },
+    { url = "https://files.pythonhosted.org/packages/50/4f/62faab3b479dfdcb741fe9e3f0323e2a7d5cd1ab2edc73221d57ad4834b2/yarl-1.20.1-cp311-cp311-win32.whl", hash = "sha256:597f40615b8d25812f14562699e287f0dcc035d25eb74da72cae043bb884d773", size = 81485, upload-time = "2025-06-10T00:43:41.038Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/09/d9c7942f8f05c32ec72cd5c8e041c8b29b5807328b68b4801ff2511d4d5e/yarl-1.20.1-cp311-cp311-win_amd64.whl", hash = "sha256:26ef53a9e726e61e9cd1cda6b478f17e350fb5800b4bd1cd9fe81c4d91cfeb2e", size = 86686, upload-time = "2025-06-10T00:43:42.692Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/9a/cb7fad7d73c69f296eda6815e4a2c7ed53fc70c2f136479a91c8e5fbdb6d/yarl-1.20.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdcc4cd244e58593a4379fe60fdee5ac0331f8eb70320a24d591a3be197b94a9", size = 133667, upload-time = "2025-06-10T00:43:44.369Z" },
+    { url = "https://files.pythonhosted.org/packages/67/38/688577a1cb1e656e3971fb66a3492501c5a5df56d99722e57c98249e5b8a/yarl-1.20.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b29a2c385a5f5b9c7d9347e5812b6f7ab267193c62d282a540b4fc528c8a9d2a", size = 91025, upload-time = "2025-06-10T00:43:46.295Z" },
+    { url = "https://files.pythonhosted.org/packages/50/ec/72991ae51febeb11a42813fc259f0d4c8e0507f2b74b5514618d8b640365/yarl-1.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1112ae8154186dfe2de4732197f59c05a83dc814849a5ced892b708033f40dc2", size = 89709, upload-time = "2025-06-10T00:43:48.22Z" },
+    { url = "https://files.pythonhosted.org/packages/99/da/4d798025490e89426e9f976702e5f9482005c548c579bdae792a4c37769e/yarl-1.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90bbd29c4fe234233f7fa2b9b121fb63c321830e5d05b45153a2ca68f7d310ee", size = 352287, upload-time = "2025-06-10T00:43:49.924Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/26/54a15c6a567aac1c61b18aa0f4b8aa2e285a52d547d1be8bf48abe2b3991/yarl-1.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:680e19c7ce3710ac4cd964e90dad99bf9b5029372ba0c7cbfcd55e54d90ea819", size = 345429, upload-time = "2025-06-10T00:43:51.7Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/95/9dcf2386cb875b234353b93ec43e40219e14900e046bf6ac118f94b1e353/yarl-1.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a979218c1fdb4246a05efc2cc23859d47c89af463a90b99b7c56094daf25a16", size = 365429, upload-time = "2025-06-10T00:43:53.494Z" },
+    { url = "https://files.pythonhosted.org/packages/91/b2/33a8750f6a4bc224242a635f5f2cff6d6ad5ba651f6edcccf721992c21a0/yarl-1.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255b468adf57b4a7b65d8aad5b5138dce6a0752c139965711bdcb81bc370e1b6", size = 363862, upload-time = "2025-06-10T00:43:55.766Z" },
+    { url = "https://files.pythonhosted.org/packages/98/28/3ab7acc5b51f4434b181b0cee8f1f4b77a65919700a355fb3617f9488874/yarl-1.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a97d67108e79cfe22e2b430d80d7571ae57d19f17cda8bb967057ca8a7bf5bfd", size = 355616, upload-time = "2025-06-10T00:43:58.056Z" },
+    { url = "https://files.pythonhosted.org/packages/36/a3/f666894aa947a371724ec7cd2e5daa78ee8a777b21509b4252dd7bd15e29/yarl-1.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8570d998db4ddbfb9a590b185a0a33dbf8aafb831d07a5257b4ec9948df9cb0a", size = 339954, upload-time = "2025-06-10T00:43:59.773Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/81/5f466427e09773c04219d3450d7a1256138a010b6c9f0af2d48565e9ad13/yarl-1.20.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97c75596019baae7c71ccf1d8cc4738bc08134060d0adfcbe5642f778d1dca38", size = 365575, upload-time = "2025-06-10T00:44:02.051Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/e3/e4b0ad8403e97e6c9972dd587388940a032f030ebec196ab81a3b8e94d31/yarl-1.20.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1c48912653e63aef91ff988c5432832692ac5a1d8f0fb8a33091520b5bbe19ef", size = 365061, upload-time = "2025-06-10T00:44:04.196Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/99/b8a142e79eb86c926f9f06452eb13ecb1bb5713bd01dc0038faf5452e544/yarl-1.20.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4c3ae28f3ae1563c50f3d37f064ddb1511ecc1d5584e88c6b7c63cf7702a6d5f", size = 364142, upload-time = "2025-06-10T00:44:06.527Z" },
+    { url = "https://files.pythonhosted.org/packages/34/f2/08ed34a4a506d82a1a3e5bab99ccd930a040f9b6449e9fd050320e45845c/yarl-1.20.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e9642f27036283550f5f57dc6156c51084b458570b9d0d96100c8bebb186a8", size = 381894, upload-time = "2025-06-10T00:44:08.379Z" },
+    { url = "https://files.pythonhosted.org/packages/92/f8/9a3fbf0968eac704f681726eff595dce9b49c8a25cd92bf83df209668285/yarl-1.20.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2c26b0c49220d5799f7b22c6838409ee9bc58ee5c95361a4d7831f03cc225b5a", size = 383378, upload-time = "2025-06-10T00:44:10.51Z" },
+    { url = "https://files.pythonhosted.org/packages/af/85/9363f77bdfa1e4d690957cd39d192c4cacd1c58965df0470a4905253b54f/yarl-1.20.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564ab3d517e3d01c408c67f2e5247aad4019dcf1969982aba3974b4093279004", size = 374069, upload-time = "2025-06-10T00:44:12.834Z" },
+    { url = "https://files.pythonhosted.org/packages/35/99/9918c8739ba271dcd935400cff8b32e3cd319eaf02fcd023d5dcd487a7c8/yarl-1.20.1-cp312-cp312-win32.whl", hash = "sha256:daea0d313868da1cf2fac6b2d3a25c6e3a9e879483244be38c8e6a41f1d876a5", size = 81249, upload-time = "2025-06-10T00:44:14.731Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/83/5d9092950565481b413b31a23e75dd3418ff0a277d6e0abf3729d4d1ce25/yarl-1.20.1-cp312-cp312-win_amd64.whl", hash = "sha256:48ea7d7f9be0487339828a4de0360d7ce0efc06524a48e1810f945c45b813698", size = 86710, upload-time = "2025-06-10T00:44:16.716Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/e1/2411b6d7f769a07687acee88a062af5833cf1966b7266f3d8dfb3d3dc7d3/yarl-1.20.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:0b5ff0fbb7c9f1b1b5ab53330acbfc5247893069e7716840c8e7d5bb7355038a", size = 131811, upload-time = "2025-06-10T00:44:18.933Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/27/584394e1cb76fb771371770eccad35de400e7b434ce3142c2dd27392c968/yarl-1.20.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:14f326acd845c2b2e2eb38fb1346c94f7f3b01a4f5c788f8144f9b630bfff9a3", size = 90078, upload-time = "2025-06-10T00:44:20.635Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/9a/3246ae92d4049099f52d9b0fe3486e3b500e29b7ea872d0f152966fc209d/yarl-1.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f60e4ad5db23f0b96e49c018596707c3ae89f5d0bd97f0ad3684bcbad899f1e7", size = 88748, upload-time = "2025-06-10T00:44:22.34Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/25/35afe384e31115a1a801fbcf84012d7a066d89035befae7c5d4284df1e03/yarl-1.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:49bdd1b8e00ce57e68ba51916e4bb04461746e794e7c4d4bbc42ba2f18297691", size = 349595, upload-time = "2025-06-10T00:44:24.314Z" },
+    { url = "https://files.pythonhosted.org/packages/28/2d/8aca6cb2cabc8f12efcb82749b9cefecbccfc7b0384e56cd71058ccee433/yarl-1.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:66252d780b45189975abfed839616e8fd2dbacbdc262105ad7742c6ae58f3e31", size = 342616, upload-time = "2025-06-10T00:44:26.167Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/e9/1312633d16b31acf0098d30440ca855e3492d66623dafb8e25b03d00c3da/yarl-1.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59174e7332f5d153d8f7452a102b103e2e74035ad085f404df2e40e663a22b28", size = 361324, upload-time = "2025-06-10T00:44:27.915Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/a0/688cc99463f12f7669eec7c8acc71ef56a1521b99eab7cd3abb75af887b0/yarl-1.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e3968ec7d92a0c0f9ac34d5ecfd03869ec0cab0697c91a45db3fbbd95fe1b653", size = 359676, upload-time = "2025-06-10T00:44:30.041Z" },
+    { url = "https://files.pythonhosted.org/packages/af/44/46407d7f7a56e9a85a4c207724c9f2c545c060380718eea9088f222ba697/yarl-1.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1a4fbb50e14396ba3d375f68bfe02215d8e7bc3ec49da8341fe3157f59d2ff5", size = 352614, upload-time = "2025-06-10T00:44:32.171Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/91/31163295e82b8d5485d31d9cf7754d973d41915cadce070491778d9c9825/yarl-1.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11a62c839c3a8eac2410e951301309426f368388ff2f33799052787035793b02", size = 336766, upload-time = "2025-06-10T00:44:34.494Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/8e/c41a5bc482121f51c083c4c2bcd16b9e01e1cf8729e380273a952513a21f/yarl-1.20.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:041eaa14f73ff5a8986b4388ac6bb43a77f2ea09bf1913df7a35d4646db69e53", size = 364615, upload-time = "2025-06-10T00:44:36.856Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/5b/61a3b054238d33d70ea06ebba7e58597891b71c699e247df35cc984ab393/yarl-1.20.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:377fae2fef158e8fd9d60b4c8751387b8d1fb121d3d0b8e9b0be07d1b41e83dc", size = 360982, upload-time = "2025-06-10T00:44:39.141Z" },
+    { url = "https://files.pythonhosted.org/packages/df/a3/6a72fb83f8d478cb201d14927bc8040af901811a88e0ff2da7842dd0ed19/yarl-1.20.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1c92f4390e407513f619d49319023664643d3339bd5e5a56a3bebe01bc67ec04", size = 369792, upload-time = "2025-06-10T00:44:40.934Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/af/4cc3c36dfc7c077f8dedb561eb21f69e1e9f2456b91b593882b0b18c19dc/yarl-1.20.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d25ddcf954df1754ab0f86bb696af765c5bfaba39b74095f27eececa049ef9a4", size = 382049, upload-time = "2025-06-10T00:44:42.854Z" },
+    { url = "https://files.pythonhosted.org/packages/19/3a/e54e2c4752160115183a66dc9ee75a153f81f3ab2ba4bf79c3c53b33de34/yarl-1.20.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:909313577e9619dcff8c31a0ea2aa0a2a828341d92673015456b3ae492e7317b", size = 384774, upload-time = "2025-06-10T00:44:45.275Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/20/200ae86dabfca89060ec6447649f219b4cbd94531e425e50d57e5f5ac330/yarl-1.20.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:793fd0580cb9664548c6b83c63b43c477212c0260891ddf86809e1c06c8b08f1", size = 374252, upload-time = "2025-06-10T00:44:47.31Z" },
+    { url = "https://files.pythonhosted.org/packages/83/75/11ee332f2f516b3d094e89448da73d557687f7d137d5a0f48c40ff211487/yarl-1.20.1-cp313-cp313-win32.whl", hash = "sha256:468f6e40285de5a5b3c44981ca3a319a4b208ccc07d526b20b12aeedcfa654b7", size = 81198, upload-time = "2025-06-10T00:44:49.164Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/ba/39b1ecbf51620b40ab402b0fc817f0ff750f6d92712b44689c2c215be89d/yarl-1.20.1-cp313-cp313-win_amd64.whl", hash = "sha256:495b4ef2fea40596bfc0affe3837411d6aa3371abcf31aac0ccc4bdd64d4ef5c", size = 86346, upload-time = "2025-06-10T00:44:51.182Z" },
+    { url = "https://files.pythonhosted.org/packages/43/c7/669c52519dca4c95153c8ad96dd123c79f354a376346b198f438e56ffeb4/yarl-1.20.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:f60233b98423aab21d249a30eb27c389c14929f47be8430efa7dbd91493a729d", size = 138826, upload-time = "2025-06-10T00:44:52.883Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/42/fc0053719b44f6ad04a75d7f05e0e9674d45ef62f2d9ad2c1163e5c05827/yarl-1.20.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6f3eff4cc3f03d650d8755c6eefc844edde99d641d0dcf4da3ab27141a5f8ddf", size = 93217, upload-time = "2025-06-10T00:44:54.658Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/7f/fa59c4c27e2a076bba0d959386e26eba77eb52ea4a0aac48e3515c186b4c/yarl-1.20.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:69ff8439d8ba832d6bed88af2c2b3445977eba9a4588b787b32945871c2444e3", size = 92700, upload-time = "2025-06-10T00:44:56.784Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/d4/062b2f48e7c93481e88eff97a6312dca15ea200e959f23e96d8ab898c5b8/yarl-1.20.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cf34efa60eb81dd2645a2e13e00bb98b76c35ab5061a3989c7a70f78c85006d", size = 347644, upload-time = "2025-06-10T00:44:59.071Z" },
+    { url = "https://files.pythonhosted.org/packages/89/47/78b7f40d13c8f62b499cc702fdf69e090455518ae544c00a3bf4afc9fc77/yarl-1.20.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8e0fe9364ad0fddab2688ce72cb7a8e61ea42eff3c7caeeb83874a5d479c896c", size = 323452, upload-time = "2025-06-10T00:45:01.605Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/2b/490d3b2dc66f52987d4ee0d3090a147ea67732ce6b4d61e362c1846d0d32/yarl-1.20.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f64fbf81878ba914562c672024089e3401974a39767747691c65080a67b18c1", size = 346378, upload-time = "2025-06-10T00:45:03.946Z" },
+    { url = "https://files.pythonhosted.org/packages/66/ad/775da9c8a94ce925d1537f939a4f17d782efef1f973039d821cbe4bcc211/yarl-1.20.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6342d643bf9a1de97e512e45e4b9560a043347e779a173250824f8b254bd5ce", size = 353261, upload-time = "2025-06-10T00:45:05.992Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/23/0ed0922b47a4f5c6eb9065d5ff1e459747226ddce5c6a4c111e728c9f701/yarl-1.20.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56dac5f452ed25eef0f6e3c6a066c6ab68971d96a9fb441791cad0efba6140d3", size = 335987, upload-time = "2025-06-10T00:45:08.227Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/49/bc728a7fe7d0e9336e2b78f0958a2d6b288ba89f25a1762407a222bf53c3/yarl-1.20.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7d7f497126d65e2cad8dc5f97d34c27b19199b6414a40cb36b52f41b79014be", size = 329361, upload-time = "2025-06-10T00:45:10.11Z" },
+    { url = "https://files.pythonhosted.org/packages/93/8f/b811b9d1f617c83c907e7082a76e2b92b655400e61730cd61a1f67178393/yarl-1.20.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:67e708dfb8e78d8a19169818eeb5c7a80717562de9051bf2413aca8e3696bf16", size = 346460, upload-time = "2025-06-10T00:45:12.055Z" },
+    { url = "https://files.pythonhosted.org/packages/70/fd/af94f04f275f95da2c3b8b5e1d49e3e79f1ed8b6ceb0f1664cbd902773ff/yarl-1.20.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:595c07bc79af2494365cc96ddeb772f76272364ef7c80fb892ef9d0649586513", size = 334486, upload-time = "2025-06-10T00:45:13.995Z" },
+    { url = "https://files.pythonhosted.org/packages/84/65/04c62e82704e7dd0a9b3f61dbaa8447f8507655fd16c51da0637b39b2910/yarl-1.20.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7bdd2f80f4a7df852ab9ab49484a4dee8030023aa536df41f2d922fd57bf023f", size = 342219, upload-time = "2025-06-10T00:45:16.479Z" },
+    { url = "https://files.pythonhosted.org/packages/91/95/459ca62eb958381b342d94ab9a4b6aec1ddec1f7057c487e926f03c06d30/yarl-1.20.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c03bfebc4ae8d862f853a9757199677ab74ec25424d0ebd68a0027e9c639a390", size = 350693, upload-time = "2025-06-10T00:45:18.399Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/00/d393e82dd955ad20617abc546a8f1aee40534d599ff555ea053d0ec9bf03/yarl-1.20.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:344d1103e9c1523f32a5ed704d576172d2cabed3122ea90b1d4e11fe17c66458", size = 355803, upload-time = "2025-06-10T00:45:20.677Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/ed/c5fb04869b99b717985e244fd93029c7a8e8febdfcffa06093e32d7d44e7/yarl-1.20.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:88cab98aa4e13e1ade8c141daeedd300a4603b7132819c484841bb7af3edce9e", size = 341709, upload-time = "2025-06-10T00:45:23.221Z" },
+    { url = "https://files.pythonhosted.org/packages/24/fd/725b8e73ac2a50e78a4534ac43c6addf5c1c2d65380dd48a9169cc6739a9/yarl-1.20.1-cp313-cp313t-win32.whl", hash = "sha256:b121ff6a7cbd4abc28985b6028235491941b9fe8fe226e6fdc539c977ea1739d", size = 86591, upload-time = "2025-06-10T00:45:25.793Z" },
+    { url = "https://files.pythonhosted.org/packages/94/c3/b2e9f38bc3e11191981d57ea08cab2166e74ea770024a646617c9cddd9f6/yarl-1.20.1-cp313-cp313t-win_amd64.whl", hash = "sha256:541d050a355bbbc27e55d906bc91cb6fe42f96c01413dd0f4ed5a5240513874f", size = 93003, upload-time = "2025-06-10T00:45:27.752Z" },
+    { url = "https://files.pythonhosted.org/packages/01/75/0d37402d208d025afa6b5b8eb80e466d267d3fd1927db8e317d29a94a4cb/yarl-1.20.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e42ba79e2efb6845ebab49c7bf20306c4edf74a0b20fc6b2ccdd1a219d12fad3", size = 134259, upload-time = "2025-06-10T00:45:29.882Z" },
+    { url = "https://files.pythonhosted.org/packages/73/84/1fb6c85ae0cf9901046f07d0ac9eb162f7ce6d95db541130aa542ed377e6/yarl-1.20.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:41493b9b7c312ac448b7f0a42a089dffe1d6e6e981a2d76205801a023ed26a2b", size = 91269, upload-time = "2025-06-10T00:45:32.917Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/9c/eae746b24c4ea29a5accba9a06c197a70fa38a49c7df244e0d3951108861/yarl-1.20.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f5a5928ff5eb13408c62a968ac90d43f8322fd56d87008b8f9dabf3c0f6ee983", size = 89995, upload-time = "2025-06-10T00:45:35.066Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/30/693e71003ec4bc1daf2e4cf7c478c417d0985e0a8e8f00b2230d517876fc/yarl-1.20.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30c41ad5d717b3961b2dd785593b67d386b73feca30522048d37298fee981805", size = 325253, upload-time = "2025-06-10T00:45:37.052Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/a2/5264dbebf90763139aeb0b0b3154763239398400f754ae19a0518b654117/yarl-1.20.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:59febc3969b0781682b469d4aca1a5cab7505a4f7b85acf6db01fa500fa3f6ba", size = 320897, upload-time = "2025-06-10T00:45:39.962Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/17/77c7a89b3c05856489777e922f41db79ab4faf58621886df40d812c7facd/yarl-1.20.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d2b6fb3622b7e5bf7a6e5b679a69326b4279e805ed1699d749739a61d242449e", size = 340696, upload-time = "2025-06-10T00:45:41.915Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/55/28409330b8ef5f2f681f5b478150496ec9cf3309b149dab7ec8ab5cfa3f0/yarl-1.20.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:749d73611db8d26a6281086f859ea7ec08f9c4c56cec864e52028c8b328db723", size = 335064, upload-time = "2025-06-10T00:45:43.893Z" },
+    { url = "https://files.pythonhosted.org/packages/85/58/cb0257cbd4002828ff735f44d3c5b6966c4fd1fc8cc1cd3cd8a143fbc513/yarl-1.20.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9427925776096e664c39e131447aa20ec738bdd77c049c48ea5200db2237e000", size = 327256, upload-time = "2025-06-10T00:45:46.393Z" },
+    { url = "https://files.pythonhosted.org/packages/53/f6/c77960370cfa46f6fb3d6a5a79a49d3abfdb9ef92556badc2dcd2748bc2a/yarl-1.20.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff70f32aa316393eaf8222d518ce9118148eddb8a53073c2403863b41033eed5", size = 316389, upload-time = "2025-06-10T00:45:48.358Z" },
+    { url = "https://files.pythonhosted.org/packages/64/ab/be0b10b8e029553c10905b6b00c64ecad3ebc8ace44b02293a62579343f6/yarl-1.20.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c7ddf7a09f38667aea38801da8b8d6bfe81df767d9dfc8c88eb45827b195cd1c", size = 340481, upload-time = "2025-06-10T00:45:50.663Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/c3/3f327bd3905a4916029bf5feb7f86dcf864c7704f099715f62155fb386b2/yarl-1.20.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:57edc88517d7fc62b174fcfb2e939fbc486a68315d648d7e74d07fac42cec240", size = 336941, upload-time = "2025-06-10T00:45:52.554Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/42/040bdd5d3b3bb02b4a6ace4ed4075e02f85df964d6e6cb321795d2a6496a/yarl-1.20.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:dab096ce479d5894d62c26ff4f699ec9072269d514b4edd630a393223f45a0ee", size = 339936, upload-time = "2025-06-10T00:45:54.919Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/1c/911867b8e8c7463b84dfdc275e0d99b04b66ad5132b503f184fe76be8ea4/yarl-1.20.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:14a85f3bd2d7bb255be7183e5d7d6e70add151a98edf56a770d6140f5d5f4010", size = 360163, upload-time = "2025-06-10T00:45:56.87Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/31/8c389f6c6ca0379b57b2da87f1f126c834777b4931c5ee8427dd65d0ff6b/yarl-1.20.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:2c89b5c792685dd9cd3fa9761c1b9f46fc240c2a3265483acc1565769996a3f8", size = 359108, upload-time = "2025-06-10T00:45:58.869Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/09/ae4a649fb3964324c70a3e2b61f45e566d9ffc0affd2b974cbf628957673/yarl-1.20.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:69e9b141de5511021942a6866990aea6d111c9042235de90e08f94cf972ca03d", size = 351875, upload-time = "2025-06-10T00:46:01.45Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/43/bbb4ed4c34d5bb62b48bf957f68cd43f736f79059d4f85225ab1ef80f4b9/yarl-1.20.1-cp39-cp39-win32.whl", hash = "sha256:b5f307337819cdfdbb40193cad84978a029f847b0a357fbe49f712063cfc4f06", size = 82293, upload-time = "2025-06-10T00:46:03.763Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/cd/ce185848a7dba68ea69e932674b5c1a42a1852123584bccc5443120f857c/yarl-1.20.1-cp39-cp39-win_amd64.whl", hash = "sha256:eae7bfe2069f9c1c5b05fc7fe5d612e5bbc089a39309904ee8b829e322dcad00", size = 87385, upload-time = "2025-06-10T00:46:05.655Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542, upload-time = "2025-06-10T00:46:07.521Z" },
+]
+
+[[package]]
+name = "zipp"
+version = "3.23.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" },
 ]
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index 089a799280d..5699bd4d536 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,5 +1,5 @@
 # We keep this pinned to keep clippy and rustfmt in sync between local and CI.
 # Feel free to upgrade to bring in new lints.
 [toolchain]
-channel = "1.91.0"
+channel = "1.94.0"
 components = ["rustfmt", "clippy", "rust-analyzer"]
diff --git a/rust/arrow-scalar/Cargo.toml b/rust/arrow-scalar/Cargo.toml
index c3d3f9181c3..f127e6c0bfa 100644
--- a/rust/arrow-scalar/Cargo.toml
+++ b/rust/arrow-scalar/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "arrow-scalar"
+name = "lance-arrow-scalar"
 version = "57.0.0"
 edition.workspace = true
 authors.workspace = true
diff --git a/rust/arrow-scalar/README.md b/rust/arrow-scalar/README.md
new file mode 100644
index 00000000000..7173a116262
--- /dev/null
+++ b/rust/arrow-scalar/README.md
@@ -0,0 +1,57 @@
+# lance-arrow-scalar
+
+A scalar type backed by Apache Arrow arrays with `Ord`, `Hash`, and `Eq` support.
+
+## Overview
+
+`ArrowScalar` wraps a single-element Arrow array and provides comparison and hashing operations by leveraging Apache Arrow's `OwnedRow` representation. This ensures:
+
+- **Correct total ordering** for all Arrow types
+- **Proper NaN handling** for floating-point values
+- **Consistent null ordering**
+- **O(1) comparisons** via cached row bytes
+
+## Features
+
+- `Eq`, `Ord`, and `Hash` traits for Arrow scalar values
+- Support for all Arrow data types
+- Serde serialization/deserialization support
+- Zero-copy conversion from Arrow arrays
+
+## Usage
+
+Add to your `Cargo.toml`:
+
+```toml
+[dependencies]
+lance-arrow-scalar = "57.0.0"
+```
+
+Then use in your code:
+
+```rust
+use lance_arrow_scalar::ArrowScalar;
+
+// Create from primitive types
+let a = ArrowScalar::from(42i32);
+let b = ArrowScalar::from(100i32);
+assert!(a < b);
+
+// Create from strings
+let s1 = ArrowScalar::from("hello");
+let s2 = ArrowScalar::from("world");
+assert!(s1 < s2);
+
+// Use in collections
+use std::collections::HashMap;
+let mut map = HashMap::new();
+map.insert(ArrowScalar::from("key"), ArrowScalar::from(123));
+```
+
+## Cross-Type Comparison
+
+Comparing scalars of different data types produces an arbitrary but consistent ordering based on the underlying row bytes. This allows scalars to be used as keys in sorted collections regardless of type, though the ordering across types is not semantically meaningful.
+
+## Implementation Details
+
+Comparisons and hashing are delegated to [`arrow_row::OwnedRow`], which provides efficient byte-level operations. The row representation is cached at construction time, making all comparison and hashing operations O(1).
diff --git a/rust/arrow-scalar/src/lib.rs b/rust/arrow-scalar/src/lib.rs
index bcb72e0c2aa..04246589296 100644
--- a/rust/arrow-scalar/src/lib.rs
+++ b/rust/arrow-scalar/src/lib.rs
@@ -40,7 +40,7 @@ type Result<T> = std::result::Result<T, ArrowError>;
 /// # Examples
 ///
 /// ```
-/// use arrow_scalar::ArrowScalar;
+/// use lance_arrow_scalar::ArrowScalar;
 ///
 /// let a = ArrowScalar::from(1i32);
 /// let b = ArrowScalar::from(2i32);
diff --git a/rust/arrow-stats/Cargo.toml b/rust/arrow-stats/Cargo.toml
new file mode 100644
index 00000000000..e57ea681034
--- /dev/null
+++ b/rust/arrow-stats/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+name = "lance-arrow-stats"
+version = "57.0.0"
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+description = "Statistics accumulator for Arrow arrays (min, max, null_count, nan_count)"
+keywords.workspace = true
+categories.workspace = true
+rust-version.workspace = true
+readme = "README.md"
+
+[dependencies]
+arrow-array = { workspace = true }
+arrow-schema = { workspace = true }
+lance-arrow-scalar = { workspace = true }
+half = { workspace = true }
+
+[dev-dependencies]
+arrow-select = { workspace = true }
+proptest = { workspace = true }
+rstest = { workspace = true }
+
+[lints]
+workspace = true
diff --git a/rust/arrow-stats/README.md b/rust/arrow-stats/README.md
new file mode 100644
index 00000000000..553f6de2e69
--- /dev/null
+++ b/rust/arrow-stats/README.md
@@ -0,0 +1,62 @@
+# lance-arrow-stats
+
+Statistics accumulator for [Apache Arrow](https://arrow.apache.org/) arrays.
+
+Computes min, max, null count, NaN count, and buffer memory usage over one or
+more batches of Arrow data. Designed for use in Lance's columnar storage layer
+where page-level statistics drive predicate pushdown and query planning.
+
+## Usage
+
+```rust
+use arrow_array::{Int32Array, ArrayRef};
+use lance_arrow_stats::StatisticsAccumulator;
+use arrow_schema::DataType;
+use std::sync::Arc;
+
+let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+
+let batch: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), None, Some(1), Some(4)]));
+acc.update(&batch).unwrap();
+
+let stats = acc.finish();
+assert_eq!(stats.null_count, 1);
+```
+
+## Tracked Statistics
+
+| Statistic       | Description                                              |
+| --------------- | -------------------------------------------------------- |
+| `min`           | Minimum non-null, non-NaN value (`ArrowScalar`)          |
+| `max`           | Maximum non-null, non-NaN value (`ArrowScalar`)          |
+| `null_count`    | Total number of null values                              |
+| `nan_count`     | Total NaN values (float and float-list types only)       |
+| `item_nulls`    | Null items inside list entries (list types only)          |
+| `buffer_memory` | Total Arrow buffer memory in bytes                       |
+
+## Supported Types
+
+- **Numeric** &mdash; Int8–Int64, UInt8–UInt64, Float16/32/64
+- **Temporal** &mdash; Date32/64, Time32/64, Timestamp, Duration
+- **Boolean**
+- **String** &mdash; Utf8, LargeUtf8
+- **Binary** &mdash; Binary, LargeBinary
+- **List** &mdash; List, LargeList, FixedSizeList (computes stats over items)
+
+Dictionary, run-end encoded, and view types are accepted but min/max will be
+`None`.
+
+## Merging
+
+Accumulators of the same data type can be merged, which is useful for combining
+statistics computed in parallel across different pages or files:
+
+```rust
+use lance_arrow_stats::StatisticsAccumulator;
+use arrow_schema::DataType;
+
+let mut a = StatisticsAccumulator::new(&DataType::Float32);
+let mut b = StatisticsAccumulator::new(&DataType::Float32);
+// ... update each with different batches ...
+a.merge(&b).unwrap();
+```
diff --git a/rust/arrow-stats/proptest-regressions/lib.txt b/rust/arrow-stats/proptest-regressions/lib.txt
new file mode 100644
index 00000000000..8794b465e9c
--- /dev/null
+++ b/rust/arrow-stats/proptest-regressions/lib.txt
@@ -0,0 +1,8 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 81b0445f36fa8f491c1fb3162f51b61c8be140d5b2a1e792c42b4bdb7f1b6a62 # shrinks to values = [0.0, -0.0]
+cc 8651fce939497f33c6dafd842937d95965af97833bfbbd10df30d5ea00dbd07d # shrinks to values = [Some(0.0), Some(-0.0)]
diff --git a/rust/arrow-stats/src/lib.rs b/rust/arrow-stats/src/lib.rs
new file mode 100644
index 00000000000..5c00a015749
--- /dev/null
+++ b/rust/arrow-stats/src/lib.rs
@@ -0,0 +1,1294 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Statistics accumulator for streams of Arrow arrays.
+//!
+//! Tracks min, max, null_count, and optional nan_count across batches of arrays sharing
+//! the same [`DataType`]. Uses [`ArrowScalar`] for extrema tracking.
+//!
+//! # Example
+//!
+//! ```
+//! use std::sync::Arc;
+//! use arrow_array::{ArrayRef, Int32Array};
+//! use arrow_schema::DataType;
+//! use lance_arrow_stats::StatisticsAccumulator;
+//!
+//! let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+//! let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]));
+//! acc.update(&array).unwrap();
+//!
+//! let stats = acc.finish();
+//! assert_eq!(stats.null_count, 1);
+//! assert!(stats.nan_count.is_none());
+//! ```
+//!
+//! # Data Type Support
+//!
+//! All basic types are supported.  Every data type supports `null_count` and `buffer_memory`.
+//! The `nan_count` field is `Some` only for floating-point types (including lists of floats)
+//! and `None` for all other types.
+//!
+//! # List Types
+//!
+//! List types are supported.  The `item_nulls` field will be set to the number of null items within list entries.
+//! This will be `Some` only for list types.
+//!
+//! # String Types / Binary Types
+//!
+//! String & binary types are supported.  Binary comparison will be used for string types to calculate the min and the
+//! max.  This works for ASCII but may be surprising for special characters.  For example, "é" would sort after "z".
+//!
+//! In addition, string view and binary view types are supported.
+//!
+//! # Unsupported Types
+//!
+//! Special encodings (dictionary, run end encoded, view types) are not currently fully supported.  The min and max will
+//! be set to `None` for these types.  The `nan_count` will also be `None` unless the underlying
+//! type is a floating-point type.
+//!
+//! Structs are not currently supported.
+
+mod nan;
+
+use arrow_array::cast::AsArray;
+use arrow_array::types::*;
+use arrow_array::{Array, ArrayRef};
+use arrow_schema::{ArrowError, DataType};
+use lance_arrow_scalar::ArrowScalar;
+
+use nan::count_nans;
+
+type Result<T> = std::result::Result<T, ArrowError>;
+
+/// Returns true if the data type can contain NaN values (float primitives
+/// or list types whose items are floats).
+fn can_have_nan(data_type: &DataType) -> bool {
+    match data_type {
+        DataType::Float16 | DataType::Float32 | DataType::Float64 => true,
+        DataType::List(f) | DataType::LargeList(f) => can_have_nan(f.data_type()),
+        DataType::FixedSizeList(f, _) => can_have_nan(f.data_type()),
+        _ => false,
+    }
+}
+
+/// Accumulated statistics for a stream of Arrow arrays of a single [`DataType`].
+#[derive(Debug, Clone)]
+pub struct StatisticsAccumulator {
+    data_type: DataType,
+    min: Option<ArrowScalar>,
+    max: Option<ArrowScalar>,
+    null_count: u64,
+    /// Count of NaN values. `Some` only for floating-point types (or lists of floats).
+    nan_count: Option<u64>,
+    /// Number of null items within list entries. `Some` only for list types.
+    item_nulls: Option<u64>,
+    buffer_memory: u64,
+}
+
+/// Snapshot of accumulated statistics.
+#[derive(Debug, Clone)]
+pub struct Statistics {
+    pub min: Option<ArrowScalar>,
+    pub max: Option<ArrowScalar>,
+    pub null_count: u64,
+    /// Count of NaN values. `None` for non-floating-point types.
+    pub nan_count: Option<u64>,
+    /// Number of null items within list entries. `None` for non-list types.
+    pub item_nulls: Option<u64>,
+    /// Total buffer memory in bytes across all arrays seen by this accumulator.
+    pub buffer_memory: u64,
+}
+
+impl StatisticsAccumulator {
+    /// Create a new accumulator for arrays of the given data type.
+    pub fn new(data_type: &DataType) -> Self {
+        let item_nulls = match data_type {
+            DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _) => Some(0),
+            _ => None,
+        };
+        let nan_count = if can_have_nan(data_type) {
+            Some(0)
+        } else {
+            None
+        };
+        Self {
+            data_type: data_type.clone(),
+            min: None,
+            max: None,
+            null_count: 0,
+            nan_count,
+            item_nulls,
+            buffer_memory: 0,
+        }
+    }
+
+    /// Returns the data type this accumulator expects.
+    pub fn data_type(&self) -> &DataType {
+        &self.data_type
+    }
+
+    /// Update with a new batch of values.
+    pub fn update(&mut self, array: &ArrayRef) -> Result<()> {
+        if array.data_type() != &self.data_type {
+            return Err(ArrowError::InvalidArgumentError(format!(
+                "Type mismatch: expected {:?}, got {:?}",
+                self.data_type,
+                array.data_type()
+            )));
+        }
+
+        if array.is_empty() {
+            return Ok(());
+        }
+
+        self.buffer_memory += array.get_buffer_memory_size() as u64;
+        self.null_count += array.null_count() as u64;
+
+        match array.data_type() {
+            DataType::List(_) => {
+                let list = array.as_list::<i32>();
+                self.update_items(
+                    (0..list.len())
+                        .filter(|&i| !list.is_null(i))
+                        .map(|i| list.value(i)),
+                )
+            }
+            DataType::LargeList(_) => {
+                let list = array.as_list::<i64>();
+                self.update_items(
+                    (0..list.len())
+                        .filter(|&i| !list.is_null(i))
+                        .map(|i| list.value(i)),
+                )
+            }
+            DataType::FixedSizeList(_, _) => {
+                let list = array.as_fixed_size_list();
+                self.update_items(
+                    (0..list.len())
+                        .filter(|&i| !list.is_null(i))
+                        .map(|i| list.value(i)),
+                )
+            }
+            _ => {
+                if let Some(ref mut nan_count) = self.nan_count {
+                    *nan_count += count_nans(array);
+                }
+                let (batch_min, batch_max) = find_min_max(array)?;
+                self.update_min(batch_min);
+                self.update_max(batch_max);
+                Ok(())
+            }
+        }
+    }
+
+    /// Process items from list entries, updating min/max, nan_count, and item_nulls.
+    fn update_items(&mut self, items: impl Iterator<Item = ArrayRef>) -> Result<()> {
+        for item_array in items {
+            self.update_item(&item_array)?;
+        }
+        Ok(())
+    }
+
+    /// Process a single item array. If it is itself a list type, recurse into
+    /// its non-null entries; otherwise treat it as a leaf and compute min/max.
+    fn update_item(&mut self, item_array: &ArrayRef) -> Result<()> {
+        if item_array.is_empty() {
+            return Ok(());
+        }
+        if let Some(ref mut item_nulls) = self.item_nulls {
+            *item_nulls += item_array.null_count() as u64;
+        }
+        match item_array.data_type() {
+            DataType::List(_) => {
+                let list = item_array.as_list::<i32>();
+                for i in 0..list.len() {
+                    if !list.is_null(i) {
+                        self.update_item(&list.value(i))?;
+                    }
+                }
+            }
+            DataType::LargeList(_) => {
+                let list = item_array.as_list::<i64>();
+                for i in 0..list.len() {
+                    if !list.is_null(i) {
+                        self.update_item(&list.value(i))?;
+                    }
+                }
+            }
+            DataType::FixedSizeList(_, _) => {
+                let list = item_array.as_fixed_size_list();
+                for i in 0..list.len() {
+                    if !list.is_null(i) {
+                        self.update_item(&list.value(i))?;
+                    }
+                }
+            }
+            _ => {
+                if let Some(ref mut nan_count) = self.nan_count {
+                    *nan_count += count_nans(item_array);
+                }
+                let (batch_min, batch_max) = find_min_max(item_array)?;
+                self.update_min(batch_min);
+                self.update_max(batch_max);
+            }
+        }
+        Ok(())
+    }
+
+    fn update_min(&mut self, batch_min: Option<ArrowScalar>) {
+        if let Some(new_min) = batch_min {
+            self.min = Some(match self.min.take() {
+                Some(cur) if cur <= new_min => cur,
+                _ => new_min,
+            });
+        }
+    }
+
+    fn update_max(&mut self, batch_max: Option<ArrowScalar>) {
+        if let Some(new_max) = batch_max {
+            self.max = Some(match self.max.take() {
+                Some(cur) if cur >= new_max => cur,
+                _ => new_max,
+            });
+        }
+    }
+
+    /// Merge another accumulator into this one.
+    pub fn merge(&mut self, other: &Self) -> Result<()> {
+        if self.data_type != other.data_type {
+            return Err(ArrowError::InvalidArgumentError(format!(
+                "Type mismatch: expected {:?}, got {:?}",
+                self.data_type, other.data_type
+            )));
+        }
+
+        self.null_count += other.null_count;
+        if let (Some(a), Some(b)) = (&mut self.nan_count, other.nan_count) {
+            *a += b;
+        }
+        self.buffer_memory += other.buffer_memory;
+
+        if let (Some(a), Some(b)) = (&mut self.item_nulls, other.item_nulls) {
+            *a += b;
+        }
+
+        if let Some(ref other_min) = other.min {
+            self.min = Some(match self.min.take() {
+                Some(cur) if cur <= *other_min => cur,
+                _ => other_min.clone(),
+            });
+        }
+
+        if let Some(ref other_max) = other.max {
+            self.max = Some(match self.max.take() {
+                Some(cur) if cur >= *other_max => cur,
+                _ => other_max.clone(),
+            });
+        }
+
+        Ok(())
+    }
+
+    /// Consume the accumulator and return a statistics snapshot.
+    pub fn finish(self) -> Statistics {
+        Statistics {
+            min: self.min,
+            max: self.max,
+            null_count: self.null_count,
+            nan_count: self.nan_count,
+            item_nulls: self.item_nulls,
+            buffer_memory: self.buffer_memory,
+        }
+    }
+
+    /// Return a snapshot of the current statistics without consuming the accumulator.
+    pub fn statistics(&self) -> Statistics {
+        Statistics {
+            min: self.min.clone(),
+            max: self.max.clone(),
+            null_count: self.null_count,
+            nan_count: self.nan_count,
+            item_nulls: self.item_nulls,
+            buffer_memory: self.buffer_memory,
+        }
+    }
+
+    /// Reset all statistics back to initial state.
+    pub fn reset(&mut self) {
+        self.min = None;
+        self.max = None;
+        self.null_count = 0;
+        if let Some(ref mut nan_count) = self.nan_count {
+            *nan_count = 0;
+        }
+        if let Some(ref mut item_nulls) = self.item_nulls {
+            *item_nulls = 0;
+        }
+        self.buffer_memory = 0;
+    }
+}
+
+macro_rules! find_extrema_primitive {
+    ($array:expr, $arrow_type:ty) => {{
+        let typed = $array.as_primitive::<$arrow_type>();
+        let mut min_idx: Option<usize> = None;
+        let mut max_idx: Option<usize> = None;
+        let mut min_val = None;
+        let mut max_val = None;
+        for i in 0..typed.len() {
+            if typed.is_null(i) {
+                continue;
+            }
+            let v = typed.value(i);
+            if min_val.is_none() || v < *min_val.as_ref().unwrap() {
+                min_val = Some(v);
+                min_idx = Some(i);
+            }
+            if max_val.is_none() || v > *max_val.as_ref().unwrap() {
+                max_val = Some(v);
+                max_idx = Some(i);
+            }
+        }
+        (min_idx, max_idx)
+    }};
+}
+
+macro_rules! find_extrema_float {
+    ($array:expr, $arrow_type:ty) => {{
+        let typed = $array.as_primitive::<$arrow_type>();
+        let mut min_idx: Option<usize> = None;
+        let mut max_idx: Option<usize> = None;
+        let mut min_val = None;
+        let mut max_val = None;
+        for i in 0..typed.len() {
+            if typed.is_null(i) {
+                continue;
+            }
+            let v = typed.value(i);
+            if v.is_nan() {
+                continue;
+            }
+            // Use total_cmp for a consistent total ordering that
+            // distinguishes -0.0 from 0.0 (matching ArrowScalar's Ord).
+            if min_val.is_none()
+                || v.total_cmp(min_val.as_ref().unwrap()) == std::cmp::Ordering::Less
+            {
+                min_val = Some(v);
+                min_idx = Some(i);
+            }
+            if max_val.is_none()
+                || v.total_cmp(max_val.as_ref().unwrap()) == std::cmp::Ordering::Greater
+            {
+                max_val = Some(v);
+                max_idx = Some(i);
+            }
+        }
+        (min_idx, max_idx)
+    }};
+}
+
+macro_rules! find_extrema_bytes {
+    ($array:expr, $cast:ident :: < $offset:ty >) => {{
+        let typed = $array.$cast::<$offset>();
+        let mut min_idx: Option<usize> = None;
+        let mut max_idx: Option<usize> = None;
+        let mut min_val = None;
+        let mut max_val = None;
+        for i in 0..typed.len() {
+            if typed.is_null(i) {
+                continue;
+            }
+            let v = typed.value(i);
+            if min_val.is_none() || v < min_val.unwrap() {
+                min_val = Some(v);
+                min_idx = Some(i);
+            }
+            if max_val.is_none() || v > max_val.unwrap() {
+                max_val = Some(v);
+                max_idx = Some(i);
+            }
+        }
+        (min_idx, max_idx)
+    }};
+}
+
+fn find_min_max(array: &ArrayRef) -> Result<(Option<ArrowScalar>, Option<ArrowScalar>)> {
+    let (min_idx, max_idx) = find_min_max_indices(array)?;
+
+    let min_scalar = min_idx
+        .map(|i| ArrowScalar::try_new(array, i))
+        .transpose()?;
+    let max_scalar = max_idx
+        .map(|i| ArrowScalar::try_new(array, i))
+        .transpose()?;
+
+    Ok((min_scalar, max_scalar))
+}
+
+fn find_min_max_indices(array: &ArrayRef) -> Result<(Option<usize>, Option<usize>)> {
+    use DataType::*;
+
+    let result = match array.data_type() {
+        // Integer types
+        Int8 => find_extrema_primitive!(array, Int8Type),
+        Int16 => find_extrema_primitive!(array, Int16Type),
+        Int32 => find_extrema_primitive!(array, Int32Type),
+        Int64 => find_extrema_primitive!(array, Int64Type),
+        UInt8 => find_extrema_primitive!(array, UInt8Type),
+        UInt16 => find_extrema_primitive!(array, UInt16Type),
+        UInt32 => find_extrema_primitive!(array, UInt32Type),
+        UInt64 => find_extrema_primitive!(array, UInt64Type),
+
+        // Float types (skip NaN)
+        Float16 => find_extrema_float!(array, Float16Type),
+        Float32 => find_extrema_float!(array, Float32Type),
+        Float64 => find_extrema_float!(array, Float64Type),
+
+        // Temporal types
+        Date32 => find_extrema_primitive!(array, Date32Type),
+        Date64 => find_extrema_primitive!(array, Date64Type),
+        Time32(arrow_schema::TimeUnit::Second) => {
+            find_extrema_primitive!(array, Time32SecondType)
+        }
+        Time32(arrow_schema::TimeUnit::Millisecond) => {
+            find_extrema_primitive!(array, Time32MillisecondType)
+        }
+        Time64(arrow_schema::TimeUnit::Microsecond) => {
+            find_extrema_primitive!(array, Time64MicrosecondType)
+        }
+        Time64(arrow_schema::TimeUnit::Nanosecond) => {
+            find_extrema_primitive!(array, Time64NanosecondType)
+        }
+        Timestamp(arrow_schema::TimeUnit::Second, _) => {
+            find_extrema_primitive!(array, TimestampSecondType)
+        }
+        Timestamp(arrow_schema::TimeUnit::Millisecond, _) => {
+            find_extrema_primitive!(array, TimestampMillisecondType)
+        }
+        Timestamp(arrow_schema::TimeUnit::Microsecond, _) => {
+            find_extrema_primitive!(array, TimestampMicrosecondType)
+        }
+        Timestamp(arrow_schema::TimeUnit::Nanosecond, _) => {
+            find_extrema_primitive!(array, TimestampNanosecondType)
+        }
+        Duration(arrow_schema::TimeUnit::Second) => {
+            find_extrema_primitive!(array, DurationSecondType)
+        }
+        Duration(arrow_schema::TimeUnit::Millisecond) => {
+            find_extrema_primitive!(array, DurationMillisecondType)
+        }
+        Duration(arrow_schema::TimeUnit::Microsecond) => {
+            find_extrema_primitive!(array, DurationMicrosecondType)
+        }
+        Duration(arrow_schema::TimeUnit::Nanosecond) => {
+            find_extrema_primitive!(array, DurationNanosecondType)
+        }
+
+        // Boolean
+        Boolean => {
+            let typed = array.as_boolean();
+            let mut min_idx: Option<usize> = None;
+            let mut max_idx: Option<usize> = None;
+            let mut min_val: Option<bool> = None;
+            let mut max_val: Option<bool> = None;
+            for i in 0..typed.len() {
+                if typed.is_null(i) {
+                    continue;
+                }
+                let v = typed.value(i);
+                if min_val.is_none() || (!v && min_val.unwrap()) {
+                    min_val = Some(v);
+                    min_idx = Some(i);
+                }
+                if max_val.is_none() || (v && !max_val.unwrap()) {
+                    max_val = Some(v);
+                    max_idx = Some(i);
+                }
+            }
+            (min_idx, max_idx)
+        }
+
+        // String types
+        Utf8 => find_extrema_bytes!(array, as_string::<i32>),
+        LargeUtf8 => find_extrema_bytes!(array, as_string::<i64>),
+
+        // Binary types
+        Binary => find_extrema_bytes!(array, as_binary::<i32>),
+        LargeBinary => find_extrema_bytes!(array, as_binary::<i64>),
+
+        // For unsupported types we skip min/max (and nan_count).
+        // null_count and buffer_memory are already tracked above.
+        _ => return Ok((None, None)),
+    };
+
+    Ok(result)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::*;
+    use arrow_schema::DataType;
+    use rstest::rstest;
+
+    use super::*;
+
+    #[test]
+    fn test_empty_array() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+        let array: ArrayRef = Arc::new(Int32Array::from(Vec::<i32>::new()));
+        acc.update(&array).unwrap();
+        let stats = acc.finish();
+        assert!(stats.min.is_none());
+        assert!(stats.max.is_none());
+        assert_eq!(stats.null_count, 0);
+        assert_eq!(stats.nan_count, None);
+    }
+
+    #[test]
+    fn test_all_nulls() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+        let array: ArrayRef = Arc::new(Int32Array::from(vec![None, None, None]));
+        acc.update(&array).unwrap();
+        let stats = acc.finish();
+        assert!(stats.min.is_none());
+        assert!(stats.max.is_none());
+        assert_eq!(stats.null_count, 3);
+    }
+
+    #[test]
+    fn test_single_value() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+        let array: ArrayRef = Arc::new(Int32Array::from(vec![42]));
+        acc.update(&array).unwrap();
+        let stats = acc.finish();
+        assert_eq!(stats.min.as_ref().unwrap(), stats.max.as_ref().unwrap());
+        assert_eq!(format!("{}", stats.min.unwrap()), "42");
+    }
+
+    #[test]
+    fn test_basic_int_stats() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+        let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 5, 3, 2, 4]));
+        acc.update(&array).unwrap();
+        let stats = acc.finish();
+        assert_eq!(format!("{}", stats.min.unwrap()), "1");
+        assert_eq!(format!("{}", stats.max.unwrap()), "5");
+        assert_eq!(stats.null_count, 0);
+        assert_eq!(stats.nan_count, None);
+    }
+
+    #[test]
+    fn test_with_nulls() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+        let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]));
+        acc.update(&array).unwrap();
+        let stats = acc.finish();
+        assert_eq!(format!("{}", stats.min.unwrap()), "1");
+        assert_eq!(format!("{}", stats.max.unwrap()), "3");
+        assert_eq!(stats.null_count, 1);
+    }
+
+    #[test]
+    fn test_float_nan_excluded() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Float64);
+        let array: ArrayRef = Arc::new(Float64Array::from(vec![1.0, f64::NAN, 3.0]));
+        acc.update(&array).unwrap();
+        let stats = acc.finish();
+        assert_eq!(format!("{}", stats.min.unwrap()), "1.0");
+        assert_eq!(format!("{}", stats.max.unwrap()), "3.0");
+        assert_eq!(stats.nan_count, Some(1));
+    }
+
+    #[test]
+    fn test_all_nan() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Float64);
+        let array: ArrayRef = Arc::new(Float64Array::from(vec![f64::NAN, f64::NAN]));
+        acc.update(&array).unwrap();
+        let stats = acc.finish();
+        assert!(stats.min.is_none());
+        assert!(stats.max.is_none());
+        assert_eq!(stats.nan_count, Some(2));
+    }
+
+    #[test]
+    fn test_null_and_nan() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Float64);
+        let array: ArrayRef = Arc::new(Float64Array::from(vec![None, Some(f64::NAN)]));
+        acc.update(&array).unwrap();
+        let stats = acc.finish();
+        assert!(stats.min.is_none());
+        assert!(stats.max.is_none());
+        assert_eq!(stats.null_count, 1);
+        assert_eq!(stats.nan_count, Some(1));
+    }
+
+    #[test]
+    fn test_multiple_updates() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+        let a1: ArrayRef = Arc::new(Int32Array::from(vec![5, 3]));
+        let a2: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(7)]));
+        acc.update(&a1).unwrap();
+        acc.update(&a2).unwrap();
+        let stats = acc.finish();
+        assert_eq!(format!("{}", stats.min.unwrap()), "1");
+        assert_eq!(format!("{}", stats.max.unwrap()), "7");
+        assert_eq!(stats.null_count, 1);
+    }
+
+    #[test]
+    fn test_merge() {
+        let mut acc1 = StatisticsAccumulator::new(&DataType::Int32);
+        let a1: ArrayRef = Arc::new(Int32Array::from(vec![1, 5]));
+        acc1.update(&a1).unwrap();
+
+        let mut acc2 = StatisticsAccumulator::new(&DataType::Int32);
+        let a2: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), None, Some(10)]));
+        acc2.update(&a2).unwrap();
+
+        acc1.merge(&acc2).unwrap();
+        let stats = acc1.finish();
+        assert_eq!(format!("{}", stats.min.unwrap()), "1");
+        assert_eq!(format!("{}", stats.max.unwrap()), "10");
+        assert_eq!(stats.null_count, 1);
+    }
+
+    #[test]
+    fn test_merge_type_mismatch() {
+        let acc1 = StatisticsAccumulator::new(&DataType::Int32);
+        let acc2 = StatisticsAccumulator::new(&DataType::Float64);
+        let mut acc1 = acc1;
+        assert!(acc1.merge(&acc2).is_err());
+    }
+
+    #[test]
+    fn test_type_mismatch_error() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+        let array: ArrayRef = Arc::new(Float64Array::from(vec![1.0]));
+        assert!(acc.update(&array).is_err());
+    }
+
+    #[test]
+    fn test_reset() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+        let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]));
+        acc.update(&array).unwrap();
+        acc.reset();
+        let stats = acc.finish();
+        assert!(stats.min.is_none());
+        assert!(stats.max.is_none());
+        assert_eq!(stats.null_count, 0);
+        assert_eq!(stats.nan_count, None);
+    }
+
+    #[test]
+    fn test_string_stats() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Utf8);
+        let array: ArrayRef = Arc::new(StringArray::from(vec!["apple", "cherry", "banana"]));
+        acc.update(&array).unwrap();
+        let stats = acc.finish();
+        assert_eq!(format!("{}", stats.min.unwrap()), "apple");
+        assert_eq!(format!("{}", stats.max.unwrap()), "cherry");
+    }
+
+    #[test]
+    fn test_boolean_stats() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Boolean);
+        let array: ArrayRef = Arc::new(BooleanArray::from(vec![true, false]));
+        acc.update(&array).unwrap();
+        let stats = acc.finish();
+        assert_eq!(format!("{}", stats.min.unwrap()), "false");
+        assert_eq!(format!("{}", stats.max.unwrap()), "true");
+    }
+
+    #[rstest]
+    #[case::i32(
+        DataType::Int32,
+        Arc::new(Int32Array::from(vec![3, 1, 2])) as ArrayRef,
+        "1", "3"
+    )]
+    #[case::i64(
+        DataType::Int64,
+        Arc::new(Int64Array::from(vec![30, 10, 20])) as ArrayRef,
+        "10", "30"
+    )]
+    #[case::u32(
+        DataType::UInt32,
+        Arc::new(UInt32Array::from(vec![3, 1, 2])) as ArrayRef,
+        "1", "3"
+    )]
+    #[case::u64(
+        DataType::UInt64,
+        Arc::new(UInt64Array::from(vec![30, 10, 20])) as ArrayRef,
+        "10", "30"
+    )]
+    #[case::f32(
+        DataType::Float32,
+        Arc::new(Float32Array::from(vec![3.0f32, 1.0, 2.0])) as ArrayRef,
+        "1.0", "3.0"
+    )]
+    #[case::f64(
+        DataType::Float64,
+        Arc::new(Float64Array::from(vec![3.0f64, 1.0, 2.0])) as ArrayRef,
+        "1.0", "3.0"
+    )]
+    fn test_rstest_primitives(
+        #[case] dt: DataType,
+        #[case] array: ArrayRef,
+        #[case] expected_min: &str,
+        #[case] expected_max: &str,
+    ) {
+        let mut acc = StatisticsAccumulator::new(&dt);
+        acc.update(&array).unwrap();
+        let stats = acc.finish();
+        assert_eq!(format!("{}", stats.min.unwrap()), expected_min);
+        assert_eq!(format!("{}", stats.max.unwrap()), expected_max);
+    }
+
+    #[test]
+    fn test_statistics_does_not_consume() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+        let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+        acc.update(&array).unwrap();
+        let s1 = acc.statistics();
+        let s2 = acc.statistics();
+        assert_eq!(format!("{}", s1.min.unwrap()), "1");
+        assert_eq!(format!("{}", s2.max.unwrap()), "3");
+    }
+
+    #[test]
+    fn test_merge_into_empty() {
+        let mut acc1 = StatisticsAccumulator::new(&DataType::Int32);
+        let mut acc2 = StatisticsAccumulator::new(&DataType::Int32);
+        let array: ArrayRef = Arc::new(Int32Array::from(vec![5, 10]));
+        acc2.update(&array).unwrap();
+
+        acc1.merge(&acc2).unwrap();
+        let stats = acc1.finish();
+        assert_eq!(format!("{}", stats.min.unwrap()), "5");
+        assert_eq!(format!("{}", stats.max.unwrap()), "10");
+    }
+
+    #[test]
+    fn test_buffer_memory() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+        let a1: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+        let a2: ArrayRef = Arc::new(Int32Array::from(vec![4, 5]));
+        let expected = a1.get_buffer_memory_size() + a2.get_buffer_memory_size();
+        acc.update(&a1).unwrap();
+        acc.update(&a2).unwrap();
+        let stats = acc.finish();
+        assert_eq!(stats.buffer_memory, expected as u64);
+        assert!(stats.buffer_memory > 0);
+    }
+
+    #[test]
+    fn test_buffer_memory_reset() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+        let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+        acc.update(&array).unwrap();
+        assert!(acc.statistics().buffer_memory > 0);
+        acc.reset();
+        assert_eq!(acc.statistics().buffer_memory, 0);
+    }
+
+    #[test]
+    fn test_non_list_item_nulls_is_none() {
+        let mut acc = StatisticsAccumulator::new(&DataType::Int32);
+        let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+        acc.update(&array).unwrap();
+        let stats = acc.finish();
+        assert_eq!(stats.item_nulls, None);
+    }
+
+    mod list_tests {
+        use super::*;
+        use arrow_array::builder::{Int32Builder, LargeListBuilder, ListBuilder};
+        use arrow_schema::Field;
+
+        fn list_data_type() -> DataType {
+            DataType::List(Arc::new(Field::new("item", DataType::Int32, true)))
+        }
+
+        fn large_list_data_type() -> DataType {
+            DataType::LargeList(Arc::new(Field::new("item", DataType::Int32, true)))
+        }
+
+        /// Build a ListArray from a slice of optional lists of optional i32.
+        fn build_list_array(rows: &[Option<&[Option<i32>]>]) -> ArrayRef {
+            let mut builder = ListBuilder::new(Int32Builder::new());
+            for row in rows {
+                match row {
+                    Some(items) => {
+                        for item in *items {
+                            match item {
+                                Some(v) => builder.values().append_value(*v),
+                                None => builder.values().append_null(),
+                            }
+                        }
+                        builder.append(true);
+                    }
+                    None => builder.append(false),
+                }
+            }
+            Arc::new(builder.finish())
+        }
+
+        /// Build a LargeListArray from a slice of optional lists of optional i32.
+        fn build_large_list_array(rows: &[Option<&[Option<i32>]>]) -> ArrayRef {
+            let mut builder = LargeListBuilder::new(Int32Builder::new());
+            for row in rows {
+                match row {
+                    Some(items) => {
+                        for item in *items {
+                            match item {
+                                Some(v) => builder.values().append_value(*v),
+                                None => builder.values().append_null(),
+                            }
+                        }
+                        builder.append(true);
+                    }
+                    None => builder.append(false),
+                }
+            }
+            Arc::new(builder.finish())
+        }
+
+        #[test]
+        fn test_list_basic() {
+            // [[1, 5], [3, 2, 4]]
+            let array = build_list_array(&[
+                Some(&[Some(1), Some(5)]),
+                Some(&[Some(3), Some(2), Some(4)]),
+            ]);
+            let mut acc = StatisticsAccumulator::new(&list_data_type());
+            acc.update(&array).unwrap();
+            let stats = acc.finish();
+            assert_eq!(format!("{}", stats.min.unwrap()), "1");
+            assert_eq!(format!("{}", stats.max.unwrap()), "5");
+            assert_eq!(stats.null_count, 0);
+            assert_eq!(stats.item_nulls, Some(0));
+            assert_eq!(stats.nan_count, None);
+        }
+
+        #[test]
+        fn test_list_with_null_items() {
+            // [[1, null, 5], [null, 3]]
+            let array =
+                build_list_array(&[Some(&[Some(1), None, Some(5)]), Some(&[None, Some(3)])]);
+            let mut acc = StatisticsAccumulator::new(&list_data_type());
+            acc.update(&array).unwrap();
+            let stats = acc.finish();
+            assert_eq!(format!("{}", stats.min.unwrap()), "1");
+            assert_eq!(format!("{}", stats.max.unwrap()), "5");
+            assert_eq!(stats.null_count, 0);
+            assert_eq!(stats.item_nulls, Some(2));
+        }
+
+        #[test]
+        fn test_list_with_null_lists() {
+            // [[1, 2], null, [3]]
+            let array = build_list_array(&[Some(&[Some(1), Some(2)]), None, Some(&[Some(3)])]);
+            let mut acc = StatisticsAccumulator::new(&list_data_type());
+            acc.update(&array).unwrap();
+            let stats = acc.finish();
+            assert_eq!(format!("{}", stats.min.unwrap()), "1");
+            assert_eq!(format!("{}", stats.max.unwrap()), "3");
+            assert_eq!(stats.null_count, 1);
+            assert_eq!(stats.item_nulls, Some(0));
+        }
+
+        #[test]
+        fn test_list_with_null_lists_and_null_items() {
+            // [[1, null], null, [null, 3]]
+            let array = build_list_array(&[Some(&[Some(1), None]), None, Some(&[None, Some(3)])]);
+            let mut acc = StatisticsAccumulator::new(&list_data_type());
+            acc.update(&array).unwrap();
+            let stats = acc.finish();
+            assert_eq!(format!("{}", stats.min.unwrap()), "1");
+            assert_eq!(format!("{}", stats.max.unwrap()), "3");
+            assert_eq!(stats.null_count, 1);
+            assert_eq!(stats.item_nulls, Some(2));
+        }
+
+        #[test]
+        fn test_list_all_null_lists() {
+            let array = build_list_array(&[None, None]);
+            let mut acc = StatisticsAccumulator::new(&list_data_type());
+            acc.update(&array).unwrap();
+            let stats = acc.finish();
+            assert!(stats.min.is_none());
+            assert!(stats.max.is_none());
+            assert_eq!(stats.null_count, 2);
+            assert_eq!(stats.item_nulls, Some(0));
+        }
+
+        #[test]
+        fn test_list_empty_lists() {
+            // [[], [1], []]
+            let array = build_list_array(&[Some(&[]), Some(&[Some(1)]), Some(&[])]);
+            let mut acc = StatisticsAccumulator::new(&list_data_type());
+            acc.update(&array).unwrap();
+            let stats = acc.finish();
+            assert_eq!(format!("{}", stats.min.unwrap()), "1");
+            assert_eq!(format!("{}", stats.max.unwrap()), "1");
+            assert_eq!(stats.null_count, 0);
+            assert_eq!(stats.item_nulls, Some(0));
+        }
+
+        #[test]
+        fn test_list_all_items_null() {
+            // [[null, null]]
+            let array = build_list_array(&[Some(&[None, None])]);
+            let mut acc = StatisticsAccumulator::new(&list_data_type());
+            acc.update(&array).unwrap();
+            let stats = acc.finish();
+            assert!(stats.min.is_none());
+            assert!(stats.max.is_none());
+            assert_eq!(stats.null_count, 0);
+            assert_eq!(stats.item_nulls, Some(2));
+        }
+
+        #[test]
+        fn test_list_multiple_updates() {
+            let a1 = build_list_array(&[Some(&[Some(5), Some(3)])]);
+            let a2 = build_list_array(&[Some(&[Some(1), None]), None, Some(&[Some(7)])]);
+            let mut acc = StatisticsAccumulator::new(&list_data_type());
+            acc.update(&a1).unwrap();
+            acc.update(&a2).unwrap();
+            let stats = acc.finish();
+            assert_eq!(format!("{}", stats.min.unwrap()), "1");
+            assert_eq!(format!("{}", stats.max.unwrap()), "7");
+            assert_eq!(stats.null_count, 1);
+            assert_eq!(stats.item_nulls, Some(1));
+        }
+
+        #[test]
+        fn test_list_merge() {
+            let a1 = build_list_array(&[Some(&[Some(1), Some(5)])]);
+            let a2 = build_list_array(&[Some(&[Some(3), None]), None, Some(&[Some(10)])]);
+
+            let mut acc1 = StatisticsAccumulator::new(&list_data_type());
+            acc1.update(&a1).unwrap();
+            let mut acc2 = StatisticsAccumulator::new(&list_data_type());
+            acc2.update(&a2).unwrap();
+
+            acc1.merge(&acc2).unwrap();
+            let stats = acc1.finish();
+            assert_eq!(format!("{}", stats.min.unwrap()), "1");
+            assert_eq!(format!("{}", stats.max.unwrap()), "10");
+            assert_eq!(stats.null_count, 1);
+            assert_eq!(stats.item_nulls, Some(1));
+        }
+
+        #[test]
+        fn test_list_reset() {
+            let array = build_list_array(&[Some(&[Some(1), None])]);
+            let mut acc = StatisticsAccumulator::new(&list_data_type());
+            acc.update(&array).unwrap();
+            acc.reset();
+            let stats = acc.finish();
+            assert!(stats.min.is_none());
+            assert!(stats.max.is_none());
+            assert_eq!(stats.null_count, 0);
+            assert_eq!(stats.item_nulls, Some(0));
+        }
+
+        #[test]
+        fn test_large_list() {
+            let array =
+                build_large_list_array(&[Some(&[Some(10), None, Some(1)]), None, Some(&[Some(5)])]);
+            let mut acc = StatisticsAccumulator::new(&large_list_data_type());
+            acc.update(&array).unwrap();
+            let stats = acc.finish();
+            assert_eq!(format!("{}", stats.min.unwrap()), "1");
+            assert_eq!(format!("{}", stats.max.unwrap()), "10");
+            assert_eq!(stats.null_count, 1);
+            assert_eq!(stats.item_nulls, Some(1));
+        }
+
+        /// Build a List<List<Int32>> array from nested slices.
+        ///
+        /// Each outer Option represents an outer list entry (None = null outer list).
+        /// Each inner Option<&[Option<i32>]> represents an inner list entry
+        /// (None = null inner list).
+        #[allow(clippy::type_complexity)]
+        fn build_nested_list_array(rows: &[Option<&[Option<&[Option<i32>]>]>]) -> ArrayRef {
+            let inner_builder = ListBuilder::new(Int32Builder::new());
+            let mut builder = ListBuilder::new(inner_builder);
+            for row in rows {
+                match row {
+                    Some(inner_lists) => {
+                        let inner_builder = builder.values();
+                        for inner_list in *inner_lists {
+                            match inner_list {
+                                Some(items) => {
+                                    for item in *items {
+                                        match item {
+                                            Some(v) => {
+                                                inner_builder.values().append_value(*v);
+                                            }
+                                            None => {
+                                                inner_builder.values().append_null();
+                                            }
+                                        }
+                                    }
+                                    inner_builder.append(true);
+                                }
+                                None => {
+                                    inner_builder.append(false);
+                                }
+                            }
+                        }
+                        builder.append(true);
+                    }
+                    None => builder.append(false),
+                }
+            }
+            Arc::new(builder.finish())
+        }
+
+        fn nested_list_data_type() -> DataType {
+            DataType::List(Arc::new(Field::new(
+                "item",
+                DataType::List(Arc::new(Field::new("item", DataType::Int32, true))),
+                true,
+            )))
+        }
+
+        #[test]
+        fn test_nested_list() {
+            // [[[1, 2], [3]], null, [[null, 5], null, [6]]]
+            let array = build_nested_list_array(&[
+                Some(&[Some(&[Some(1), Some(2)][..]), Some(&[Some(3)])]),
+                None,
+                Some(&[Some(&[None, Some(5)]), None, Some(&[Some(6)])]),
+            ]);
+
+            let mut acc = StatisticsAccumulator::new(&nested_list_data_type());
+            acc.update(&array).unwrap();
+            let stats = acc.finish();
+
+            // min/max should be computed across all leaf int32 values
+            assert_eq!(format!("{}", stats.min.unwrap()), "1");
+            assert_eq!(format!("{}", stats.max.unwrap()), "6");
+            // null_count: only the one null outer list
+            assert_eq!(stats.null_count, 1);
+            // item_nulls: 1 null int32 + 1 null inner list = 2
+            assert_eq!(stats.item_nulls, Some(2));
+        }
+    }
+
+    mod proptests {
+        use super::*;
+        use arrow_select::take::take;
+        use proptest::prelude::*;
+
+        /// Shuffle an array by applying a random permutation via the `take` kernel.
+        fn shuffle(array: &ArrayRef, permutation: &[usize]) -> ArrayRef {
+            let indices =
+                UInt32Array::from(permutation.iter().map(|&i| i as u32).collect::<Vec<_>>());
+            take(array.as_ref(), &indices, None).unwrap()
+        }
+
+        /// Compute stats for an array, returning (min, max) as Option<ArrowScalar>.
+        fn compute_stats(array: &ArrayRef) -> (Option<ArrowScalar>, Option<ArrowScalar>) {
+            let mut acc = StatisticsAccumulator::new(array.data_type());
+            acc.update(array).unwrap();
+            let stats = acc.finish();
+            (stats.min, stats.max)
+        }
+
+        macro_rules! prop_test_full {
+            ($name:ident, $array_ty:ty, $elem_strategy:expr) => {
+                proptest! {
+                    #[test]
+                    fn $name(
+                        values in proptest::collection::vec($elem_strategy, 1..100usize),
+                    ) {
+                        let len = values.len();
+                        let array: ArrayRef = Arc::new(<$array_ty>::from(values));
+                        let (orig_min, orig_max) = compute_stats(&array);
+
+                        // min <= max when both exist
+                        if let (Some(mn), Some(mx)) = (&orig_min, &orig_max) {
+                            prop_assert!(mn <= mx, "min {:?} > max {:?}", mn, mx);
+                        }
+
+                        // Reverse the array as a simple permutation
+                        let rev_indices: Vec<usize> = (0..len).rev().collect();
+                        let reversed = shuffle(&array, &rev_indices);
+                        let (rev_min, rev_max) = compute_stats(&reversed);
+                        prop_assert_eq!(&orig_min, &rev_min, "min changed after reverse");
+                        prop_assert_eq!(&orig_max, &rev_max, "max changed after reverse");
+                    }
+                }
+            };
+        }
+
+        macro_rules! prop_test_nullable_full {
+            ($name:ident, $array_ty:ty, $elem_strategy:expr) => {
+                proptest! {
+                    #[test]
+                    fn $name(
+                        values in proptest::collection::vec(
+                            proptest::option::of($elem_strategy), 1..100usize
+                        ),
+                    ) {
+                        let len = values.len();
+                        let array: ArrayRef = Arc::new(<$array_ty>::from(values));
+                        let (orig_min, orig_max) = compute_stats(&array);
+
+                        if let (Some(mn), Some(mx)) = (&orig_min, &orig_max) {
+                            prop_assert!(mn <= mx, "min {:?} > max {:?}", mn, mx);
+                        }
+
+                        let rev_indices: Vec<usize> = (0..len).rev().collect();
+                        let reversed = shuffle(&array, &rev_indices);
+                        let (rev_min, rev_max) = compute_stats(&reversed);
+                        prop_assert_eq!(&orig_min, &rev_min, "min changed after reverse");
+                        prop_assert_eq!(&orig_max, &rev_max, "max changed after reverse");
+
+                        // Also verify null_count and nan_count are invariant
+                        let mut acc_orig = StatisticsAccumulator::new(array.data_type());
+                        acc_orig.update(&array).unwrap();
+                        let mut acc_rev = StatisticsAccumulator::new(array.data_type());
+                        acc_rev.update(&reversed).unwrap();
+                        prop_assert_eq!(
+                            acc_orig.statistics().null_count,
+                            acc_rev.statistics().null_count,
+                            "null_count changed after shuffle"
+                        );
+                        prop_assert_eq!(
+                            acc_orig.statistics().nan_count,
+                            acc_rev.statistics().nan_count,
+                            "nan_count changed after shuffle"
+                        );
+                    }
+                }
+            };
+        }
+
+        // --- Integer types ---
+        prop_test_full!(prop_i32, Int32Array, any::<i32>());
+        prop_test_full!(prop_i64, Int64Array, any::<i64>());
+        prop_test_full!(prop_u32, UInt32Array, any::<u32>());
+        prop_test_full!(prop_u64, UInt64Array, any::<u64>());
+        prop_test_full!(prop_i8, Int8Array, any::<i8>());
+        prop_test_full!(prop_i16, Int16Array, any::<i16>());
+        prop_test_full!(prop_u8, UInt8Array, any::<u8>());
+        prop_test_full!(prop_u16, UInt16Array, any::<u16>());
+
+        // --- Nullable integer types ---
+        prop_test_nullable_full!(prop_i32_nullable, Int32Array, any::<i32>());
+        prop_test_nullable_full!(prop_i64_nullable, Int64Array, any::<i64>());
+        prop_test_nullable_full!(prop_u32_nullable, UInt32Array, any::<u32>());
+
+        // --- Float types (with NaN) ---
+        prop_test_full!(prop_f32, Float32Array, any::<f32>());
+        prop_test_full!(prop_f64, Float64Array, any::<f64>());
+        prop_test_nullable_full!(prop_f64_nullable, Float64Array, any::<f64>());
+
+        // --- String type ---
+        prop_test_full!(prop_string, StringArray, "[a-z]{0,20}");
+        prop_test_nullable_full!(prop_string_nullable, StringArray, "[a-z]{0,20}");
+
+        // --- Boolean type ---
+        prop_test_full!(prop_bool, BooleanArray, any::<bool>());
+        prop_test_nullable_full!(prop_bool_nullable, BooleanArray, any::<bool>());
+
+        // --- Random permutation shuffle test (uses prop_shuffle) ---
+        proptest! {
+            #[test]
+            fn prop_random_permutation_i32(
+                values in proptest::collection::vec(
+                    proptest::option::of(any::<i32>()), 1..100usize
+                ),
+            ) {
+                let len = values.len();
+                let array: ArrayRef = Arc::new(Int32Array::from(values));
+                let (orig_min, orig_max) = compute_stats(&array);
+
+                if let (Some(mn), Some(mx)) = (&orig_min, &orig_max) {
+                    prop_assert!(mn <= mx);
+                }
+
+                // Create and shuffle a permutation
+                let mut perm: Vec<usize> = (0..len).collect();
+                // Deterministic "shuffle" using a reversal + rotation
+                perm.reverse();
+                if len > 1 {
+                    perm.rotate_left(len / 2);
+                }
+
+                let shuffled = shuffle(&array, &perm);
+                let (shuf_min, shuf_max) = compute_stats(&shuffled);
+                prop_assert_eq!(&orig_min, &shuf_min);
+                prop_assert_eq!(&orig_max, &shuf_max);
+            }
+        }
+
+        proptest! {
+            /// Verify that splitting an array into two chunks and merging
+            /// the accumulators gives the same result as processing the
+            /// whole array at once.
+            #[test]
+            fn prop_merge_consistent_i32(
+                values in proptest::collection::vec(
+                    proptest::option::of(any::<i32>()), 2..100usize
+                ),
+            ) {
+                let array: ArrayRef = Arc::new(Int32Array::from(values.clone()));
+                let split = values.len() / 2;
+
+                let mut full_acc = StatisticsAccumulator::new(&DataType::Int32);
+                full_acc.update(&array).unwrap();
+
+                let left: ArrayRef = Arc::new(Int32Array::from(values[..split].to_vec()));
+                let right: ArrayRef = Arc::new(Int32Array::from(values[split..].to_vec()));
+                let mut left_acc = StatisticsAccumulator::new(&DataType::Int32);
+                left_acc.update(&left).unwrap();
+                let mut right_acc = StatisticsAccumulator::new(&DataType::Int32);
+                right_acc.update(&right).unwrap();
+                left_acc.merge(&right_acc).unwrap();
+
+                let full_stats = full_acc.finish();
+                let merged_stats = left_acc.finish();
+
+                prop_assert_eq!(&full_stats.min, &merged_stats.min);
+                prop_assert_eq!(&full_stats.max, &merged_stats.max);
+                prop_assert_eq!(full_stats.null_count, merged_stats.null_count);
+            }
+        }
+    }
+
+    #[test]
+    fn test_unsupported_type_tracks_null_count_and_memory() {
+        use arrow_array::builder::{Int32Builder, StructBuilder};
+        use arrow_schema::Field;
+
+        let fields = vec![Field::new("a", DataType::Int32, true)];
+        let mut builder = StructBuilder::new(fields, vec![Box::new(Int32Builder::new()) as _]);
+        for _ in 0..3 {
+            builder
+                .field_builder::<Int32Builder>(0)
+                .unwrap()
+                .append_null();
+            builder.append_null();
+        }
+        let struct_array: ArrayRef = Arc::new(builder.finish());
+
+        let dt = struct_array.data_type().clone();
+        let mut acc = StatisticsAccumulator::new(&dt);
+        acc.update(&struct_array).unwrap();
+
+        let stats = acc.finish();
+        assert!(stats.min.is_none());
+        assert!(stats.max.is_none());
+        assert_eq!(stats.null_count, 3);
+        assert_eq!(stats.nan_count, None);
+        assert!(stats.buffer_memory > 0);
+    }
+}
diff --git a/rust/arrow-stats/src/nan.rs b/rust/arrow-stats/src/nan.rs
new file mode 100644
index 00000000000..b8b2fd3e89c
--- /dev/null
+++ b/rust/arrow-stats/src/nan.rs
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use arrow_array::cast::AsArray;
+use arrow_array::types::{Float16Type, Float32Type, Float64Type};
+use arrow_array::{Array, ArrayRef};
+
+macro_rules! count_nans_typed {
+    ($array:expr, $arrow_type:ty) => {{
+        let typed = $array.as_primitive::<$arrow_type>();
+        let mut count = 0u64;
+        for i in 0..typed.len() {
+            if !typed.is_null(i) && typed.value(i).is_nan() {
+                count += 1;
+            }
+        }
+        count
+    }};
+}
+
+/// Count the number of non-null NaN values in an array.
+///
+/// Returns 0 for non-float types.
+pub fn count_nans(array: &ArrayRef) -> u64 {
+    use arrow_schema::DataType::*;
+    match array.data_type() {
+        Float16 => count_nans_typed!(array, Float16Type),
+        Float32 => count_nans_typed!(array, Float32Type),
+        Float64 => count_nans_typed!(array, Float64Type),
+        _ => 0,
+    }
+}
diff --git a/rust/examples/src/full_text_search.rs b/rust/examples/src/full_text_search.rs
index 22e701c5863..8269f590ee8 100644
--- a/rust/examples/src/full_text_search.rs
+++ b/rust/examples/src/full_text_search.rs
@@ -15,8 +15,8 @@ use arrow::datatypes::UInt64Type;
 use arrow_schema::{DataType, Field, Schema};
 use itertools::Itertools;
 use lance::Dataset;
+use lance::index::DatasetIndexExt;
 use lance_datagen::{RowCount, array};
-use lance_index::DatasetIndexExt;
 use lance_index::scalar::inverted::flat_full_text_search;
 use lance_index::scalar::{FullTextSearchQuery, InvertedIndexParams};
 use object_store::path::Path;
diff --git a/rust/examples/src/ivf_hnsw.rs b/rust/examples/src/ivf_hnsw.rs
index 296c03b19d6..c1898e10682 100644
--- a/rust/examples/src/ivf_hnsw.rs
+++ b/rust/examples/src/ivf_hnsw.rs
@@ -11,11 +11,12 @@ use clap::Parser;
 use futures::TryStreamExt;
 use lance::Dataset;
 use lance::dataset::ProjectionRequest;
+use lance::index::DatasetIndexExt;
 use lance::index::vector::VectorIndexParams;
+use lance_index::IndexType;
 use lance_index::vector::hnsw::builder::HnswBuildParams;
 use lance_index::vector::ivf::IvfBuildParams;
 use lance_index::vector::sq::builder::SQBuildParams;
-use lance_index::{DatasetIndexExt, IndexType};
 use lance_linalg::distance::MetricType;
 
 #[derive(Parser, Debug)]
diff --git a/rust/lance-arrow/Cargo.toml b/rust/lance-arrow/Cargo.toml
index a9f03cfcb28..5ec6904617f 100644
--- a/rust/lance-arrow/Cargo.toml
+++ b/rust/lance-arrow/Cargo.toml
@@ -18,6 +18,7 @@ arrow-array = { workspace = true }
 arrow-buffer = { workspace = true }
 arrow-data = { workspace = true }
 arrow-cast = { workspace = true }
+arrow-ipc = { workspace = true }
 arrow-ord = { workspace = true }
 arrow-schema = { workspace = true }
 arrow-select = { workspace = true }
diff --git a/rust/lance-arrow/src/cast.rs b/rust/lance-arrow/src/cast.rs
deleted file mode 100644
index 0fa4273ceea..00000000000
--- a/rust/lance-arrow/src/cast.rs
+++ /dev/null
@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The Lance Authors
-
-use std::sync::Arc;
-
-use arrow_array::{Array, ArrayRef, FixedSizeListArray};
-use arrow_cast::CastOptions;
-use arrow_schema::{ArrowError, DataType};
-
-/// Customized [`arrow_cast::cast_with_options`] that handles cases not supported upstream yet.
-pub fn cast_with_options(
-    array: &dyn Array,
-    to_type: &DataType,
-    cast_options: &CastOptions,
-) -> Result<ArrayRef, ArrowError> {
-    use DataType::*;
-    match (array.data_type(), to_type) {
-        (FixedSizeList(_, size_from), FixedSizeList(to_field, size_to)) if size_from == size_to => {
-            let array = array.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
-            let values = cast_with_options(array.values(), to_field.data_type(), cast_options)?;
-            Ok(Arc::new(FixedSizeListArray::try_new(
-                to_field.clone(),
-                *size_from,
-                values,
-                array.nulls().cloned(),
-            )?))
-        }
-        _ => arrow_cast::cast_with_options(array, to_type, cast_options),
-    }
-}
diff --git a/rust/lance-arrow/src/deepcopy.rs b/rust/lance-arrow/src/deepcopy.rs
index a82a64aa872..b747b24d466 100644
--- a/rust/lance-arrow/src/deepcopy.rs
+++ b/rust/lance-arrow/src/deepcopy.rs
@@ -94,7 +94,7 @@ pub fn deep_copy_batch_sliced(batch: &RecordBatch) -> crate::Result<RecordBatch>
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
     use std::sync::Arc;
 
     use arrow_array::{Array, Int32Array, RecordBatch, StringArray};
diff --git a/rust/lance-arrow/src/ipc.rs b/rust/lance-arrow/src/ipc.rs
new file mode 100644
index 00000000000..1c6364c4525
--- /dev/null
+++ b/rust/lance-arrow/src/ipc.rs
@@ -0,0 +1,406 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Zero-copy Arrow IPC stream read/write utilities.
+//!
+//! Provides helpers for serializing and deserializing [`RecordBatch`]es as
+//! self-delimiting Arrow IPC streams using synchronous [`Read`]/[`Write`] I/O.
+//!
+//! These are designed for embedding IPC streams inside larger binary formats
+//! (e.g. a cache entry that contains multiple IPC sections). Each stream is
+//! self-delimiting (schema + batches + EOS marker) and can be read back
+//! independently.
+//!
+//! # Zero-copy reads
+//!
+//! [`read_ipc_stream`] and [`read_ipc_stream_single`] take `&Bytes` and use
+//! [`Bytes::slice`] to produce each message buffer. Because `Bytes::slice`
+//! increments a reference count rather than copying, the resulting
+//! [`Buffer`]s — and the array data decoded from them by [`FileDecoder`] —
+//! are all backed by the same allocation as the input.
+
+use std::io::{Read, Write};
+use std::sync::Arc;
+
+use arrow_array::RecordBatch;
+use arrow_buffer::Buffer;
+use arrow_ipc::convert::fb_to_schema;
+use arrow_ipc::reader::FileDecoder;
+use arrow_ipc::root_as_message;
+use arrow_ipc::writer::StreamWriter;
+use arrow_schema::ArrowError;
+use bytes::Bytes;
+
+// ---------------------------------------------------------------------------
+// Length-prefixed byte utilities
+// ---------------------------------------------------------------------------
+
+/// Write `data` prefixed by its length as a little-endian `u64`.
+///
+/// Paired with [`read_len_prefixed_bytes`].
+pub fn write_len_prefixed_bytes(writer: &mut dyn Write, data: &[u8]) -> Result<(), ArrowError> {
+    writer
+        .write_all(&(data.len() as u64).to_le_bytes())
+        .map_err(|e| ArrowError::IoError(e.to_string(), e))?;
+    writer
+        .write_all(data)
+        .map_err(|e| ArrowError::IoError(e.to_string(), e))
+}
+
+/// Read a byte slice written by [`write_len_prefixed_bytes`].
+///
+/// Reads an 8-byte little-endian length then exactly that many bytes.
+pub fn read_len_prefixed_bytes(reader: &mut dyn Read) -> Result<Vec<u8>, ArrowError> {
+    let mut len_buf = [0u8; 8];
+    reader
+        .read_exact(&mut len_buf)
+        .map_err(|e| ArrowError::IoError(e.to_string(), e))?;
+    let len = u64::from_le_bytes(len_buf) as usize;
+    let mut buf = vec![0u8; len];
+    reader
+        .read_exact(&mut buf)
+        .map_err(|e| ArrowError::IoError(e.to_string(), e))?;
+    Ok(buf)
+}
+
+// ---------------------------------------------------------------------------
+// IPC stream utilities
+// ---------------------------------------------------------------------------
+
+// 4-byte continuation marker used by modern Arrow IPC streams.
+const IPC_CONTINUATION: [u8; 4] = [0xff; 4];
+
+/// Write `batch` as a single-batch Arrow IPC stream to `writer`.
+pub fn write_ipc_stream(batch: &RecordBatch, writer: &mut dyn Write) -> Result<(), ArrowError> {
+    let mut sw = StreamWriter::try_new(&mut *writer, batch.schema_ref())?;
+    sw.write(batch)?;
+    sw.finish()
+}
+
+/// Write all batches from `iter` as a single Arrow IPC stream to `writer`.
+///
+/// `iter` must yield at least one batch; the schema is inferred from the first
+/// batch. Returns `ArrowError::InvalidArgumentError` if the iterator is empty.
+/// If you need to write an empty stream (schema only, no rows), construct a
+/// `StreamWriter` directly.
+pub fn write_ipc_stream_batches<I>(iter: I, writer: &mut dyn Write) -> Result<(), ArrowError>
+where
+    I: IntoIterator<Item = RecordBatch>,
+{
+    let mut iter = iter.into_iter();
+    let first = iter
+        .next()
+        .ok_or_else(|| ArrowError::InvalidArgumentError("no batches to serialize".into()))?;
+    let mut sw = StreamWriter::try_new(&mut *writer, first.schema_ref())?;
+    sw.write(&first)?;
+    for batch in iter {
+        sw.write(&batch)?;
+    }
+    sw.finish()
+}
+
+/// Read one complete Arrow IPC stream message from `data` as a zero-copy [`Buffer`].
+///
+/// Parses the first message starting at byte 0 of `data`. Returns `None` on
+/// EOS (size field == 0) or empty input. The returned [`Buffer`] is backed by
+/// `data`'s allocation — no bytes are copied.
+///
+/// The caller should advance its position by `buf.len()` after each call.
+fn read_one_ipc_message(data: &Bytes) -> Result<Option<Buffer>, ArrowError> {
+    let bytes = data.as_ref();
+
+    if bytes.is_empty() {
+        return Ok(None);
+    }
+    if bytes.len() < 4 {
+        return Err(ArrowError::IoError(
+            "IPC: truncated header".into(),
+            std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC header"),
+        ));
+    }
+
+    let has_continuation = bytes[..4] == IPC_CONTINUATION;
+    let (size_bytes, prefix_len): ([u8; 4], usize) = if has_continuation {
+        if bytes.len() < 8 {
+            return Err(ArrowError::IoError(
+                "IPC: truncated header after continuation".into(),
+                std::io::Error::new(
+                    std::io::ErrorKind::UnexpectedEof,
+                    "truncated after continuation",
+                ),
+            ));
+        }
+        (bytes[4..8].try_into().unwrap(), 8)
+    } else {
+        (bytes[..4].try_into().unwrap(), 4)
+    };
+
+    let meta_size = u32::from_le_bytes(size_bytes) as usize;
+    if meta_size == 0 {
+        return Ok(None); // EOS
+    }
+
+    let meta_end = prefix_len + meta_size;
+    if bytes.len() < meta_end {
+        return Err(ArrowError::IoError(
+            "IPC: truncated metadata".into(),
+            std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC metadata"),
+        ));
+    }
+
+    let msg = root_as_message(&bytes[prefix_len..meta_end])
+        .map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?;
+    let body_len = msg.bodyLength() as usize;
+
+    let total = meta_end + body_len;
+    if bytes.len() < total {
+        return Err(ArrowError::IoError(
+            "IPC: truncated body".into(),
+            std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC body"),
+        ));
+    }
+
+    // Zero-copy: Bytes::slice shares the backing allocation; Buffer::from
+    // wraps it without copying.
+    Ok(Some(Buffer::from(data.slice(0..total))))
+}
+
+/// Read a length-prefixed byte slice at `offset` in `data`, advancing `offset`.
+///
+/// Reads an 8-byte little-endian length, then slices exactly that many bytes
+/// from `data`. The returned [`Bytes`] is zero-copy (shares `data`'s allocation).
+pub fn read_len_prefixed_bytes_at(data: &Bytes, offset: &mut usize) -> Result<Bytes, ArrowError> {
+    let bytes = data.as_ref();
+    let len_end = offset
+        .checked_add(8)
+        .filter(|&e| e <= bytes.len())
+        .ok_or_else(|| {
+            ArrowError::IoError(
+                "length-prefixed bytes: truncated length field".into(),
+                std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated length"),
+            )
+        })?;
+    let len = u64::from_le_bytes(bytes[*offset..len_end].try_into().unwrap()) as usize;
+    *offset = len_end;
+    let data_end = offset
+        .checked_add(len)
+        .filter(|&e| e <= bytes.len())
+        .ok_or_else(|| {
+            ArrowError::IoError(
+                "length-prefixed bytes: truncated data".into(),
+                std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated data"),
+            )
+        })?;
+    let result = data.slice(*offset..data_end);
+    *offset = data_end;
+    Ok(result)
+}
+
+/// Read all [`RecordBatch`]es from one Arrow IPC stream starting at `offset`,
+/// advancing `offset` past the stream (including the EOS marker).
+///
+/// Zero-copy: array buffers borrow from `data`'s allocation.
+pub fn read_ipc_stream_at(
+    data: &Bytes,
+    offset: &mut usize,
+) -> Result<Vec<RecordBatch>, ArrowError> {
+    let batches = read_ipc_stream(&data.slice(*offset..))?;
+
+    // Recompute how many bytes were consumed by re-parsing message sizes.
+    // We can't get this from read_ipc_stream directly, so we re-walk the
+    // message headers (metadata only, no body re-read) to sum up lengths.
+    let slice = &data.as_ref()[*offset..];
+    let mut consumed = 0usize;
+    loop {
+        let rem = &slice[consumed..];
+        if rem.is_empty() {
+            break;
+        }
+        let has_cont = rem.len() >= 4 && rem[..4] == IPC_CONTINUATION;
+        let (size_bytes, prefix_len): ([u8; 4], usize) = if has_cont {
+            if rem.len() < 8 {
+                break;
+            }
+            (rem[4..8].try_into().unwrap(), 8)
+        } else {
+            if rem.len() < 4 {
+                break;
+            }
+            (rem[..4].try_into().unwrap(), 4)
+        };
+        let meta_size = u32::from_le_bytes(size_bytes) as usize;
+        if meta_size == 0 {
+            // EOS — consume it and stop.
+            consumed += prefix_len;
+            break;
+        }
+        let meta_end = prefix_len + meta_size;
+        if rem.len() < meta_end {
+            break;
+        }
+        let msg = root_as_message(&rem[prefix_len..meta_end])
+            .map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?;
+        let body_len = msg.bodyLength() as usize;
+        consumed += meta_end + body_len;
+    }
+    *offset += consumed;
+
+    Ok(batches)
+}
+
+/// Read exactly one [`RecordBatch`] from one Arrow IPC stream starting at `offset`,
+/// advancing `offset` past the stream (including the EOS marker).
+///
+/// Zero-copy: array buffers borrow from `data`'s allocation.
+pub fn read_ipc_stream_single_at(
+    data: &Bytes,
+    offset: &mut usize,
+) -> Result<RecordBatch, ArrowError> {
+    let mut batches = read_ipc_stream_at(data, offset)?;
+    match batches.len() {
+        1 => Ok(batches.remove(0)),
+        n => Err(ArrowError::ParseError(format!(
+            "expected exactly 1 IPC record batch, got {n}"
+        ))),
+    }
+}
+
+/// Extract the prefix length and metadata size from a raw IPC message buffer.
+///
+/// Modern IPC streams have an 8-byte prefix `[continuation: 4][size: 4]`.
+/// Legacy streams have a 4-byte prefix `[size: 4]`. Returns `(prefix_len, meta_size)`.
+fn parse_ipc_message_prefix(buf: &Buffer) -> Result<(usize, usize), ArrowError> {
+    let has_continuation = buf.len() >= 4 && buf[..4] == [0xff; 4];
+    if has_continuation {
+        if buf.len() < 8 {
+            return Err(ArrowError::ParseError(
+                "IPC message buffer too short".into(),
+            ));
+        }
+        let meta_size = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize;
+        Ok((8, meta_size))
+    } else {
+        if buf.len() < 4 {
+            return Err(ArrowError::ParseError(
+                "IPC message buffer too short".into(),
+            ));
+        }
+        let meta_size = u32::from_le_bytes(buf[..4].try_into().unwrap()) as usize;
+        Ok((4, meta_size))
+    }
+}
+
+/// Read all [`RecordBatch`]es from one Arrow IPC stream.
+///
+/// Zero-copy: each batch's array data buffers are borrowed from the input
+/// message buffer(s) and not copied during decoding.
+///
+/// Uses [`FileDecoder`] directly (rather than `StreamDecoder`) to avoid a
+/// known edge case where `StreamDecoder` does not produce a batch for messages
+/// with a zero-length body when the message exactly fills the decode buffer.
+pub fn read_ipc_stream(data: &Bytes) -> Result<Vec<RecordBatch>, ArrowError> {
+    let mut offset = 0usize;
+
+    let schema_buf = read_one_ipc_message(&data.slice(offset..))?.ok_or_else(|| {
+        ArrowError::ParseError("IPC stream: expected schema message, got EOS".into())
+    })?;
+    offset += schema_buf.len();
+
+    let (prefix_len, meta_size) = parse_ipc_message_prefix(&schema_buf)?;
+    let schema_msg = root_as_message(&schema_buf[prefix_len..prefix_len + meta_size])
+        .map_err(|e| ArrowError::ParseError(format!("IPC schema parse error: {e}")))?;
+    let schema = Arc::new(fb_to_schema(schema_msg.header_as_schema().ok_or_else(
+        || ArrowError::ParseError("IPC stream: first message is not a schema".into()),
+    )?));
+    let mut decoder = FileDecoder::new(schema, schema_msg.version());
+
+    let mut batches = Vec::new();
+
+    loop {
+        let Some(buf) = read_one_ipc_message(&data.slice(offset..))? else {
+            break;
+        };
+        offset += buf.len();
+
+        let (prefix_len, meta_size) = parse_ipc_message_prefix(&buf)?;
+        let msg = root_as_message(&buf[prefix_len..prefix_len + meta_size])
+            .map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?;
+        let body_len = msg.bodyLength() as usize;
+
+        // Block offset = 0 since the buffer starts at the message boundary.
+        // metaDataLength = prefix_len + meta_size (prefix + flatbuf + padding).
+        let block = arrow_ipc::Block::new(0, (prefix_len + meta_size) as i32, body_len as i64);
+
+        match msg.header_type() {
+            arrow_ipc::MessageHeader::RecordBatch => {
+                if let Some(batch) = decoder.read_record_batch(&block, &buf)? {
+                    batches.push(batch);
+                }
+            }
+            arrow_ipc::MessageHeader::DictionaryBatch => {
+                decoder.read_dictionary(&block, &buf)?;
+            }
+            _ => break,
+        }
+    }
+
+    Ok(batches)
+}
+
+/// Read exactly one [`RecordBatch`] from one Arrow IPC stream.
+pub fn read_ipc_stream_single(data: &Bytes) -> Result<RecordBatch, ArrowError> {
+    let mut batches = read_ipc_stream(data)?;
+    match batches.len() {
+        1 => Ok(batches.remove(0)),
+        n => Err(ArrowError::ParseError(format!(
+            "expected exactly 1 IPC record batch, got {n}"
+        ))),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use arrow_array::{ArrayRef, record_batch};
+
+    use super::*;
+
+    #[test]
+    fn test_ipc_roundtrip() {
+        let batch1 = record_batch!(
+            ("int", Int32, [1, 2, 3]),
+            ("str", Utf8, ["foo", "bar", "baz"])
+        )
+        .unwrap();
+        let batch2 = record_batch!(("int", Int32, [4, 5]), ("str", Utf8, ["qux", "quux"])).unwrap();
+        let batches = vec![batch1.clone(), batch2.clone()];
+
+        let mut buf = Vec::new();
+        write_ipc_stream_batches(batches, &mut buf).unwrap();
+
+        let data = Bytes::from(buf);
+
+        let batches = read_ipc_stream(&data).unwrap();
+        assert_eq!(batches.len(), 2);
+        assert_eq!(batches[0], batch1);
+        assert_eq!(batches[1], batch2);
+
+        let data_base = data.as_ptr() as usize;
+        let data_end = data_base + data.len();
+        let assert_col_zero_copy = |array: &ArrayRef| {
+            for buffer in array.to_data().buffers() {
+                let ptr = buffer.as_ptr() as usize;
+                assert!(
+                    ptr >= data_base && ptr < data_end,
+                    "buffer at {ptr:#x} is not backed by the input Bytes allocation \
+                     [{data_base:#x}..{data_end:#x})"
+                );
+            }
+        };
+
+        for batch in &batches {
+            assert_eq!(batch.schema(), batch1.schema());
+            assert_col_zero_copy(batch.column(0));
+            assert_col_zero_copy(batch.column(1));
+        }
+    }
+}
diff --git a/rust/lance-arrow/src/lib.rs b/rust/lance-arrow/src/lib.rs
index 3bb77076cb3..f27696f64a9 100644
--- a/rust/lance-arrow/src/lib.rs
+++ b/rust/lance-arrow/src/lib.rs
@@ -30,7 +30,7 @@ pub mod floats;
 use crate::list::ListArrayExt;
 pub use floats::*;
 
-pub mod cast;
+pub mod ipc;
 pub mod json;
 pub mod list;
 pub mod memory;
@@ -1154,6 +1154,12 @@ fn adjust_child_validity(
         Some(p) => p,
     };
 
+    // Fast path: DataType::Null arrays are always entirely null by definition and cannot
+    // carry an explicit null bitmap (Arrow rejects it). No adjustment is needed.
+    if child.data_type() == &DataType::Null {
+        return child.clone();
+    }
+
     let child_validity = child.nulls();
 
     // Compute the new validity: child_validity AND parent_validity
@@ -1559,7 +1565,7 @@ impl BufferExt for arrow_buffer::Buffer {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use arrow_array::{Float32Array, Int32Array, StructArray};
+    use arrow_array::{Float32Array, Int32Array, NullArray, StructArray};
     use arrow_array::{ListArray, StringArray, new_empty_array, new_null_array};
     use arrow_buffer::OffsetBuffer;
 
@@ -1986,6 +1992,31 @@ mod tests {
         assert!(width_values.is_null(2)); // width is null when right struct was null
     }
 
+    #[test]
+    fn test_merge_null_typed_column_with_parent_validity() {
+        // Reproduces ENT-990: panic in adjust_child_validity when a Null-typed column
+        // exists on one side and the parent struct has null rows.
+        // Arrow's Null type has no null bitmap, so passing one to ArrayData::try_new panics.
+        let left_struct = StructArray::new(
+            Fields::from(vec![Field::new("a", DataType::Int32, true)]),
+            vec![Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef],
+            Some(vec![true, false].into()),
+        );
+        let right_struct = StructArray::new(
+            Fields::from(vec![Field::new("b", DataType::Null, true)]),
+            vec![Arc::new(NullArray::new(2)) as ArrayRef],
+            Some(vec![true, false].into()),
+        );
+
+        // Previously panicked: "Arrays of type Null cannot contain a null bitmask"
+        let merged = merge(&left_struct, &right_struct);
+        assert_eq!(merged.len(), 2);
+        let b_col = merged.column_by_name("b").unwrap();
+        // DataType::Null implies all-null by definition; no null bitmap is stored.
+        assert_eq!(b_col.data_type(), &DataType::Null);
+        assert_eq!(b_col.len(), 2);
+    }
+
     #[test]
     fn test_merge_with_schema_with_nullable_struct_list_schema_mismatch() {
         // left_list setup
diff --git a/rust/lance-arrow/src/stream.rs b/rust/lance-arrow/src/stream.rs
index 37ecd05663f..18f532d0519 100644
--- a/rust/lance-arrow/src/stream.rs
+++ b/rust/lance-arrow/src/stream.rs
@@ -41,7 +41,9 @@ where
             }
 
             // Pull batches until we reach the byte target or exhaust input.
-            while !state.done && state.acc_bytes < state.min_bytes {
+            // Always pull at least one batch so that min_bytes=0 works.
+            while !state.done && (state.accumulated.is_empty() || state.acc_bytes < state.min_bytes)
+            {
                 match state.input.next().await {
                     Some(Ok(batch)) => {
                         state.acc_bytes += batch.get_array_memory_size();
@@ -291,6 +293,30 @@ mod tests {
         assert_eq!(values, expected);
     }
 
+    #[test]
+    fn test_min_bytes_zero_still_yields_all_rows() {
+        // When min_bytes=0, the stream should still yield every batch.
+        // This is the "chop only, don't coalesce" use case.
+        let batches: Vec<_> = (0..5).map(|_| make_batch(100)).collect();
+        let batch_bytes = batches[0].get_array_memory_size();
+        let result = collect_rechunked(batches, 0, batch_bytes * 2);
+        assert_eq!(total_rows(&result), 500);
+    }
+
+    #[test]
+    fn test_min_bytes_zero_slices_oversized() {
+        // min_bytes=0 with a small max_bytes should still slice large batches.
+        let batch = make_batch(1000);
+        let bytes = batch.get_array_memory_size();
+        let result = collect_rechunked(vec![batch], 0, bytes / 4);
+        assert_eq!(total_rows(&result), 1000);
+        assert!(
+            result.len() >= 4,
+            "expected at least 4 slices, got {}",
+            result.len()
+        );
+    }
+
     #[test]
     fn test_error_propagation() {
         let input = stream::iter(vec![
diff --git a/rust/lance-core/src/cache.rs b/rust/lance-core/src/cache.rs
deleted file mode 100644
index 6ceea807116..00000000000
--- a/rust/lance-core/src/cache.rs
+++ /dev/null
@@ -1,806 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The Lance Authors
-
-//! Cache implementation
-
-use std::any::{Any, TypeId};
-use std::borrow::Cow;
-use std::sync::{
-    Arc,
-    atomic::{AtomicU64, Ordering},
-};
-
-use futures::{Future, FutureExt};
-use moka::future::Cache;
-
-use crate::Result;
-
-pub use deepsize::{Context, DeepSizeOf};
-
-type ArcAny = Arc<dyn Any + Send + Sync>;
-
-#[derive(Clone)]
-pub struct SizedRecord {
-    record: ArcAny,
-    size_accessor: Arc<dyn Fn(&ArcAny) -> usize + Send + Sync>,
-}
-
-impl std::fmt::Debug for SizedRecord {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("SizedRecord")
-            .field("record", &self.record)
-            .finish()
-    }
-}
-
-impl DeepSizeOf for SizedRecord {
-    fn deep_size_of_children(&self, _: &mut Context) -> usize {
-        (self.size_accessor)(&self.record)
-    }
-}
-
-impl SizedRecord {
-    fn new<T: DeepSizeOf + Send + Sync + 'static>(record: Arc<T>) -> Self {
-        // +8 for the size of the Arc pointer itself
-        let size_accessor =
-            |record: &ArcAny| -> usize { record.downcast_ref::<T>().unwrap().deep_size_of() + 8 };
-        Self {
-            record,
-            size_accessor: Arc::new(size_accessor),
-        }
-    }
-}
-
-#[derive(Clone)]
-pub struct LanceCache {
-    cache: Arc<Cache<(String, TypeId), SizedRecord>>,
-    prefix: String,
-    hits: Arc<AtomicU64>,
-    misses: Arc<AtomicU64>,
-}
-
-impl std::fmt::Debug for LanceCache {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("LanceCache")
-            .field("cache", &self.cache)
-            .finish()
-    }
-}
-
-impl DeepSizeOf for LanceCache {
-    fn deep_size_of_children(&self, _: &mut Context) -> usize {
-        self.cache
-            .iter()
-            .map(|(_, v)| (v.size_accessor)(&v.record))
-            .sum()
-    }
-}
-
-impl LanceCache {
-    pub fn with_capacity(capacity: usize) -> Self {
-        let cache = Cache::builder()
-            .max_capacity(capacity as u64)
-            .weigher(|_, v: &SizedRecord| {
-                (v.size_accessor)(&v.record).try_into().unwrap_or(u32::MAX)
-            })
-            .support_invalidation_closures()
-            .build();
-        Self {
-            cache: Arc::new(cache),
-            prefix: String::new(),
-            hits: Arc::new(AtomicU64::new(0)),
-            misses: Arc::new(AtomicU64::new(0)),
-        }
-    }
-
-    pub fn no_cache() -> Self {
-        Self {
-            cache: Arc::new(Cache::new(0)),
-            prefix: String::new(),
-            hits: Arc::new(AtomicU64::new(0)),
-            misses: Arc::new(AtomicU64::new(0)),
-        }
-    }
-
-    /// Appends a prefix to the cache key
-    ///
-    /// If this cache already has a prefix, the new prefix will be appended to
-    /// the existing one.
-    ///
-    /// Prefixes are used to create a namespace for the cache keys to avoid
-    /// collisions between different caches.
-    pub fn with_key_prefix(&self, prefix: &str) -> Self {
-        Self {
-            cache: self.cache.clone(),
-            prefix: format!("{}{}/", self.prefix, prefix),
-            hits: self.hits.clone(),
-            misses: self.misses.clone(),
-        }
-    }
-
-    fn get_key(&self, key: &str) -> String {
-        if self.prefix.is_empty() {
-            key.to_string()
-        } else {
-            format!("{}/{}", self.prefix, key)
-        }
-    }
-
-    /// Invalidate all entries in the cache that start with the given prefix
-    ///
-    /// The given prefix is appended to the existing prefix of the cache. If you
-    /// want to invalidate all at the current prefix, pass an empty string.
-    pub fn invalidate_prefix(&self, prefix: &str) {
-        let full_prefix = format!("{}{}", self.prefix, prefix);
-        self.cache
-            .invalidate_entries_if(move |(key, _typeid), _value| key.starts_with(&full_prefix))
-            .expect("Cache configured correctly");
-    }
-
-    pub async fn size(&self) -> usize {
-        self.cache.run_pending_tasks().await;
-        self.cache.entry_count() as usize
-    }
-
-    pub fn approx_size(&self) -> usize {
-        self.cache.entry_count() as usize
-    }
-
-    pub async fn size_bytes(&self) -> usize {
-        self.cache.run_pending_tasks().await;
-        self.approx_size_bytes()
-    }
-
-    pub fn approx_size_bytes(&self) -> usize {
-        self.cache.weighted_size() as usize
-    }
-
-    async fn insert<T: DeepSizeOf + Send + Sync + 'static>(&self, key: &str, metadata: Arc<T>) {
-        let key = self.get_key(key);
-        let record = SizedRecord::new(metadata);
-        tracing::trace!(
-            target: "lance_cache::insert",
-            key = key,
-            type_id = std::any::type_name::<T>(),
-            size = (record.size_accessor)(&record.record),
-        );
-        self.cache.insert((key, TypeId::of::<T>()), record).await;
-    }
-
-    pub async fn insert_unsized<T: DeepSizeOf + Send + Sync + 'static + ?Sized>(
-        &self,
-        key: &str,
-        metadata: Arc<T>,
-    ) {
-        // In order to make the data Sized, we wrap in another pointer.
-        self.insert(key, Arc::new(metadata)).await
-    }
-
-    async fn get<T: DeepSizeOf + Send + Sync + 'static>(&self, key: &str) -> Option<Arc<T>> {
-        let key = self.get_key(key);
-        if let Some(metadata) = self.cache.get(&(key, TypeId::of::<T>())).await {
-            self.hits.fetch_add(1, Ordering::Relaxed);
-            Some(metadata.record.clone().downcast::<T>().unwrap())
-        } else {
-            self.misses.fetch_add(1, Ordering::Relaxed);
-            None
-        }
-    }
-
-    pub async fn get_unsized<T: DeepSizeOf + Send + Sync + 'static + ?Sized>(
-        &self,
-        key: &str,
-    ) -> Option<Arc<T>> {
-        let outer = self.get::<Arc<T>>(key).await?;
-        Some(outer.as_ref().clone())
-    }
-
-    /// Get an item
-    ///
-    /// If it exists in the cache return that
-    ///
-    /// If it doesn't then run `loader` to load the item, insert into cache, and return
-    async fn get_or_insert<T: DeepSizeOf + Send + Sync + 'static, F, Fut>(
-        &self,
-        key: String,
-        loader: F,
-    ) -> Result<Arc<T>>
-    where
-        F: FnOnce(&str) -> Fut,
-        Fut: Future<Output = Result<T>> + Send,
-    {
-        let full_key = self.get_key(&key);
-        let cache_key = (full_key, TypeId::of::<T>());
-
-        // Use optionally_get_with to handle concurrent requests
-        let hits = self.hits.clone();
-        let misses = self.misses.clone();
-
-        // Use oneshot channels to track both errors and whether init was run
-        let (error_tx, error_rx) = tokio::sync::oneshot::channel();
-        let (init_run_tx, mut init_run_rx) = tokio::sync::oneshot::channel();
-
-        let init = Box::pin(async move {
-            let _ = init_run_tx.send(());
-            misses.fetch_add(1, Ordering::Relaxed);
-            match loader(&key).await {
-                Ok(value) => Some(SizedRecord::new(Arc::new(value))),
-                Err(e) => {
-                    let _ = error_tx.send(e);
-                    None
-                }
-            }
-        });
-
-        match self.cache.optionally_get_with(cache_key, init).await {
-            Some(metadata) => {
-                // Check if init was run or if this was a cache hit
-                match init_run_rx.try_recv() {
-                    Ok(()) => {
-                        // Init was run, miss was already recorded
-                    }
-                    Err(_) => {
-                        // Init was not run, this is a cache hit
-                        hits.fetch_add(1, Ordering::Relaxed);
-                    }
-                }
-                Ok(metadata.record.clone().downcast::<T>().unwrap())
-            }
-            None => {
-                // The loader returned an error, retrieve it from the channel
-                match error_rx.await {
-                    Ok(err) => Err(err),
-                    Err(_) => Err(crate::Error::internal(
-                        "Failed to retrieve error from cache loader",
-                    )),
-                }
-            }
-        }
-    }
-
-    pub async fn stats(&self) -> CacheStats {
-        self.cache.run_pending_tasks().await;
-        CacheStats {
-            hits: self.hits.load(Ordering::Relaxed),
-            misses: self.misses.load(Ordering::Relaxed),
-            num_entries: self.cache.entry_count() as usize,
-            size_bytes: self.cache.weighted_size() as usize,
-        }
-    }
-
-    pub async fn clear(&self) {
-        self.cache.invalidate_all();
-        self.cache.run_pending_tasks().await;
-        self.hits.store(0, Ordering::Relaxed);
-        self.misses.store(0, Ordering::Relaxed);
-    }
-
-    // CacheKey-based methods
-    pub async fn insert_with_key<K>(&self, cache_key: &K, metadata: Arc<K::ValueType>)
-    where
-        K: CacheKey,
-        K::ValueType: DeepSizeOf + Send + Sync + 'static,
-    {
-        self.insert(&cache_key.key(), metadata).boxed().await
-    }
-
-    pub async fn get_with_key<K>(&self, cache_key: &K) -> Option<Arc<K::ValueType>>
-    where
-        K: CacheKey,
-        K::ValueType: DeepSizeOf + Send + Sync + 'static,
-    {
-        self.get::<K::ValueType>(&cache_key.key()).boxed().await
-    }
-
-    pub async fn get_or_insert_with_key<K, F, Fut>(
-        &self,
-        cache_key: K,
-        loader: F,
-    ) -> Result<Arc<K::ValueType>>
-    where
-        K: CacheKey,
-        K::ValueType: DeepSizeOf + Send + Sync + 'static,
-        F: FnOnce() -> Fut,
-        Fut: Future<Output = Result<K::ValueType>> + Send,
-    {
-        let key_str = cache_key.key().into_owned();
-        Box::pin(self.get_or_insert(key_str, |_| loader())).await
-    }
-
-    pub async fn insert_unsized_with_key<K>(&self, cache_key: &K, metadata: Arc<K::ValueType>)
-    where
-        K: UnsizedCacheKey,
-        K::ValueType: DeepSizeOf + Send + Sync + 'static,
-    {
-        self.insert_unsized(&cache_key.key(), metadata)
-            .boxed()
-            .await
-    }
-
-    pub async fn get_unsized_with_key<K>(&self, cache_key: &K) -> Option<Arc<K::ValueType>>
-    where
-        K: UnsizedCacheKey,
-        K::ValueType: DeepSizeOf + Send + Sync + 'static,
-    {
-        self.get_unsized::<K::ValueType>(&cache_key.key())
-            .boxed()
-            .await
-    }
-}
-
-/// A weak reference to a LanceCache, used by indices to avoid circular references.
-/// When the original cache is dropped, operations on this will gracefully no-op.
-#[derive(Clone, Debug)]
-pub struct WeakLanceCache {
-    inner: std::sync::Weak<Cache<(String, TypeId), SizedRecord>>,
-    prefix: String,
-    hits: Arc<AtomicU64>,
-    misses: Arc<AtomicU64>,
-}
-
-impl WeakLanceCache {
-    /// Create a weak reference from a strong LanceCache
-    pub fn from(cache: &LanceCache) -> Self {
-        Self {
-            inner: Arc::downgrade(&cache.cache),
-            prefix: cache.prefix.clone(),
-            hits: cache.hits.clone(),
-            misses: cache.misses.clone(),
-        }
-    }
-
-    /// Appends a prefix to the cache key
-    pub fn with_key_prefix(&self, prefix: &str) -> Self {
-        Self {
-            inner: self.inner.clone(),
-            prefix: format!("{}{}/", self.prefix, prefix),
-            hits: self.hits.clone(),
-            misses: self.misses.clone(),
-        }
-    }
-
-    fn get_key(&self, key: &str) -> String {
-        if self.prefix.is_empty() {
-            key.to_string()
-        } else {
-            format!("{}/{}", self.prefix, key)
-        }
-    }
-
-    /// Get an item from cache if the cache is still alive
-    pub async fn get<T: DeepSizeOf + Send + Sync + 'static>(&self, key: &str) -> Option<Arc<T>> {
-        let cache = self.inner.upgrade()?;
-        let key = self.get_key(key);
-        if let Some(metadata) = cache.get(&(key, TypeId::of::<T>())).await {
-            self.hits.fetch_add(1, Ordering::Relaxed);
-            Some(metadata.record.clone().downcast::<T>().unwrap())
-        } else {
-            self.misses.fetch_add(1, Ordering::Relaxed);
-            None
-        }
-    }
-
-    /// Insert an item if the cache is still alive
-    /// Returns true if the item was inserted, false if the cache is no longer available
-    pub async fn insert<T: DeepSizeOf + Send + Sync + 'static>(
-        &self,
-        key: &str,
-        value: Arc<T>,
-    ) -> bool {
-        if let Some(cache) = self.inner.upgrade() {
-            let key = self.get_key(key);
-            let record = SizedRecord::new(value);
-            cache.insert((key, TypeId::of::<T>()), record).await;
-            true
-        } else {
-            log::warn!("WeakLanceCache: cache no longer available, unable to insert item");
-            false
-        }
-    }
-
-    /// Get or insert an item, computing it if necessary
-    pub async fn get_or_insert<T, F, Fut>(&self, key: &str, f: F) -> Result<Arc<T>>
-    where
-        T: DeepSizeOf + Send + Sync + 'static,
-        F: FnOnce() -> Fut,
-        Fut: Future<Output = Result<T>> + Send,
-    {
-        if let Some(cache) = self.inner.upgrade() {
-            let full_key = self.get_key(key);
-            let cache_key = (full_key.clone(), TypeId::of::<T>());
-
-            // Use optionally_get_with to handle concurrent requests properly
-            let hits = self.hits.clone();
-            let misses = self.misses.clone();
-
-            // Track whether init was run (for metrics)
-            let (init_run_tx, mut init_run_rx) = tokio::sync::oneshot::channel();
-            let (error_tx, error_rx) = tokio::sync::oneshot::channel();
-
-            let init = Box::pin(async move {
-                let _ = init_run_tx.send(());
-                misses.fetch_add(1, Ordering::Relaxed);
-                match f().await {
-                    Ok(value) => Some(SizedRecord::new(Arc::new(value))),
-                    Err(e) => {
-                        let _ = error_tx.send(e);
-                        None
-                    }
-                }
-            });
-
-            match cache.optionally_get_with(cache_key, init).await {
-                Some(record) => {
-                    // Check if init was run or if this was a cache hit
-                    match init_run_rx.try_recv() {
-                        Ok(()) => {
-                            // Init was run, miss was already recorded
-                        }
-                        Err(_) => {
-                            // Init was not run, this was a cache hit
-                            hits.fetch_add(1, Ordering::Relaxed);
-                        }
-                    }
-                    Ok(record.record.clone().downcast::<T>().unwrap())
-                }
-                None => {
-                    // Init returned None, which means there was an error
-                    match error_rx.await {
-                        Ok(e) => Err(e),
-                        Err(_) => Err(crate::Error::internal(
-                            "Failed to receive error from cache init function".to_string(),
-                        )),
-                    }
-                }
-            }
-        } else {
-            log::warn!("WeakLanceCache: cache no longer available, computing without caching");
-            f().await.map(Arc::new)
-        }
-    }
-
-    /// Get or insert an item with a cache key type
-    pub async fn get_or_insert_with_key<K, F, Fut>(
-        &self,
-        cache_key: K,
-        loader: F,
-    ) -> Result<Arc<K::ValueType>>
-    where
-        K: CacheKey,
-        K::ValueType: DeepSizeOf + Send + Sync + 'static,
-        F: FnOnce() -> Fut,
-        Fut: Future<Output = Result<K::ValueType>> + Send,
-    {
-        let key_str = cache_key.key().into_owned();
-        self.get_or_insert(&key_str, loader).await
-    }
-
-    /// Insert with a cache key type
-    /// Returns true if the item was inserted, false if the cache is no longer available
-    pub async fn insert_with_key<K>(&self, cache_key: &K, value: Arc<K::ValueType>) -> bool
-    where
-        K: CacheKey,
-        K::ValueType: DeepSizeOf + Send + Sync + 'static,
-    {
-        let key_str = cache_key.key().into_owned();
-        self.insert(&key_str, value).await
-    }
-
-    /// Get with a cache key type
-    pub async fn get_with_key<K>(&self, cache_key: &K) -> Option<Arc<K::ValueType>>
-    where
-        K: CacheKey,
-        K::ValueType: DeepSizeOf + Send + Sync + 'static,
-    {
-        let key_str = cache_key.key().into_owned();
-        self.get(&key_str).await
-    }
-
-    /// Get unsized item from cache
-    pub async fn get_unsized<T: DeepSizeOf + Send + Sync + 'static + ?Sized>(
-        &self,
-        key: &str,
-    ) -> Option<Arc<T>> {
-        // For unsized types, we store Arc<T> directly
-        let cache = self.inner.upgrade()?;
-        let key = self.get_key(key);
-        if let Some(metadata) = cache.get(&(key, TypeId::of::<Arc<T>>())).await {
-            metadata
-                .record
-                .clone()
-                .downcast::<Arc<T>>()
-                .ok()
-                .map(|arc| arc.as_ref().clone())
-        } else {
-            None
-        }
-    }
-
-    /// Insert unsized item into cache
-    pub async fn insert_unsized<T: DeepSizeOf + Send + Sync + 'static + ?Sized>(
-        &self,
-        key: &str,
-        value: Arc<T>,
-    ) {
-        if let Some(cache) = self.inner.upgrade() {
-            let key = self.get_key(key);
-            let record = SizedRecord::new(Arc::new(value));
-            cache.insert((key, TypeId::of::<Arc<T>>()), record).await;
-        } else {
-            log::warn!("WeakLanceCache: cache no longer available, unable to insert unsized item");
-        }
-    }
-
-    /// Get unsized with a cache key type
-    pub async fn get_unsized_with_key<K>(&self, cache_key: &K) -> Option<Arc<K::ValueType>>
-    where
-        K: UnsizedCacheKey,
-        K::ValueType: DeepSizeOf + Send + Sync + 'static,
-    {
-        let key_str = cache_key.key();
-        self.get_unsized(&key_str).await
-    }
-
-    /// Insert unsized with a cache key type
-    pub async fn insert_unsized_with_key<K>(&self, cache_key: &K, value: Arc<K::ValueType>)
-    where
-        K: UnsizedCacheKey,
-        K::ValueType: DeepSizeOf + Send + Sync + 'static,
-    {
-        let key_str = cache_key.key();
-        self.insert_unsized(&key_str, value).await
-    }
-}
-
-pub trait CacheKey {
-    type ValueType;
-
-    fn key(&self) -> Cow<'_, str>;
-}
-
-pub trait UnsizedCacheKey {
-    type ValueType: ?Sized;
-
-    fn key(&self) -> Cow<'_, str>;
-}
-
-#[derive(Debug, Clone)]
-pub struct CacheStats {
-    /// Number of times `get`, `get_unsized`, or `get_or_insert` found an item in the cache.
-    pub hits: u64,
-    /// Number of times `get`, `get_unsized`, or `get_or_insert` did not find an item in the cache.
-    pub misses: u64,
-    /// Number of entries currently in the cache.
-    pub num_entries: usize,
-    /// Total size in bytes of all entries in the cache.
-    pub size_bytes: usize,
-}
-
-impl CacheStats {
-    pub fn hit_ratio(&self) -> f32 {
-        if self.hits + self.misses == 0 {
-            0.0
-        } else {
-            self.hits as f32 / (self.hits + self.misses) as f32
-        }
-    }
-
-    pub fn miss_ratio(&self) -> f32 {
-        if self.hits + self.misses == 0 {
-            0.0
-        } else {
-            self.misses as f32 / (self.hits + self.misses) as f32
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[tokio::test]
-    async fn test_cache_bytes() {
-        let item = Arc::new(vec![1, 2, 3]);
-        let item_size = item.deep_size_of(); // Size of Arc<Vec<i32>>
-        let capacity = 10 * item_size;
-
-        let cache = LanceCache::with_capacity(capacity);
-        assert_eq!(cache.size_bytes().await, 0);
-        assert_eq!(cache.approx_size_bytes(), 0);
-
-        let item = Arc::new(vec![1, 2, 3]);
-        cache.insert("key", item.clone()).await;
-        assert_eq!(cache.size().await, 1);
-        assert_eq!(cache.size_bytes().await, item_size);
-        assert_eq!(cache.approx_size_bytes(), item_size);
-
-        let retrieved = cache.get::<Vec<i32>>("key").await.unwrap();
-        assert_eq!(*retrieved, *item);
-
-        // Test eviction based on size
-        for i in 0..20 {
-            cache
-                .insert(&format!("key_{}", i), Arc::new(vec![i, i, i]))
-                .await;
-        }
-        assert_eq!(cache.size_bytes().await, capacity);
-        assert_eq!(cache.size().await, 10);
-    }
-
-    #[tokio::test]
-    async fn test_cache_trait_objects() {
-        #[derive(Debug, DeepSizeOf)]
-        struct MyType(i32);
-
-        trait MyTrait: DeepSizeOf + Send + Sync + Any {
-            fn as_any(&self) -> &dyn Any;
-        }
-
-        impl MyTrait for MyType {
-            fn as_any(&self) -> &dyn Any {
-                self
-            }
-        }
-
-        let item = Arc::new(MyType(42));
-        let item_dyn: Arc<dyn MyTrait> = item;
-
-        let cache = LanceCache::with_capacity(1000);
-        cache.insert_unsized("test", item_dyn).await;
-
-        let retrieved = cache.get_unsized::<dyn MyTrait>("test").await.unwrap();
-        let retrieved = retrieved.as_any().downcast_ref::<MyType>().unwrap();
-        assert_eq!(retrieved.0, 42);
-    }
-
-    #[tokio::test]
-    async fn test_cache_stats_basic() {
-        let cache = LanceCache::with_capacity(1000);
-
-        // Initially no hits or misses
-        let stats = cache.stats().await;
-        assert_eq!(stats.hits, 0);
-        assert_eq!(stats.misses, 0);
-
-        // Miss on first get
-        let result = cache.get::<Vec<i32>>("nonexistent");
-        assert!(result.await.is_none());
-        let stats = cache.stats().await;
-        assert_eq!(stats.hits, 0);
-        assert_eq!(stats.misses, 1);
-
-        // Insert and then hit
-        cache.insert("key1", Arc::new(vec![1, 2, 3])).await;
-        let result = cache.get::<Vec<i32>>("key1");
-        assert!(result.await.is_some());
-        let stats = cache.stats().await;
-        assert_eq!(stats.hits, 1);
-        assert_eq!(stats.misses, 1);
-
-        // Another hit
-        let result = cache.get::<Vec<i32>>("key1");
-        assert!(result.await.is_some());
-        let stats = cache.stats().await;
-        assert_eq!(stats.hits, 2);
-        assert_eq!(stats.misses, 1);
-
-        // Another miss
-        let result = cache.get::<Vec<i32>>("nonexistent2");
-        assert!(result.await.is_none());
-        let stats = cache.stats().await;
-        assert_eq!(stats.hits, 2);
-        assert_eq!(stats.misses, 2);
-    }
-
-    #[tokio::test]
-    async fn test_cache_stats_with_prefixes() {
-        let base_cache = LanceCache::with_capacity(1000);
-        let prefixed_cache = base_cache.with_key_prefix("test");
-
-        // Stats should be shared between base and prefixed cache
-        let stats = base_cache.stats().await;
-        assert_eq!(stats.hits, 0);
-        assert_eq!(stats.misses, 0);
-
-        let stats = prefixed_cache.stats().await;
-        assert_eq!(stats.hits, 0);
-        assert_eq!(stats.misses, 0);
-
-        // Miss on prefixed cache
-        let result = prefixed_cache.get::<Vec<i32>>("key1");
-        assert!(result.await.is_none());
-
-        // Both should show the miss
-        let stats = base_cache.stats().await;
-        assert_eq!(stats.hits, 0);
-        assert_eq!(stats.misses, 1);
-
-        let stats = prefixed_cache.stats().await;
-        assert_eq!(stats.hits, 0);
-        assert_eq!(stats.misses, 1);
-
-        // Insert through prefixed cache and hit
-        prefixed_cache.insert("key1", Arc::new(vec![1, 2, 3])).await;
-        let result = prefixed_cache.get::<Vec<i32>>("key1");
-        assert!(result.await.is_some());
-
-        // Both should show the hit
-        let stats = base_cache.stats().await;
-        assert_eq!(stats.hits, 1);
-        assert_eq!(stats.misses, 1);
-
-        let stats = prefixed_cache.stats().await;
-        assert_eq!(stats.hits, 1);
-        assert_eq!(stats.misses, 1);
-    }
-
-    #[tokio::test]
-    async fn test_cache_stats_unsized() {
-        #[derive(Debug, DeepSizeOf)]
-        struct MyType(i32);
-
-        trait MyTrait: DeepSizeOf + Send + Sync + Any {}
-
-        impl MyTrait for MyType {}
-
-        let cache = LanceCache::with_capacity(1000);
-
-        // Miss on unsized get
-        let result = cache.get_unsized::<dyn MyTrait>("test");
-        assert!(result.await.is_none());
-        let stats = cache.stats().await;
-        assert_eq!(stats.hits, 0);
-        assert_eq!(stats.misses, 1);
-
-        // Insert and hit on unsized
-        let item = Arc::new(MyType(42));
-        let item_dyn: Arc<dyn MyTrait> = item;
-        cache.insert_unsized("test", item_dyn).await;
-
-        let result = cache.get_unsized::<dyn MyTrait>("test");
-        assert!(result.await.is_some());
-        let stats = cache.stats().await;
-        assert_eq!(stats.hits, 1);
-        assert_eq!(stats.misses, 1);
-    }
-
-    #[tokio::test]
-    async fn test_cache_stats_get_or_insert() {
-        let cache = LanceCache::with_capacity(1000);
-
-        // First call should be a miss and load the value
-        let result: Arc<Vec<i32>> = cache
-            .get_or_insert("key1".to_string(), |_key| async { Ok(vec![1, 2, 3]) })
-            .await
-            .unwrap();
-        assert_eq!(*result, vec![1, 2, 3]);
-
-        let stats = cache.stats().await;
-        assert_eq!(stats.hits, 0);
-        assert_eq!(stats.misses, 1);
-
-        // Second call should be a hit
-        let result: Arc<Vec<i32>> = cache
-            .get_or_insert("key1".to_string(), |_key| async {
-                panic!("Should not be called")
-            })
-            .await
-            .unwrap();
-        assert_eq!(*result, vec![1, 2, 3]);
-
-        let stats = cache.stats().await;
-        assert_eq!(stats.hits, 1);
-        assert_eq!(stats.misses, 1);
-
-        // Different key should be another miss
-        let result: Arc<Vec<i32>> = cache
-            .get_or_insert("key2".to_string(), |_key| async { Ok(vec![4, 5, 6]) })
-            .await
-            .unwrap();
-        assert_eq!(*result, vec![4, 5, 6]);
-
-        let stats = cache.stats().await;
-        assert_eq!(stats.hits, 1);
-        assert_eq!(stats.misses, 2);
-    }
-}
diff --git a/rust/lance-core/src/cache/backend.rs b/rust/lance-core/src/cache/backend.rs
new file mode 100644
index 00000000000..237254c464f
--- /dev/null
+++ b/rust/lance-core/src/cache/backend.rs
@@ -0,0 +1,140 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Backend interface for cache implementors.
+//!
+//! This module defines the trait that custom cache backends must implement,
+//! along with the key and entry types they operate on. Most callers should
+//! use [`LanceCache`](super::LanceCache) instead of interacting with
+//! backends directly.
+
+use std::any::Any;
+use std::pin::Pin;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use futures::Future;
+
+use crate::Result;
+
+use super::CacheCodec;
+
+/// A type-erased cache entry.
+pub type CacheEntry = Arc<dyn Any + Send + Sync>;
+
+/// Structured cache key passed to [`CacheBackend`] methods.
+///
+/// CacheBackend impls receive these ready-made from [`LanceCache`](super::LanceCache)
+/// — you do not construct them yourself. Composed of three parts:
+/// - **prefix**: scopes the key to a dataset or index (e.g. `"s3://bucket/dataset/"`)
+/// - **key**: identifies the specific entry (e.g. `"42"` for a version number)
+/// - **type_name**: distinguishes different value types stored under the same
+///   user key (e.g. `"Vec<IndexMetadata>"`)
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct InternalCacheKey {
+    prefix: Arc<str>,
+    key: Arc<str>,
+    type_name: &'static str,
+}
+
+impl InternalCacheKey {
+    pub fn new(prefix: Arc<str>, key: Arc<str>, type_name: &'static str) -> Self {
+        Self {
+            prefix,
+            key,
+            type_name,
+        }
+    }
+
+    pub fn prefix(&self) -> &str {
+        &self.prefix
+    }
+
+    pub fn key(&self) -> &str {
+        &self.key
+    }
+
+    pub fn type_name(&self) -> &'static str {
+        self.type_name
+    }
+
+    /// Returns true if this key's prefix starts with the given string.
+    pub fn starts_with(&self, prefix: &str) -> bool {
+        self.prefix.starts_with(prefix)
+    }
+}
+
+/// Low-level pluggable cache backend.
+///
+/// Implementations store entries keyed by [`InternalCacheKey`] and return
+/// type-erased [`CacheEntry`] values.
+/// [`LanceCache`](super::LanceCache) handles key construction and type safety;
+/// backend authors only need to implement storage and eviction.
+#[async_trait]
+pub trait CacheBackend: Send + Sync + std::fmt::Debug {
+    /// Look up an entry by its key.
+    ///
+    /// `codec` is provided so that persistent backends can deserialize the
+    /// entry from storage. In-memory backends can ignore it. When `codec`
+    /// is `None`, the entry type does not support serialization yet and
+    /// must be stored in-memory.
+    ///
+    /// The goal is for all cache entry types to eventually have codecs,
+    /// at which point the `Option` will be removed.
+    async fn get(&self, key: &InternalCacheKey, codec: Option<CacheCodec>) -> Option<CacheEntry>;
+
+    /// Store an entry. `size_bytes` is used for eviction accounting.
+    ///
+    /// See [`get`](Self::get) for codec semantics.
+    async fn insert(
+        &self,
+        key: &InternalCacheKey,
+        entry: CacheEntry,
+        size_bytes: usize,
+        codec: Option<CacheCodec>,
+    );
+
+    /// Get an existing entry or compute it from `loader`.
+    ///
+    /// Implementations should deduplicate concurrent loads for the same key
+    /// so the loader runs at most once.
+    ///
+    /// Returns `(entry, was_cached)` where `was_cached` is `true` if the entry
+    /// was already present in the cache (the loader was not invoked).
+    ///
+    /// See [`get`](Self::get) for codec semantics.
+    async fn get_or_insert<'a>(
+        &self,
+        key: &InternalCacheKey,
+        loader: Pin<Box<dyn Future<Output = Result<(CacheEntry, usize)>> + Send + 'a>>,
+        codec: Option<CacheCodec>,
+    ) -> Result<(CacheEntry, bool)>;
+
+    /// Remove all entries whose prefix starts with the given string.
+    async fn invalidate_prefix(&self, prefix: &str);
+
+    /// Remove all entries.
+    async fn clear(&self);
+
+    /// Number of entries currently stored (may flush pending operations).
+    async fn num_entries(&self) -> usize;
+
+    /// Total weighted size in bytes of all stored entries (may flush pending operations).
+    async fn size_bytes(&self) -> usize;
+
+    /// Approximate number of entries, callable from synchronous contexts.
+    /// Backends that cannot provide this cheaply should return 0.
+    fn approx_num_entries(&self) -> usize {
+        0
+    }
+
+    /// Approximate weighted size in bytes, callable from synchronous contexts.
+    /// Used by `DeepSizeOf` to report cache memory usage.
+    /// Backends that cannot provide this cheaply should return 0.
+    ///
+    /// Assumes entries do not share underlying buffers; if they do, the
+    /// returned total may overcount.
+    fn approx_size_bytes(&self) -> usize {
+        0
+    }
+}
diff --git a/rust/lance-core/src/cache/codec.rs b/rust/lance-core/src/cache/codec.rs
new file mode 100644
index 00000000000..34e5264bb28
--- /dev/null
+++ b/rust/lance-core/src/cache/codec.rs
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Serialization codecs for cache entries.
+//!
+//! Implement [`CacheCodecImpl`] on concrete types, then use
+//! [`CacheCodec::from_impl`] to produce a type-erased codec for the cache.
+
+use std::sync::Arc;
+
+use bytes::Bytes;
+
+use crate::Result;
+
+// ---------------------------------------------------------------------------
+// CacheCodecImpl — trait for serializable cache entry types
+// ---------------------------------------------------------------------------
+
+/// Serialization trait for cache entries.
+///
+/// **Experimental**: the serialized format is not stable and may change
+/// between releases without notice.
+///
+/// Implement this on concrete types that need to survive serialization
+/// through a persistent cache backend. Then wire it into a [`CacheKey`](super::CacheKey)
+/// via [`CacheCodec::from_impl`]:
+///
+/// ```ignore
+/// impl CacheCodecImpl for MyData {
+///     fn serialize(&self, w: &mut dyn Write) -> Result<()> { /* ... */ }
+///     fn deserialize(data: &Bytes) -> Result<Self> { /* ... */ }
+/// }
+///
+/// impl CacheKey for MyDataKey {
+///     type ValueType = MyData;
+///     fn codec() -> Option<CacheCodec> {
+///         Some(CacheCodec::from_impl::<MyData>())
+///     }
+///     // ...
+/// }
+/// ```
+pub trait CacheCodecImpl: Send + Sync {
+    fn serialize(&self, writer: &mut dyn std::io::Write) -> Result<()>;
+
+    fn deserialize(data: &Bytes) -> Result<Self>
+    where
+        Self: Sized;
+}
+
+// ---------------------------------------------------------------------------
+// CacheCodec — type-erased codec passed to backends
+// ---------------------------------------------------------------------------
+
+pub(crate) type ArcAny = Arc<dyn std::any::Any + Send + Sync>;
+
+/// Type-erased codec for serializing and deserializing cache entries.
+///
+/// `CacheCodec` is two plain function pointers — it is `Copy` and has no
+/// heap allocation. Construct one via [`CacheCodec::from_impl`] for types
+/// that implement [`CacheCodecImpl`], or [`CacheCodec::new`] for custom
+/// cases (e.g. when the orphan rule prevents a direct impl).
+#[derive(Copy, Clone)]
+pub struct CacheCodec {
+    pub(crate) serialize: fn(&ArcAny, &mut dyn std::io::Write) -> Result<()>,
+    pub(crate) deserialize: fn(&Bytes) -> Result<ArcAny>,
+}
+
+impl std::fmt::Debug for CacheCodec {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("CacheCodec").finish_non_exhaustive()
+    }
+}
+
+fn serialize_via_impl<T: CacheCodecImpl + 'static>(
+    any: &ArcAny,
+    writer: &mut dyn std::io::Write,
+) -> Result<()> {
+    let val = any
+        .downcast_ref::<T>()
+        .expect("CacheCodec::serialize called with wrong type (this is a bug in the cache layer)");
+    val.serialize(writer)
+}
+
+fn deserialize_via_impl<T: CacheCodecImpl + 'static>(data: &Bytes) -> Result<ArcAny> {
+    let val = T::deserialize(data)?;
+    Ok(Arc::new(val) as ArcAny)
+}
+
+impl CacheCodec {
+    /// Create a `CacheCodec` from plain function pointers.
+    ///
+    /// Prefer [`from_impl`](Self::from_impl) when the value type implements
+    /// [`CacheCodecImpl`]. Use this for types where a direct impl isn't
+    /// possible (e.g. orphan rule prevents it).
+    pub fn new(
+        serialize: fn(&ArcAny, &mut dyn std::io::Write) -> Result<()>,
+        deserialize: fn(&Bytes) -> Result<ArcAny>,
+    ) -> Self {
+        Self {
+            serialize,
+            deserialize,
+        }
+    }
+
+    /// Create a `CacheCodec` from a [`CacheCodecImpl`] implementation.
+    ///
+    /// For **sized** types stored directly in the cache. The codec
+    /// downcasts `&dyn Any` to `&T` for serialization and returns `Arc<T>`
+    /// from deserialization.
+    pub fn from_impl<T: CacheCodecImpl + 'static>() -> Self {
+        Self {
+            serialize: serialize_via_impl::<T>,
+            deserialize: deserialize_via_impl::<T>,
+        }
+    }
+
+    pub fn serialize(&self, value: &ArcAny, writer: &mut dyn std::io::Write) -> Result<()> {
+        (self.serialize)(value, writer)
+    }
+
+    pub fn deserialize(&self, data: &Bytes) -> Result<ArcAny> {
+        (self.deserialize)(data)
+    }
+}
diff --git a/rust/lance-core/src/cache/mod.rs b/rust/lance-core/src/cache/mod.rs
new file mode 100644
index 00000000000..ee6a728ef73
--- /dev/null
+++ b/rust/lance-core/src/cache/mod.rs
@@ -0,0 +1,873 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Lance cache system.
+//!
+//! ## For cache users
+//!
+//! Use [`LanceCache`] (or [`WeakLanceCache`]) to store and retrieve typed
+//! values. Define a [`CacheKey`] (or [`UnsizedCacheKey`] for trait objects) to
+//! describe what you're caching and its type.
+//!
+//! To make a value type serializable (so persistent backends can store it),
+//! implement [`CacheCodecImpl`] on the type, then override [`CacheKey::codec`]:
+//!
+//! ```ignore
+//! impl CacheCodecImpl for MyData {
+//!     fn serialize(&self, w: &mut dyn Write) -> Result<()> { /* ... */ }
+//!     fn deserialize(data: &Bytes) -> Result<Self> { /* ... */ }
+//! }
+//!
+//! impl CacheKey for MyDataKey {
+//!     type ValueType = MyData;
+//!     fn key(&self) -> Cow<'_, str> { /* ... */ }
+//!     fn type_name() -> &'static str { "MyData" }
+//!     fn codec() -> Option<CacheCodec> {
+//!         Some(CacheCodec::from_impl::<MyData>())
+//!     }
+//! }
+//! ```
+//!
+//! ## For backend implementors
+//!
+//! Implement [`CacheBackend`] to provide a custom storage layer (disk, Redis,
+//! etc.). Backends receive [`InternalCacheKey`] keys and type-erased
+//! [`CacheEntry`] values — the typed wrapping is handled by [`LanceCache`].
+//! See the [`backend`] module for details.
+//!
+//! ## Serialization flow
+//!
+//! When a [`CacheKey`] provides a codec via [`CacheKey::codec`]:
+//!
+//! 1. [`LanceCache`] wraps the [`CacheCodec`] and passes it to the backend
+//!    alongside the entry on `insert` and `get` calls.
+//! 2. In-memory backends (like [`MokaCacheBackend`]) ignore the codec.
+//! 3. Persistent backends use `codec.serialize(entry, writer)` on insert and
+//!    `codec.deserialize(reader)` on get to persist entries across restarts.
+
+pub mod backend;
+pub mod codec;
+mod moka;
+
+pub use backend::{CacheBackend, CacheEntry, InternalCacheKey};
+pub use codec::{CacheCodec, CacheCodecImpl};
+pub use moka::MokaCacheBackend;
+
+use std::borrow::Cow;
+use std::sync::{
+    Arc,
+    atomic::{AtomicU64, Ordering},
+};
+
+use futures::{Future, FutureExt};
+
+use crate::Result;
+
+pub use deepsize::{Context, DeepSizeOf};
+
+// ---------------------------------------------------------------------------
+// CacheKey / UnsizedCacheKey — typed key traits for cache users
+// ---------------------------------------------------------------------------
+
+/// Typed cache key for sized value types.
+///
+/// Implement this trait to define a new type of cached entry. [`LanceCache`]
+/// uses the key string and type name to construct an [`InternalCacheKey`]
+/// for the backend.
+///
+/// # Example
+///
+/// ```ignore
+/// struct MyKey { id: u64 }
+///
+/// impl CacheKey for MyKey {
+///     type ValueType = MyData;
+///     fn key(&self) -> Cow<'_, str> { self.id.to_string().into() }
+///     fn type_name() -> &'static str { "MyData" }
+/// }
+/// ```
+pub trait CacheKey {
+    type ValueType: 'static;
+
+    fn key(&self) -> Cow<'_, str>;
+
+    /// Short, stable string identifying this value type.
+    ///
+    /// Two `CacheKey` impls that store different `ValueType`s **must** return
+    /// different type names; if they collide, gets will silently return `None`
+    /// due to failed downcasts.
+    ///
+    /// Use a short literal (e.g. `"Vec<IndexMetadata>"`), not
+    /// `std::any::type_name` — the latter is not guaranteed stable across
+    /// compiler versions or build configurations.
+    fn type_name() -> &'static str;
+
+    /// Optional codec for serializing/deserializing this key's value type.
+    ///
+    /// Returns `None` by default. Cache backends that support persistence
+    /// (e.g. disk-backed caches) use this to serialize entries on insert and
+    /// deserialize on get. Types without a codec will only be stored in-memory.
+    ///
+    /// [`CacheCodec`] is `Copy` (two plain function pointers), so returning it
+    /// by value is cheap — no allocation needed.
+    fn codec() -> Option<CacheCodec> {
+        None
+    }
+}
+
+/// Like [`CacheKey`] but for unsized value types (e.g. `dyn Trait`).
+///
+/// The cache wraps values in an extra `Arc` layer internally; callers pass
+/// and receive `Arc<T>` where `T: ?Sized`.
+///
+/// Unsized cache entries are always in-memory only (no serialization codec).
+/// For serializable entries, use a sized [`CacheKey`] instead.
+pub trait UnsizedCacheKey {
+    type ValueType: 'static + ?Sized;
+
+    fn key(&self) -> Cow<'_, str>;
+
+    /// Short, stable string identifying this value type.
+    /// See [`CacheKey::type_name`] for requirements.
+    fn type_name() -> &'static str;
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+/// Size of a cached `Arc<T>`, accounting for the Arc overhead (two atomic counters).
+fn cache_entry_size<T: DeepSizeOf + ?Sized>(value: &T) -> usize {
+    value.deep_size_of() + std::mem::size_of::<std::sync::atomic::AtomicUsize>() * 2
+}
+
+/// Build an [`InternalCacheKey`] from a cache's prefix, a user key string,
+/// and a type name.
+fn build_key(prefix: &Arc<str>, key: &str, type_name: &'static str) -> InternalCacheKey {
+    InternalCacheKey::new(prefix.clone(), Arc::from(key), type_name)
+}
+
+// ---------------------------------------------------------------------------
+// LanceCache — typed wrapper around dyn CacheBackend
+// ---------------------------------------------------------------------------
+
+/// Typed cache wrapper that handles key construction and type safety.
+///
+/// Internally delegates to a [`CacheBackend`]. The default backend is
+/// [`MokaCacheBackend`]; pass a custom backend via [`LanceCache::with_backend`].
+#[derive(Clone)]
+pub struct LanceCache {
+    cache: Arc<dyn CacheBackend>,
+    prefix: Arc<str>,
+    hits: Arc<AtomicU64>,
+    misses: Arc<AtomicU64>,
+}
+
+impl std::fmt::Debug for LanceCache {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("LanceCache")
+            .field("cache", &self.cache)
+            .finish()
+    }
+}
+
+impl DeepSizeOf for LanceCache {
+    fn deep_size_of_children(&self, _: &mut Context) -> usize {
+        self.cache.approx_size_bytes()
+    }
+}
+
+impl LanceCache {
+    pub fn with_capacity(capacity: usize) -> Self {
+        Self {
+            cache: Arc::new(MokaCacheBackend::with_capacity(capacity)),
+            prefix: Arc::from(""),
+            hits: Arc::new(AtomicU64::new(0)),
+            misses: Arc::new(AtomicU64::new(0)),
+        }
+    }
+
+    /// Create a cache backed by a custom [`CacheBackend`].
+    pub fn with_backend(backend: Arc<dyn CacheBackend>) -> Self {
+        Self {
+            cache: backend,
+            prefix: Arc::from(""),
+            hits: Arc::new(AtomicU64::new(0)),
+            misses: Arc::new(AtomicU64::new(0)),
+        }
+    }
+
+    pub fn no_cache() -> Self {
+        Self {
+            cache: Arc::new(MokaCacheBackend::no_cache()),
+            prefix: Arc::from(""),
+            hits: Arc::new(AtomicU64::new(0)),
+            misses: Arc::new(AtomicU64::new(0)),
+        }
+    }
+
+    /// Create a cache with the given backend and an exact prefix string.
+    /// Unlike `with_key_prefix`, this sets the prefix verbatim (no trailing slash added).
+    pub fn with_backend_and_prefix(backend: Arc<dyn CacheBackend>, prefix: String) -> Self {
+        Self {
+            cache: backend,
+            prefix: Arc::from(prefix),
+            hits: Arc::new(AtomicU64::new(0)),
+            misses: Arc::new(AtomicU64::new(0)),
+        }
+    }
+
+    /// Appends a prefix to the cache key.
+    pub fn with_key_prefix(&self, prefix: &str) -> Self {
+        Self {
+            cache: self.cache.clone(),
+            prefix: Arc::from(format!("{}{}/", self.prefix, prefix)),
+            hits: self.hits.clone(),
+            misses: self.misses.clone(),
+        }
+    }
+
+    /// Invalidate all entries whose prefix starts with the given string.
+    pub async fn invalidate_prefix(&self, prefix: &str) {
+        let full_prefix = format!("{}{}", self.prefix, prefix);
+        self.cache.invalidate_prefix(&full_prefix).await;
+    }
+
+    pub async fn size(&self) -> usize {
+        self.cache.num_entries().await
+    }
+
+    pub fn approx_size(&self) -> usize {
+        self.cache.approx_num_entries()
+    }
+
+    pub async fn size_bytes(&self) -> usize {
+        self.cache.size_bytes().await
+    }
+
+    // -- Sized insert/get (internal, shared by sized and unsized paths) --------
+
+    async fn insert_with_id<T: DeepSizeOf + Send + Sync + 'static>(
+        &self,
+        key: &str,
+        type_name: &'static str,
+        codec: Option<CacheCodec>,
+        metadata: Arc<T>,
+    ) {
+        let size = cache_entry_size(&*metadata);
+        let cache_key = build_key(&self.prefix, key, type_name);
+        self.cache.insert(&cache_key, metadata, size, codec).await;
+    }
+
+    async fn get_with_id<T: Send + Sync + 'static>(
+        &self,
+        key: &str,
+        type_name: &'static str,
+        codec: Option<CacheCodec>,
+    ) -> Option<Arc<T>> {
+        let cache_key = build_key(&self.prefix, key, type_name);
+        if let Some(entry) = self.cache.get(&cache_key, codec).await {
+            match entry.downcast::<T>() {
+                Ok(val) => {
+                    self.hits.fetch_add(1, Ordering::Relaxed);
+                    Some(val)
+                }
+                Err(_) => {
+                    // Type mismatch: the backend returned a different concrete
+                    // type than expected (e.g. a disk cache may store
+                    // intermediate state). Treat as a miss.
+                    self.misses.fetch_add(1, Ordering::Relaxed);
+                    None
+                }
+            }
+        } else {
+            self.misses.fetch_add(1, Ordering::Relaxed);
+            None
+        }
+    }
+
+    // -- Stats / clear --------------------------------------------------------
+
+    pub async fn stats(&self) -> CacheStats {
+        CacheStats {
+            hits: self.hits.load(Ordering::Relaxed),
+            misses: self.misses.load(Ordering::Relaxed),
+            num_entries: self.cache.num_entries().await,
+            size_bytes: self.cache.size_bytes().await,
+        }
+    }
+
+    pub async fn clear(&self) {
+        self.cache.clear().await;
+        self.hits.store(0, Ordering::Relaxed);
+        self.misses.store(0, Ordering::Relaxed);
+    }
+
+    // -- CacheKey-based methods -----------------------------------------------
+
+    pub async fn insert_with_key<K>(&self, cache_key: &K, metadata: Arc<K::ValueType>)
+    where
+        K: CacheKey,
+        K::ValueType: DeepSizeOf + Send + Sync + 'static,
+    {
+        self.insert_with_id(&cache_key.key(), K::type_name(), K::codec(), metadata)
+            .boxed()
+            .await
+    }
+
+    pub async fn get_with_key<K>(&self, cache_key: &K) -> Option<Arc<K::ValueType>>
+    where
+        K: CacheKey,
+        K::ValueType: DeepSizeOf + Send + Sync + 'static,
+    {
+        self.get_with_id::<K::ValueType>(&cache_key.key(), K::type_name(), K::codec())
+            .boxed()
+            .await
+    }
+
+    pub async fn get_or_insert_with_key<K, F, Fut>(
+        &self,
+        cache_key: K,
+        loader: F,
+    ) -> Result<Arc<K::ValueType>>
+    where
+        K: CacheKey,
+        K::ValueType: DeepSizeOf + Send + Sync + 'static,
+        F: FnOnce() -> Fut + Send,
+        Fut: Future<Output = Result<K::ValueType>> + Send,
+    {
+        let key = build_key(&self.prefix, &cache_key.key(), K::type_name());
+
+        let typed_loader = Box::pin(async move {
+            let value = loader().await?;
+            let arc = Arc::new(value);
+            let size = cache_entry_size(&*arc);
+            Ok((arc as CacheEntry, size))
+        });
+
+        let (entry, was_cached) = self
+            .cache
+            .get_or_insert(&key, typed_loader, K::codec())
+            .await?;
+
+        if was_cached {
+            self.hits.fetch_add(1, Ordering::Relaxed);
+        } else {
+            self.misses.fetch_add(1, Ordering::Relaxed);
+        }
+
+        Ok(entry.downcast::<K::ValueType>().unwrap())
+    }
+
+    pub async fn insert_unsized_with_key<K>(&self, cache_key: &K, metadata: Arc<K::ValueType>)
+    where
+        K: UnsizedCacheKey,
+        K::ValueType: DeepSizeOf + Send + Sync + 'static,
+    {
+        self.insert_with_id(&cache_key.key(), K::type_name(), None, Arc::new(metadata))
+            .boxed()
+            .await
+    }
+
+    pub async fn get_unsized_with_key<K>(&self, cache_key: &K) -> Option<Arc<K::ValueType>>
+    where
+        K: UnsizedCacheKey,
+        K::ValueType: DeepSizeOf + Send + Sync + 'static,
+    {
+        let outer = self
+            .get_with_id::<Arc<K::ValueType>>(&cache_key.key(), K::type_name(), None)
+            .boxed()
+            .await?;
+        Some(outer.as_ref().clone())
+    }
+}
+
+// ---------------------------------------------------------------------------
+// WeakLanceCache
+// ---------------------------------------------------------------------------
+
+/// A weak reference to a LanceCache, used by indices to avoid circular references.
+/// When the original cache is dropped, operations on this will gracefully no-op.
+#[derive(Clone, Debug)]
+pub struct WeakLanceCache {
+    inner: std::sync::Weak<dyn CacheBackend>,
+    prefix: Arc<str>,
+    hits: Arc<AtomicU64>,
+    misses: Arc<AtomicU64>,
+}
+
+impl WeakLanceCache {
+    pub fn from(cache: &LanceCache) -> Self {
+        Self {
+            inner: Arc::downgrade(&cache.cache),
+            prefix: cache.prefix.clone(),
+            hits: cache.hits.clone(),
+            misses: cache.misses.clone(),
+        }
+    }
+
+    pub fn with_key_prefix(&self, prefix: &str) -> Self {
+        Self {
+            inner: self.inner.clone(),
+            prefix: Arc::from(format!("{}{}/", self.prefix, prefix)),
+            hits: self.hits.clone(),
+            misses: self.misses.clone(),
+        }
+    }
+
+    /// The key prefix used for all entries in this cache.
+    pub fn prefix(&self) -> &str {
+        &self.prefix
+    }
+
+    pub async fn get_with_key<K>(&self, cache_key: &K) -> Option<Arc<K::ValueType>>
+    where
+        K: CacheKey,
+        K::ValueType: DeepSizeOf + Send + Sync + 'static,
+    {
+        let cache = self.inner.upgrade()?;
+        let key = build_key(&self.prefix, &cache_key.key(), K::type_name());
+        if let Some(entry) = cache.get(&key, K::codec()).await {
+            self.hits.fetch_add(1, Ordering::Relaxed);
+            Some(entry.downcast::<K::ValueType>().unwrap())
+        } else {
+            self.misses.fetch_add(1, Ordering::Relaxed);
+            None
+        }
+    }
+
+    pub async fn insert_with_key<K>(&self, cache_key: &K, value: Arc<K::ValueType>) -> bool
+    where
+        K: CacheKey,
+        K::ValueType: DeepSizeOf + Send + Sync + 'static,
+    {
+        if let Some(cache) = self.inner.upgrade() {
+            let size = cache_entry_size(&*value);
+            let key = build_key(&self.prefix, &cache_key.key(), K::type_name());
+            cache.insert(&key, value, size, K::codec()).await;
+            true
+        } else {
+            log::warn!("WeakLanceCache: cache no longer available, unable to insert item");
+            false
+        }
+    }
+
+    /// Get or insert an item, computing it if necessary.
+    ///
+    /// Deduplication of concurrent loads is handled by the backend.
+    pub async fn get_or_insert_with_key<K, F, Fut>(
+        &self,
+        cache_key: K,
+        loader: F,
+    ) -> Result<Arc<K::ValueType>>
+    where
+        K: CacheKey,
+        K::ValueType: DeepSizeOf + Send + Sync + 'static,
+        F: FnOnce() -> Fut + Send,
+        Fut: Future<Output = Result<K::ValueType>> + Send,
+    {
+        if let Some(cache) = self.inner.upgrade() {
+            let key = build_key(&self.prefix, &cache_key.key(), K::type_name());
+            let typed_loader = Box::pin(async move {
+                let value = loader().await?;
+                let arc = Arc::new(value);
+                let size = cache_entry_size(&*arc);
+                Ok((arc as CacheEntry, size))
+            });
+            let (entry, was_cached) = cache.get_or_insert(&key, typed_loader, K::codec()).await?;
+            if was_cached {
+                self.hits.fetch_add(1, Ordering::Relaxed);
+            } else {
+                self.misses.fetch_add(1, Ordering::Relaxed);
+            }
+            Ok(entry.downcast::<K::ValueType>().unwrap())
+        } else {
+            log::warn!("WeakLanceCache: cache no longer available, computing without caching");
+            loader().await.map(Arc::new)
+        }
+    }
+
+    pub async fn get_unsized_with_key<K>(&self, cache_key: &K) -> Option<Arc<K::ValueType>>
+    where
+        K: UnsizedCacheKey,
+        K::ValueType: DeepSizeOf + Send + Sync + 'static,
+    {
+        let cache = self.inner.upgrade()?;
+        let key = build_key(&self.prefix, &cache_key.key(), K::type_name());
+        if let Some(entry) = cache.get(&key, None).await {
+            entry
+                .downcast::<Arc<K::ValueType>>()
+                .ok()
+                .map(|arc| arc.as_ref().clone())
+        } else {
+            None
+        }
+    }
+
+    pub async fn insert_unsized_with_key<K>(&self, cache_key: &K, value: Arc<K::ValueType>)
+    where
+        K: UnsizedCacheKey,
+        K::ValueType: DeepSizeOf + Send + Sync + 'static,
+    {
+        if let Some(cache) = self.inner.upgrade() {
+            let wrapper = Arc::new(value);
+            let size = cache_entry_size(&*wrapper);
+            let key = build_key(&self.prefix, &cache_key.key(), K::type_name());
+            cache.insert(&key, wrapper, size, None).await;
+        } else {
+            log::warn!("WeakLanceCache: cache no longer available, unable to insert unsized item");
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// CacheStats
+// ---------------------------------------------------------------------------
+
+#[derive(Debug, Clone)]
+pub struct CacheStats {
+    /// Number of times `get`, `get_unsized`, or `get_or_insert` found an item in the cache.
+    pub hits: u64,
+    /// Number of times `get`, `get_unsized`, or `get_or_insert` did not find an item in the cache.
+    pub misses: u64,
+    /// Number of entries currently in the cache.
+    pub num_entries: usize,
+    /// Total size in bytes of all entries in the cache.
+    pub size_bytes: usize,
+}
+
+impl CacheStats {
+    pub fn hit_ratio(&self) -> f32 {
+        if self.hits + self.misses == 0 {
+            0.0
+        } else {
+            self.hits as f32 / (self.hits + self.misses) as f32
+        }
+    }
+
+    pub fn miss_ratio(&self) -> f32 {
+        if self.hits + self.misses == 0 {
+            0.0
+        } else {
+            self.misses as f32 / (self.hits + self.misses) as f32
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::HashMap;
+    use std::marker::PhantomData;
+
+    struct TestKey<T: 'static> {
+        key: String,
+        _phantom: PhantomData<T>,
+    }
+
+    impl<T: 'static> TestKey<T> {
+        fn new(key: &str) -> Self {
+            Self {
+                key: key.to_string(),
+                _phantom: PhantomData,
+            }
+        }
+    }
+
+    impl<T: 'static> CacheKey for TestKey<T> {
+        type ValueType = T;
+        fn key(&self) -> std::borrow::Cow<'_, str> {
+            std::borrow::Cow::Borrowed(&self.key)
+        }
+        fn type_name() -> &'static str {
+            std::any::type_name::<T>()
+        }
+    }
+
+    /// Test helper: an UnsizedCacheKey for trait object values.
+    struct TestUnsizedKey<T: 'static + ?Sized> {
+        key: String,
+        _phantom: PhantomData<T>,
+    }
+
+    impl<T: 'static + ?Sized> TestUnsizedKey<T> {
+        fn new(key: &str) -> Self {
+            Self {
+                key: key.to_string(),
+                _phantom: PhantomData,
+            }
+        }
+    }
+
+    impl<T: 'static + ?Sized> UnsizedCacheKey for TestUnsizedKey<T> {
+        type ValueType = T;
+        fn key(&self) -> std::borrow::Cow<'_, str> {
+            std::borrow::Cow::Borrowed(&self.key)
+        }
+        fn type_name() -> &'static str {
+            std::any::type_name::<T>()
+        }
+    }
+
+    #[tokio::test]
+    async fn test_cache_bytes() {
+        let item = Arc::new(vec![1, 2, 3]);
+        let item_size = item.deep_size_of();
+        let capacity = 10 * item_size;
+        let cache = LanceCache::with_capacity(capacity);
+
+        cache
+            .insert_with_key(&TestKey::<Vec<i32>>::new("key"), item.clone())
+            .await;
+        assert_eq!(cache.size().await, 1);
+
+        let retrieved = cache
+            .get_with_key(&TestKey::<Vec<i32>>::new("key"))
+            .await
+            .unwrap();
+        assert_eq!(*retrieved, *item);
+
+        for i in 0..20 {
+            cache
+                .insert_with_key(
+                    &TestKey::<Vec<i32>>::new(&format!("key_{}", i)),
+                    Arc::new(vec![i, i, i]),
+                )
+                .await;
+        }
+        assert!(cache.size_bytes().await <= capacity);
+    }
+
+    #[tokio::test]
+    async fn test_cache_trait_objects() {
+        #[derive(Debug, DeepSizeOf)]
+        struct MyType(i32);
+
+        trait MyTrait: DeepSizeOf + Send + Sync + std::any::Any {
+            fn as_any(&self) -> &dyn std::any::Any;
+        }
+
+        impl MyTrait for MyType {
+            fn as_any(&self) -> &dyn std::any::Any {
+                self
+            }
+        }
+
+        let item: Arc<dyn MyTrait> = Arc::new(MyType(42));
+        let cache = LanceCache::with_capacity(1000);
+        cache
+            .insert_unsized_with_key(&TestUnsizedKey::<dyn MyTrait>::new("test"), item)
+            .await;
+
+        let retrieved = cache
+            .get_unsized_with_key(&TestUnsizedKey::<dyn MyTrait>::new("test"))
+            .await
+            .unwrap();
+        assert_eq!(retrieved.as_any().downcast_ref::<MyType>().unwrap().0, 42);
+    }
+
+    #[tokio::test]
+    async fn test_cache_stats_basic() {
+        let cache = LanceCache::with_capacity(1000);
+        assert_eq!(cache.stats().await.hits, 0);
+
+        // Miss
+        assert!(
+            cache
+                .get_with_key(&TestKey::<Vec<i32>>::new("x"))
+                .await
+                .is_none()
+        );
+        assert_eq!(cache.stats().await.misses, 1);
+
+        // Insert then hit
+        cache
+            .insert_with_key(&TestKey::new("k"), Arc::new(vec![1, 2, 3]))
+            .await;
+        assert!(
+            cache
+                .get_with_key(&TestKey::<Vec<i32>>::new("k"))
+                .await
+                .is_some()
+        );
+        assert_eq!(cache.stats().await.hits, 1);
+    }
+
+    #[tokio::test]
+    async fn test_cache_stats_with_prefixes() {
+        let base = LanceCache::with_capacity(1000);
+        let prefixed = base.with_key_prefix("ns");
+
+        assert!(
+            prefixed
+                .get_with_key(&TestKey::<Vec<i32>>::new("k"))
+                .await
+                .is_none()
+        );
+        assert_eq!(base.stats().await.misses, 1);
+
+        prefixed
+            .insert_with_key(&TestKey::new("k"), Arc::new(vec![1]))
+            .await;
+        assert!(
+            prefixed
+                .get_with_key(&TestKey::<Vec<i32>>::new("k"))
+                .await
+                .is_some()
+        );
+        assert_eq!(base.stats().await.hits, 1);
+    }
+
+    #[tokio::test]
+    async fn test_cache_get_or_insert() {
+        let cache = LanceCache::with_capacity(1000);
+
+        let v: Arc<Vec<i32>> = cache
+            .get_or_insert_with_key(TestKey::<Vec<i32>>::new("k"), || async {
+                Ok(vec![1, 2, 3])
+            })
+            .await
+            .unwrap();
+        assert_eq!(*v, vec![1, 2, 3]);
+        assert_eq!(cache.stats().await.misses, 1);
+        assert_eq!(cache.stats().await.hits, 0);
+
+        // Second call should not invoke loader and should be a hit
+        let v: Arc<Vec<i32>> = cache
+            .get_or_insert_with_key(TestKey::<Vec<i32>>::new("k"), || async {
+                panic!("should not be called")
+            })
+            .await
+            .unwrap();
+        assert_eq!(*v, vec![1, 2, 3]);
+        assert_eq!(cache.stats().await.hits, 1);
+    }
+
+    #[tokio::test]
+    async fn test_custom_backend() {
+        use async_trait::async_trait;
+        use tokio::sync::Mutex;
+
+        #[derive(Debug)]
+        struct HashMapBackend {
+            map: Mutex<HashMap<InternalCacheKey, (CacheEntry, usize)>>,
+        }
+
+        impl HashMapBackend {
+            fn new() -> Self {
+                Self {
+                    map: Mutex::new(HashMap::new()),
+                }
+            }
+        }
+
+        #[async_trait]
+        impl CacheBackend for HashMapBackend {
+            async fn get(
+                &self,
+                key: &InternalCacheKey,
+                _codec: Option<CacheCodec>,
+            ) -> Option<CacheEntry> {
+                self.map.lock().await.get(key).map(|(e, _)| e.clone())
+            }
+            async fn insert(
+                &self,
+                key: &InternalCacheKey,
+                entry: CacheEntry,
+                size_bytes: usize,
+                _codec: Option<CacheCodec>,
+            ) {
+                self.map
+                    .lock()
+                    .await
+                    .insert(key.clone(), (entry, size_bytes));
+            }
+            async fn get_or_insert<'a>(
+                &self,
+                key: &InternalCacheKey,
+                loader: std::pin::Pin<
+                    Box<dyn futures::Future<Output = Result<(CacheEntry, usize)>> + Send + 'a>,
+                >,
+                _codec: Option<CacheCodec>,
+            ) -> Result<(CacheEntry, bool)> {
+                if let Some((entry, _)) = self.map.lock().await.get(key) {
+                    Ok((entry.clone(), true))
+                } else {
+                    let (entry, size) = loader.await?;
+                    self.map
+                        .lock()
+                        .await
+                        .insert(key.clone(), (entry.clone(), size));
+                    Ok((entry, false))
+                }
+            }
+            async fn invalidate_prefix(&self, prefix: &str) {
+                self.map.lock().await.retain(|k, _| !k.starts_with(prefix));
+            }
+            async fn clear(&self) {
+                self.map.lock().await.clear();
+            }
+            async fn num_entries(&self) -> usize {
+                self.map.lock().await.len()
+            }
+            async fn size_bytes(&self) -> usize {
+                self.map.lock().await.values().map(|(_, s)| *s).sum()
+            }
+        }
+
+        let cache = LanceCache::with_backend(Arc::new(HashMapBackend::new()));
+
+        cache
+            .insert_with_key(&TestKey::new("k"), Arc::new(vec![1, 2, 3]))
+            .await;
+        assert!(
+            cache
+                .get_with_key(&TestKey::<Vec<i32>>::new("k"))
+                .await
+                .is_some()
+        );
+        // Different type at same key = miss
+        assert!(
+            cache
+                .get_with_key(&TestKey::<Vec<u8>>::new("k"))
+                .await
+                .is_none()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_get_or_insert_dedup() {
+        use std::sync::atomic::AtomicUsize;
+
+        let load_count = Arc::new(AtomicUsize::new(0));
+        let cache = LanceCache::with_capacity(10000);
+
+        let (barrier_tx, _) = tokio::sync::broadcast::channel::<()>(1);
+        let mut handles = Vec::new();
+        for _ in 0..5 {
+            let cache = cache.clone();
+            let load_count = load_count.clone();
+            let mut barrier_rx = barrier_tx.subscribe();
+            handles.push(tokio::spawn(async move {
+                barrier_rx.recv().await.ok();
+                cache
+                    .get_or_insert_with_key(TestKey::<Vec<i32>>::new("key"), || {
+                        let load_count = load_count.clone();
+                        async move {
+                            load_count.fetch_add(1, Ordering::SeqCst);
+                            tokio::task::yield_now().await;
+                            Ok(vec![1, 2, 3])
+                        }
+                    })
+                    .await
+            }));
+        }
+        barrier_tx.send(()).unwrap();
+        for h in handles {
+            let result: Arc<Vec<i32>> = h.await.unwrap().unwrap();
+            assert_eq!(*result, vec![1, 2, 3]);
+        }
+
+        assert_eq!(load_count.load(Ordering::SeqCst), 1);
+    }
+}
diff --git a/rust/lance-core/src/cache/moka.rs b/rust/lance-core/src/cache/moka.rs
new file mode 100644
index 00000000000..6be7760458a
--- /dev/null
+++ b/rust/lance-core/src/cache/moka.rs
@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::pin::Pin;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+use async_trait::async_trait;
+use futures::Future;
+
+use crate::Result;
+
+use super::CacheCodec;
+use super::backend::{CacheBackend, CacheEntry, InternalCacheKey};
+
+/// Internal record stored in the moka cache.
+#[derive(Clone, Debug)]
+struct MokaCacheEntry {
+    entry: CacheEntry,
+    size_bytes: usize,
+}
+
+/// Default [`CacheBackend`] backed by a [moka](https://crates.io/crates/moka) cache.
+///
+/// Provides weighted-capacity eviction and concurrent-load deduplication
+/// via moka's built-in `optionally_get_with`.
+pub struct MokaCacheBackend {
+    cache: moka::future::Cache<InternalCacheKey, MokaCacheEntry>,
+}
+
+impl std::fmt::Debug for MokaCacheBackend {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("MokaCacheBackend")
+            .field("entry_count", &self.cache.entry_count())
+            .finish()
+    }
+}
+
+impl MokaCacheBackend {
+    pub fn with_capacity(capacity: usize) -> Self {
+        let cache = moka::future::Cache::builder()
+            .max_capacity(capacity as u64)
+            .weigher(|_, v: &MokaCacheEntry| v.size_bytes.try_into().unwrap_or(u32::MAX))
+            .support_invalidation_closures()
+            .build();
+        Self { cache }
+    }
+
+    pub fn no_cache() -> Self {
+        Self {
+            cache: moka::future::Cache::new(0),
+        }
+    }
+}
+
+#[async_trait]
+impl CacheBackend for MokaCacheBackend {
+    async fn get(&self, key: &InternalCacheKey, _codec: Option<CacheCodec>) -> Option<CacheEntry> {
+        self.cache.get(key).await.map(|r| r.entry)
+    }
+
+    async fn insert(
+        &self,
+        key: &InternalCacheKey,
+        entry: CacheEntry,
+        size_bytes: usize,
+        _codec: Option<CacheCodec>,
+    ) {
+        self.cache
+            .insert(key.clone(), MokaCacheEntry { entry, size_bytes })
+            .await;
+    }
+
+    async fn get_or_insert<'a>(
+        &self,
+        key: &InternalCacheKey,
+        loader: Pin<Box<dyn Future<Output = Result<(CacheEntry, usize)>> + Send + 'a>>,
+        _codec: Option<CacheCodec>,
+    ) -> Result<(CacheEntry, bool)> {
+        // Use moka's built-in dedup: optionally_get_with runs the init future
+        // at most once per key, even under concurrent access.
+        let (error_tx, error_rx) = tokio::sync::oneshot::channel();
+
+        // Track whether the loader actually ran (= cache miss).
+        let was_miss = Arc::new(AtomicBool::new(false));
+        let was_miss_clone = was_miss.clone();
+
+        let init = async move {
+            was_miss_clone.store(true, Ordering::Relaxed);
+            match loader.await {
+                Ok((entry, size_bytes)) => Some(MokaCacheEntry { entry, size_bytes }),
+                Err(e) => {
+                    let _ = error_tx.send(e);
+                    None
+                }
+            }
+        };
+
+        let owned_key = key.clone();
+        match self.cache.optionally_get_with(owned_key, init).await {
+            Some(record) => {
+                let was_cached = !was_miss.load(Ordering::Relaxed);
+                Ok((record.entry, was_cached))
+            }
+            None => match error_rx.await {
+                Ok(err) => Err(err),
+                Err(_) => Err(crate::Error::internal(
+                    "Failed to retrieve error from cache loader",
+                )),
+            },
+        }
+    }
+
+    async fn invalidate_prefix(&self, prefix: &str) {
+        let prefix = prefix.to_owned();
+        self.cache
+            .invalidate_entries_if(move |key, _value| key.starts_with(&prefix))
+            .expect("Cache configured correctly");
+    }
+
+    async fn clear(&self) {
+        self.cache.invalidate_all();
+        self.cache.run_pending_tasks().await;
+    }
+
+    async fn num_entries(&self) -> usize {
+        self.cache.run_pending_tasks().await;
+        self.cache.entry_count() as usize
+    }
+
+    async fn size_bytes(&self) -> usize {
+        self.cache.run_pending_tasks().await;
+        self.cache.weighted_size() as usize
+    }
+
+    fn approx_num_entries(&self) -> usize {
+        self.cache.entry_count() as usize
+    }
+
+    fn approx_size_bytes(&self) -> usize {
+        // Iterate rather than using `weighted_size()` because moka's
+        // weighted_size can be stale without `run_pending_tasks()`, which
+        // is async and can't be called from this synchronous context.
+        self.cache.iter().map(|(_, v)| v.size_bytes).sum()
+    }
+}
diff --git a/rust/lance-core/src/datatypes.rs b/rust/lance-core/src/datatypes.rs
index 1d6feaa0218..d395420e138 100644
--- a/rust/lance-core/src/datatypes.rs
+++ b/rust/lance-core/src/datatypes.rs
@@ -123,6 +123,17 @@ fn timeunit_to_str(unit: &TimeUnit) -> &'static str {
     }
 }
 
+fn is_supported_fixed_size_list_child(data_type: &DataType, nested: bool) -> bool {
+    match data_type {
+        DataType::Struct(_) => !nested,
+        DataType::List(_) | DataType::LargeList(_) | DataType::Map(_, _) => false,
+        DataType::FixedSizeList(field, _) => {
+            is_supported_fixed_size_list_child(field.data_type(), true)
+        }
+        _ => true,
+    }
+}
+
 fn parse_timeunit(unit: &str) -> Result<TimeUnit> {
     match unit {
         "s" => Ok(TimeUnit::Second),
@@ -192,6 +203,8 @@ impl TryFrom<&DataType> for LogicalType {
                     // Don't want to directly use `bfloat16`, in case a built-in type is added
                     // that isn't identical to our extension type.
                     format!("fixed_size_list:lance.bfloat16:{}", *len)
+                } else if !is_supported_fixed_size_list_child(field.data_type(), false) {
+                    return Err(Error::schema(format!("Unsupported data type: {:?}", dt)));
                 } else {
                     format!(
                         "fixed_size_list:{}:{}",
diff --git a/rust/lance-core/src/datatypes/field.rs b/rust/lance-core/src/datatypes/field.rs
index 917d591643e..c41eb449131 100644
--- a/rust/lance-core/src/datatypes/field.rs
+++ b/rust/lance-core/src/datatypes/field.rs
@@ -526,6 +526,14 @@ impl Field {
             .unwrap_or(false)
     }
 
+    // Blob columns intentionally have two schema representations:
+    // the loaded value view (legacy LargeBinary or blob v2 struct) and the unloaded
+    // descriptor view used by projection/planning. Schema set operations need to
+    // treat them as the same logical column instead of ordinary incompatible types.
+    fn is_compatible_blob_projection(&self, other: &Self) -> bool {
+        self.is_blob() && other.is_blob() && self.is_blob_v2() == other.is_blob_v2()
+    }
+
     /// If the field is a blob, update this field with the same name and id
     /// but with the data type set to a struct of the blob description fields.
     ///
@@ -640,6 +648,10 @@ impl Field {
             )));
         };
 
+        if self.is_compatible_blob_projection(other) {
+            return Ok(self.clone());
+        }
+
         match (self.data_type(), other.data_type()) {
             (DataType::Boolean, DataType::Boolean) => Ok(self.clone()),
             (dt, other_dt)
@@ -768,6 +780,14 @@ impl Field {
             )));
         }
 
+        if self.is_compatible_blob_projection(other) {
+            return Ok(if self.id >= 0 {
+                self.clone()
+            } else {
+                other.clone()
+            });
+        }
+
         let self_type = self.data_type();
         let other_type = other.data_type();
 
@@ -1790,4 +1810,25 @@ mod tests {
         assert_eq!(field.children.len(), 5);
         assert_eq!(field.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type);
     }
+
+    #[test]
+    fn project_by_field_accepts_blob_descriptor_projection() {
+        let metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]);
+        let field: Field = ArrowField::new("blob", DataType::LargeBinary, true)
+            .with_metadata(metadata)
+            .try_into()
+            .unwrap();
+        let mut unloaded = field.clone();
+        unloaded.unloaded_mut();
+
+        let projected = field
+            .project_by_field(&unloaded, OnTypeMismatch::Error)
+            .unwrap();
+        assert_eq!(projected, field);
+
+        let unloaded_projected = unloaded
+            .project_by_field(&field, OnTypeMismatch::Error)
+            .unwrap();
+        assert_eq!(unloaded_projected, unloaded);
+    }
 }
diff --git a/rust/lance-core/src/datatypes/schema.rs b/rust/lance-core/src/datatypes/schema.rs
index ab75149b8da..9f73b153982 100644
--- a/rust/lance-core/src/datatypes/schema.rs
+++ b/rust/lance-core/src/datatypes/schema.rs
@@ -85,7 +85,6 @@ struct SchemaFieldIterPreOrder<'a> {
 }
 
 impl<'a> SchemaFieldIterPreOrder<'a> {
-    #[allow(dead_code)]
     fn new(schema: &'a Schema) -> Self {
         let mut field_stack = Vec::with_capacity(schema.fields.len() * 2);
         for field in schema.fields.iter().rev() {
diff --git a/rust/lance-core/src/utils.rs b/rust/lance-core/src/utils.rs
index 663454e001b..565036311f9 100644
--- a/rust/lance-core/src/utils.rs
+++ b/rust/lance-core/src/utils.rs
@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 pub mod address;
+pub mod aimd;
 pub mod assume;
 pub mod backoff;
 pub mod bit;
diff --git a/rust/lance-core/src/utils/aimd.rs b/rust/lance-core/src/utils/aimd.rs
new file mode 100644
index 00000000000..0cbae68ca71
--- /dev/null
+++ b/rust/lance-core/src/utils/aimd.rs
@@ -0,0 +1,623 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! AIMD (Additive Increase / Multiplicative Decrease) rate controller.
+//!
+//! This module provides a reusable AIMD algorithm for dynamically adjusting
+//! request rates. On success windows, the rate increases additively. On
+//! windows with throttle signals, the rate decreases multiplicatively.
+//!
+//! The algorithm operates in discrete time windows. At the end of each window,
+//! the throttle ratio (throttled / total) is compared against a threshold:
+//! - Above threshold: `rate = max(rate * decrease_factor, min_rate)`
+//! - At or below threshold: `rate = min(rate + additive_increment, max_rate)`
+
+use std::sync::Mutex;
+use std::time::Duration;
+
+use crate::Result;
+
+/// Configuration for the AIMD rate controller.
+///
+/// Use builder methods to customize. Defaults are tuned for cloud object stores
+/// and will start at about 40% of the max rate and require 10 seconds to reach
+/// the max rate.
+///
+/// - initial_rate: 2000 req/s
+/// - min_rate: 1 req/s
+/// - max_rate: 5000 req/s (0.0 disables ceiling)
+/// - decrease_factor: 0.5 (halve on throttle)
+/// - additive_increment: 300 req/s per success window
+/// - window_duration: 1 second
+/// - throttle_threshold: 0.0 (any throttle triggers decrease)
+#[derive(Debug, Clone)]
+pub struct AimdConfig {
+    pub initial_rate: f64,
+    pub min_rate: f64,
+    pub max_rate: f64,
+    pub decrease_factor: f64,
+    pub additive_increment: f64,
+    pub window_duration: Duration,
+    pub throttle_threshold: f64,
+}
+
+impl Default for AimdConfig {
+    fn default() -> Self {
+        Self {
+            initial_rate: 2000.0,
+            min_rate: 1.0,
+            max_rate: 5000.0,
+            decrease_factor: 0.5,
+            additive_increment: 300.0,
+            window_duration: Duration::from_secs(1),
+            throttle_threshold: 0.0,
+        }
+    }
+}
+
+impl AimdConfig {
+    pub fn with_initial_rate(self, initial_rate: f64) -> Self {
+        Self {
+            initial_rate,
+            ..self
+        }
+    }
+
+    pub fn with_min_rate(self, min_rate: f64) -> Self {
+        Self { min_rate, ..self }
+    }
+
+    pub fn with_max_rate(self, max_rate: f64) -> Self {
+        Self { max_rate, ..self }
+    }
+
+    pub fn with_decrease_factor(self, decrease_factor: f64) -> Self {
+        Self {
+            decrease_factor,
+            ..self
+        }
+    }
+
+    pub fn with_additive_increment(self, additive_increment: f64) -> Self {
+        Self {
+            additive_increment,
+            ..self
+        }
+    }
+
+    pub fn with_window_duration(self, window_duration: Duration) -> Self {
+        Self {
+            window_duration,
+            ..self
+        }
+    }
+
+    pub fn with_throttle_threshold(self, throttle_threshold: f64) -> Self {
+        Self {
+            throttle_threshold,
+            ..self
+        }
+    }
+
+    /// Validate that the configuration values are sensible.
+    pub fn validate(&self) -> Result<()> {
+        if self.initial_rate <= 0.0 {
+            return Err(crate::Error::invalid_input(format!(
+                "initial_rate must be positive, got {}",
+                self.initial_rate
+            )));
+        }
+        if self.min_rate <= 0.0 {
+            return Err(crate::Error::invalid_input(format!(
+                "min_rate must be positive, got {}",
+                self.min_rate
+            )));
+        }
+        if self.max_rate < 0.0 {
+            return Err(crate::Error::invalid_input(format!(
+                "max_rate must be non-negative (0.0 = no ceiling), got {}",
+                self.max_rate
+            )));
+        }
+        if self.max_rate > 0.0 && self.min_rate > self.max_rate {
+            return Err(crate::Error::invalid_input(format!(
+                "min_rate ({}) must not exceed max_rate ({})",
+                self.min_rate, self.max_rate
+            )));
+        }
+        if self.decrease_factor <= 0.0 || self.decrease_factor >= 1.0 {
+            return Err(crate::Error::invalid_input(format!(
+                "decrease_factor must be in (0, 1), got {}",
+                self.decrease_factor
+            )));
+        }
+        if self.additive_increment <= 0.0 {
+            return Err(crate::Error::invalid_input(format!(
+                "additive_increment must be positive, got {}",
+                self.additive_increment
+            )));
+        }
+        if self.window_duration.is_zero() {
+            return Err(crate::Error::invalid_input(
+                "window_duration must be non-zero",
+            ));
+        }
+        if !(0.0..=1.0).contains(&self.throttle_threshold) {
+            return Err(crate::Error::invalid_input(format!(
+                "throttle_threshold must be in [0.0, 1.0], got {}",
+                self.throttle_threshold
+            )));
+        }
+        if self.max_rate > 0.0 && self.initial_rate > self.max_rate {
+            return Err(crate::Error::invalid_input(format!(
+                "initial_rate ({}) must not exceed max_rate ({})",
+                self.initial_rate, self.max_rate
+            )));
+        }
+        if self.initial_rate < self.min_rate {
+            return Err(crate::Error::invalid_input(format!(
+                "initial_rate ({}) must not be below min_rate ({})",
+                self.initial_rate, self.min_rate
+            )));
+        }
+        Ok(())
+    }
+}
+
+/// Outcome of a single request, used to feed the AIMD controller.
+///
+/// Non-throttle errors (e.g. 404, network timeout) should be mapped to
+/// `Success` since they don't indicate capacity problems.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RequestOutcome {
+    Success,
+    Throttled,
+}
+
+struct AimdState {
+    rate: f64,
+    window_start: std::time::Instant,
+    success_count: u64,
+    throttle_count: u64,
+}
+
+/// AIMD rate controller.
+///
+/// Thread-safe: uses an internal `Mutex` to protect state. The lock is held
+/// only briefly during `record_outcome` and `current_rate`.
+pub struct AimdController {
+    config: AimdConfig,
+    state: Mutex<AimdState>,
+}
+
+impl std::fmt::Debug for AimdController {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("AimdController")
+            .field("config", &self.config)
+            .field("rate", &self.current_rate())
+            .finish()
+    }
+}
+
+impl AimdController {
+    /// Create a new AIMD controller with the given configuration.
+    pub fn new(config: AimdConfig) -> Result<Self> {
+        config.validate()?;
+        let rate = config.initial_rate;
+        Ok(Self {
+            config,
+            state: Mutex::new(AimdState {
+                rate,
+                window_start: std::time::Instant::now(),
+                success_count: 0,
+                throttle_count: 0,
+            }),
+        })
+    }
+
+    /// Record a request outcome and return the current rate.
+    ///
+    /// If the current time window has expired, the rate is adjusted before
+    /// recording the new outcome in a fresh window.
+    pub fn record_outcome(&self, outcome: RequestOutcome) -> f64 {
+        let mut state = self.state.lock().unwrap();
+        self.record_outcome_inner(&mut state, outcome, std::time::Instant::now())
+    }
+
+    fn record_outcome_inner(
+        &self,
+        state: &mut AimdState,
+        outcome: RequestOutcome,
+        now: std::time::Instant,
+    ) -> f64 {
+        // Check if the window has expired
+        let elapsed = now.duration_since(state.window_start);
+        if elapsed >= self.config.window_duration {
+            let total = state.success_count + state.throttle_count;
+            if total > 0 {
+                let throttle_ratio = state.throttle_count as f64 / total as f64;
+                if throttle_ratio > self.config.throttle_threshold {
+                    // Multiplicative decrease
+                    state.rate =
+                        (state.rate * self.config.decrease_factor).max(self.config.min_rate);
+                } else {
+                    // Additive increase
+                    state.rate += self.config.additive_increment;
+                    if self.config.max_rate > 0.0 {
+                        state.rate = state.rate.min(self.config.max_rate);
+                    }
+                }
+            }
+            // Reset window
+            state.window_start = now;
+            state.success_count = 0;
+            state.throttle_count = 0;
+        }
+
+        // Record this outcome
+        match outcome {
+            RequestOutcome::Success => state.success_count += 1,
+            RequestOutcome::Throttled => state.throttle_count += 1,
+        }
+
+        state.rate
+    }
+
+    /// Get the current rate without recording an outcome.
+    pub fn current_rate(&self) -> f64 {
+        self.state.lock().unwrap().rate
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rstest::rstest;
+
+    #[rstest]
+    #[case::zero_initial_rate(
+        AimdConfig::default().with_initial_rate(0.0),
+        "initial_rate must be positive"
+    )]
+    #[case::negative_min_rate(
+        AimdConfig::default().with_min_rate(-1.0),
+        "min_rate must be positive"
+    )]
+    #[case::negative_max_rate(
+        AimdConfig::default().with_max_rate(-1.0),
+        "max_rate must be non-negative"
+    )]
+    #[case::min_exceeds_max(
+        AimdConfig::default().with_min_rate(100.0).with_max_rate(10.0),
+        "min_rate (100) must not exceed max_rate (10)"
+    )]
+    #[case::decrease_factor_zero(
+        AimdConfig::default().with_decrease_factor(0.0),
+        "decrease_factor must be in (0, 1)"
+    )]
+    #[case::decrease_factor_one(
+        AimdConfig::default().with_decrease_factor(1.0),
+        "decrease_factor must be in (0, 1)"
+    )]
+    #[case::decrease_factor_over_one(
+        AimdConfig::default().with_decrease_factor(1.5),
+        "decrease_factor must be in (0, 1)"
+    )]
+    #[case::zero_additive_increment(
+        AimdConfig::default().with_additive_increment(0.0),
+        "additive_increment must be positive"
+    )]
+    #[case::zero_window_duration(
+        AimdConfig::default().with_window_duration(Duration::ZERO),
+        "window_duration must be non-zero"
+    )]
+    #[case::threshold_over_one(
+        AimdConfig::default().with_throttle_threshold(1.1),
+        "throttle_threshold must be in [0.0, 1.0]"
+    )]
+    #[case::threshold_negative(
+        AimdConfig::default().with_throttle_threshold(-0.1),
+        "throttle_threshold must be in [0.0, 1.0]"
+    )]
+    #[case::initial_exceeds_max(
+        AimdConfig::default().with_initial_rate(6000.0),
+        "initial_rate (6000) must not exceed max_rate (5000)"
+    )]
+    #[case::initial_below_min(
+        AimdConfig::default().with_initial_rate(0.5).with_min_rate(1.0),
+        "initial_rate (0.5) must not be below min_rate (1)"
+    )]
+    fn test_config_validation_rejects_invalid(
+        #[case] config: AimdConfig,
+        #[case] expected_msg: &str,
+    ) {
+        let err = config.validate().unwrap_err();
+        let msg = err.to_string();
+        assert!(
+            msg.contains(expected_msg),
+            "Expected error containing '{}', got: {}",
+            expected_msg,
+            msg
+        );
+    }
+
+    #[test]
+    fn test_default_config_is_valid() {
+        AimdConfig::default().validate().unwrap();
+    }
+
+    #[test]
+    fn test_no_ceiling_config_is_valid() {
+        AimdConfig::default().with_max_rate(0.0).validate().unwrap();
+    }
+
+    #[test]
+    fn test_additive_increase_on_success_window() {
+        let config = AimdConfig::default()
+            .with_initial_rate(100.0)
+            .with_additive_increment(10.0)
+            .with_window_duration(Duration::from_millis(100));
+        let controller = AimdController::new(config).unwrap();
+
+        // Record some successes in the first window
+        let start = std::time::Instant::now();
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, start);
+        }
+
+        // Advance past the window boundary and record another success
+        let after_window = start + Duration::from_millis(150);
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, after_window);
+        }
+
+        // Rate should have increased by additive_increment
+        assert_eq!(controller.current_rate(), 110.0);
+    }
+
+    #[test]
+    fn test_multiplicative_decrease_on_throttle_window() {
+        let config = AimdConfig::default()
+            .with_initial_rate(100.0)
+            .with_decrease_factor(0.5)
+            .with_window_duration(Duration::from_millis(100));
+        let controller = AimdController::new(config).unwrap();
+
+        let start = std::time::Instant::now();
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Throttled, start);
+        }
+
+        // Advance past window
+        let after_window = start + Duration::from_millis(150);
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, after_window);
+        }
+
+        assert_eq!(controller.current_rate(), 50.0);
+    }
+
+    #[test]
+    fn test_floor_enforcement() {
+        let config = AimdConfig::default()
+            .with_initial_rate(2.0)
+            .with_min_rate(1.0)
+            .with_decrease_factor(0.5)
+            .with_window_duration(Duration::from_millis(100));
+        let controller = AimdController::new(config).unwrap();
+
+        let start = std::time::Instant::now();
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Throttled, start);
+        }
+
+        // After decrease: 2.0 * 0.5 = 1.0 (at floor)
+        let t1 = start + Duration::from_millis(150);
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Throttled, t1);
+        }
+        assert_eq!(controller.current_rate(), 1.0);
+
+        // Another decrease should stay at floor
+        let t2 = t1 + Duration::from_millis(150);
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, t2);
+        }
+        assert_eq!(controller.current_rate(), 1.0);
+    }
+
+    #[test]
+    fn test_ceiling_enforcement() {
+        let config = AimdConfig::default()
+            .with_initial_rate(4990.0)
+            .with_max_rate(5000.0)
+            .with_additive_increment(20.0)
+            .with_window_duration(Duration::from_millis(100));
+        let controller = AimdController::new(config).unwrap();
+
+        let start = std::time::Instant::now();
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, start);
+        }
+
+        let t1 = start + Duration::from_millis(150);
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, t1);
+        }
+        // 4990 + 20 = 5010, clamped to 5000
+        assert_eq!(controller.current_rate(), 5000.0);
+    }
+
+    #[test]
+    fn test_no_ceiling_allows_unbounded_growth() {
+        let config = AimdConfig::default()
+            .with_initial_rate(100.0)
+            .with_max_rate(0.0)
+            .with_additive_increment(50.0)
+            .with_window_duration(Duration::from_millis(100));
+        let controller = AimdController::new(config).unwrap();
+
+        let start = std::time::Instant::now();
+        let mut t = start;
+
+        for _ in 0..5 {
+            {
+                let mut state = controller.state.lock().unwrap();
+                controller.record_outcome_inner(&mut state, RequestOutcome::Success, t);
+            }
+            t += Duration::from_millis(150);
+        }
+
+        // Trigger final window evaluation
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, t);
+        }
+
+        // 100 + 50*5 = 350
+        assert_eq!(controller.current_rate(), 350.0);
+    }
+
+    #[test]
+    fn test_empty_window_no_adjustment() {
+        let config = AimdConfig::default()
+            .with_initial_rate(100.0)
+            .with_window_duration(Duration::from_millis(100));
+        let controller = AimdController::new(config).unwrap();
+
+        // Don't record anything in the first window, just advance time
+        let start = std::time::Instant::now();
+        let after = start + Duration::from_millis(150);
+        {
+            let mut state = controller.state.lock().unwrap();
+            // First outcome in a new window after empty window
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, after);
+        }
+        // No adjustment because the expired window had 0 total
+        assert_eq!(controller.current_rate(), 100.0);
+    }
+
+    #[test]
+    fn test_throttle_threshold_filtering() {
+        // With threshold 0.5, less than 50% throttles should still increase
+        let config = AimdConfig::default()
+            .with_initial_rate(100.0)
+            .with_throttle_threshold(0.5)
+            .with_additive_increment(10.0)
+            .with_window_duration(Duration::from_millis(100));
+        let controller = AimdController::new(config).unwrap();
+
+        let start = std::time::Instant::now();
+        {
+            let mut state = controller.state.lock().unwrap();
+            // 1 throttle out of 3 = 33% < 50% threshold
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, start);
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, start);
+            controller.record_outcome_inner(&mut state, RequestOutcome::Throttled, start);
+        }
+
+        // Advance past window
+        let t1 = start + Duration::from_millis(150);
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, t1);
+        }
+
+        // Should have increased because 33% <= 50%
+        assert_eq!(controller.current_rate(), 110.0);
+    }
+
+    #[test]
+    fn test_throttle_threshold_triggers_decrease() {
+        // With threshold 0.5, >= 50% throttles should decrease
+        let config = AimdConfig::default()
+            .with_initial_rate(100.0)
+            .with_throttle_threshold(0.5)
+            .with_decrease_factor(0.5)
+            .with_window_duration(Duration::from_millis(100));
+        let controller = AimdController::new(config).unwrap();
+
+        let start = std::time::Instant::now();
+        {
+            let mut state = controller.state.lock().unwrap();
+            // 2 throttle out of 3 = 67% > 50% threshold
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, start);
+            controller.record_outcome_inner(&mut state, RequestOutcome::Throttled, start);
+            controller.record_outcome_inner(&mut state, RequestOutcome::Throttled, start);
+        }
+
+        let t1 = start + Duration::from_millis(150);
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, t1);
+        }
+
+        assert_eq!(controller.current_rate(), 50.0);
+    }
+
+    #[test]
+    fn test_recovery_after_decrease() {
+        let config = AimdConfig::default()
+            .with_initial_rate(100.0)
+            .with_decrease_factor(0.5)
+            .with_additive_increment(10.0)
+            .with_window_duration(Duration::from_millis(100));
+        let controller = AimdController::new(config).unwrap();
+
+        let start = std::time::Instant::now();
+
+        // Window 1: throttle → decrease to 50
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Throttled, start);
+        }
+        let t1 = start + Duration::from_millis(150);
+
+        // Window 2: success → increase to 60
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, t1);
+        }
+        let t2 = t1 + Duration::from_millis(150);
+
+        // Window 3: success → increase to 70
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, t2);
+        }
+        let t3 = t2 + Duration::from_millis(150);
+
+        // Trigger final evaluation
+        {
+            let mut state = controller.state.lock().unwrap();
+            controller.record_outcome_inner(&mut state, RequestOutcome::Success, t3);
+        }
+
+        assert_eq!(controller.current_rate(), 70.0);
+    }
+
+    #[test]
+    fn test_within_window_no_adjustment() {
+        let config = AimdConfig::default()
+            .with_initial_rate(100.0)
+            .with_window_duration(Duration::from_secs(10));
+        let controller = AimdController::new(config).unwrap();
+
+        // Record many outcomes but all within the same window
+        for _ in 0..100 {
+            controller.record_outcome(RequestOutcome::Throttled);
+        }
+
+        // Rate should still be initial since window hasn't expired
+        assert_eq!(controller.current_rate(), 100.0);
+    }
+}
diff --git a/rust/lance-core/src/utils/bit.rs b/rust/lance-core/src/utils/bit.rs
index ba4b882691d..d0c9eaf5a83 100644
--- a/rust/lance-core/src/utils/bit.rs
+++ b/rust/lance-core/src/utils/bit.rs
@@ -110,7 +110,7 @@ pub fn log_2_ceil(val: u32) -> u32 {
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
     use crate::utils::bit::{is_pwr_two, log_2_ceil, pad_bytes, pad_bytes_to, pad_bytes_u64};
 
     #[test]
diff --git a/rust/lance-core/src/utils/deletion.rs b/rust/lance-core/src/utils/deletion.rs
index 0164d11fce6..5ddfc3348e5 100644
--- a/rust/lance-core/src/utils/deletion.rs
+++ b/rust/lance-core/src/utils/deletion.rs
@@ -32,7 +32,6 @@ impl DeepSizeOf for DeletionVector {
 }
 
 impl DeletionVector {
-    #[allow(dead_code)] // Used in tests
     pub fn len(&self) -> usize {
         match self {
             Self::NoDeletions => 0,
diff --git a/rust/lance-core/src/utils/futures.rs b/rust/lance-core/src/utils/futures.rs
index 88af99f1e4e..95a1c39aaae 100644
--- a/rust/lance-core/src/utils/futures.rs
+++ b/rust/lance-core/src/utils/futures.rs
@@ -8,7 +8,7 @@ use std::{
 };
 
 use futures::{Stream, StreamExt, stream::BoxStream};
-use pin_project::pin_project;
+use pin_project::{pin_project, pinned_drop};
 use tokio::sync::Semaphore;
 use tokio_util::sync::PollSemaphore;
 
@@ -264,13 +264,64 @@ impl<S: Stream> FinallyStreamExt<S> for S {
     }
 }
 
+/// A stream wrapper that calls a function when dropped.
+///
+/// Unlike [`FinallyStream`], which fires when the inner stream yields `None`,
+/// this fires when the wrapper is dropped — even if the stream was not fully
+/// consumed.
+#[pin_project(PinnedDrop)]
+pub struct OnDropStream<S: Stream, F: FnOnce()> {
+    #[pin]
+    stream: S,
+    f: Option<F>,
+}
+
+impl<S: Stream, F: FnOnce()> OnDropStream<S, F> {
+    pub fn new(stream: S, f: F) -> Self {
+        Self { stream, f: Some(f) }
+    }
+}
+
+impl<S: Stream, F: FnOnce()> Stream for OnDropStream<S, F> {
+    type Item = S::Item;
+
+    fn poll_next(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        self.project().stream.poll_next(cx)
+    }
+}
+
+#[pinned_drop]
+impl<S: Stream, F: FnOnce()> PinnedDrop for OnDropStream<S, F> {
+    fn drop(self: std::pin::Pin<&mut Self>) {
+        let this = self.project();
+        if let Some(f) = this.f.take() {
+            f();
+        }
+    }
+}
+
+pub trait StreamOnDropExt: Stream + Sized {
+    /// Wrap this stream so that `f` is called when the stream is dropped.
+    fn on_drop<F: FnOnce()>(self, f: F) -> OnDropStream<Self, F> {
+        OnDropStream::new(self, f)
+    }
+}
+
+impl<S: Stream> StreamOnDropExt for S {}
+
 #[cfg(test)]
 mod tests {
 
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicBool, Ordering};
+
     use futures::{FutureExt, StreamExt};
     use tokio_stream::wrappers::ReceiverStream;
 
-    use crate::utils::futures::{Capacity, SharedStreamExt};
+    use crate::utils::futures::{Capacity, SharedStreamExt, StreamOnDropExt};
 
     fn is_pending(fut: &mut (impl std::future::Future + Unpin)) -> bool {
         let noop_waker = futures::task::noop_waker();
@@ -388,4 +439,53 @@ mod tests {
             right_handle.await.unwrap();
         }
     }
+
+    #[tokio::test]
+    async fn test_on_drop_fires_on_early_drop() {
+        let called = Arc::new(AtomicBool::new(false));
+        let called_clone = called.clone();
+
+        let stream = futures::stream::iter(vec![1, 2, 3]);
+        let mut stream = stream.on_drop(move || {
+            called_clone.store(true, Ordering::SeqCst);
+        });
+
+        // Consume only one item, then drop
+        assert_eq!(stream.next().await, Some(1));
+        assert!(!called.load(Ordering::SeqCst));
+        drop(stream);
+        assert!(called.load(Ordering::SeqCst));
+    }
+
+    #[tokio::test]
+    async fn test_on_drop_fires_after_exhaustion() {
+        let called = Arc::new(AtomicBool::new(false));
+        let called_clone = called.clone();
+
+        let stream = futures::stream::iter(vec![1]);
+        let mut stream = stream.on_drop(move || {
+            called_clone.store(true, Ordering::SeqCst);
+        });
+
+        assert_eq!(stream.next().await, Some(1));
+        assert_eq!(stream.next().await, None);
+        assert!(!called.load(Ordering::SeqCst));
+        drop(stream);
+        assert!(called.load(Ordering::SeqCst));
+    }
+
+    #[tokio::test]
+    async fn test_on_drop_fires_without_polling() {
+        let called = Arc::new(AtomicBool::new(false));
+        let called_clone = called.clone();
+
+        let stream = futures::stream::iter(vec![1, 2, 3]);
+        let stream = stream.on_drop(move || {
+            called_clone.store(true, Ordering::SeqCst);
+        });
+
+        assert!(!called.load(Ordering::SeqCst));
+        drop(stream);
+        assert!(called.load(Ordering::SeqCst));
+    }
 }
diff --git a/rust/lance-core/src/utils/testing.rs b/rust/lance-core/src/utils/testing.rs
index f19d66f67e2..beec69270a5 100644
--- a/rust/lance-core/src/utils/testing.rs
+++ b/rust/lance-core/src/utils/testing.rs
@@ -17,7 +17,11 @@ use std::collections::HashMap;
 use std::fmt::Debug;
 use std::future;
 use std::ops::Range;
-use std::sync::{Arc, Mutex};
+use std::pin::Pin;
+use std::sync::{
+    Arc, Mutex,
+    atomic::{AtomicUsize, Ordering},
+};
 
 // A policy function takes in the name of the operation (e.g. "put") and the location
 // that is being accessed / modified and returns an optional error.
@@ -121,6 +125,42 @@ impl std::fmt::Display for ProxyObjectStore {
     }
 }
 
+/// An object store wrapper that counts listing operations.
+///
+/// This increments the shared counter for both `list` and `list_with_delimiter`
+/// so tests can observe all listing-based directory and version discovery calls.
+#[derive(Debug)]
+pub struct CountingObjectStore {
+    target: Arc<dyn ObjectStore>,
+    listing_count: Arc<AtomicUsize>,
+}
+
+impl CountingObjectStore {
+    pub fn new(target: Arc<dyn ObjectStore>, listing_count: Arc<AtomicUsize>) -> Self {
+        Self {
+            target,
+            listing_count,
+        }
+    }
+
+    fn record_listing(&self) {
+        self.listing_count.fetch_add(1, Ordering::SeqCst);
+    }
+
+    fn delegate_list(
+        &self,
+        prefix: Option<&Path>,
+    ) -> Pin<Box<dyn futures::Stream<Item = OSResult<ObjectMeta>> + Send>> {
+        self.target.list(prefix)
+    }
+}
+
+impl std::fmt::Display for CountingObjectStore {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "CountingObjectStore({})", self.target)
+    }
+}
+
 #[async_trait]
 impl ObjectStore for ProxyObjectStore {
     async fn put_opts(
@@ -204,3 +244,65 @@ impl ObjectStore for ProxyObjectStore {
         self.target.copy_if_not_exists(from, to).await
     }
 }
+
+#[async_trait]
+impl ObjectStore for CountingObjectStore {
+    async fn put_opts(
+        &self,
+        location: &Path,
+        bytes: PutPayload,
+        opts: PutOptions,
+    ) -> OSResult<PutResult> {
+        self.target.put_opts(location, bytes, opts).await
+    }
+
+    async fn put_multipart_opts(
+        &self,
+        location: &Path,
+        opts: PutMultipartOptions,
+    ) -> OSResult<Box<dyn MultipartUpload>> {
+        self.target.put_multipart_opts(location, opts).await
+    }
+
+    async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+        self.target.get_opts(location, options).await
+    }
+
+    async fn get_range(&self, location: &Path, range: Range<u64>) -> OSResult<Bytes> {
+        self.target.get_range(location, range).await
+    }
+
+    async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+        self.target.get_ranges(location, ranges).await
+    }
+
+    async fn head(&self, location: &Path) -> OSResult<ObjectMeta> {
+        self.target.head(location).await
+    }
+
+    async fn delete(&self, location: &Path) -> OSResult<()> {
+        self.target.delete(location).await
+    }
+
+    fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+        self.record_listing();
+        self.delegate_list(prefix).boxed()
+    }
+
+    async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult<ListResult> {
+        self.record_listing();
+        self.target.list_with_delimiter(prefix).await
+    }
+
+    async fn copy(&self, from: &Path, to: &Path) -> OSResult<()> {
+        self.target.copy(from, to).await
+    }
+
+    async fn rename(&self, from: &Path, to: &Path) -> OSResult<()> {
+        self.target.rename(from, to).await
+    }
+
+    async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
+        self.target.copy_if_not_exists(from, to).await
+    }
+}
diff --git a/rust/lance-datafusion/Cargo.toml b/rust/lance-datafusion/Cargo.toml
index 2a79173c0c0..3c370cfa3c4 100644
--- a/rust/lance-datafusion/Cargo.toml
+++ b/rust/lance-datafusion/Cargo.toml
@@ -13,6 +13,7 @@ version.workspace = true
 arrow = {workspace = true, features = ["ffi"]}
 arrow-array = {workspace = true, features = ["ffi"]}
 arrow-buffer.workspace = true
+arrow-cast.workspace = true
 arrow-ord.workspace = true
 arrow-schema.workspace = true
 arrow-select.workspace = true
diff --git a/rust/lance-datafusion/src/exec.rs b/rust/lance-datafusion/src/exec.rs
index 5eb83b8c19e..108ec8f5fad 100644
--- a/rust/lance-datafusion/src/exec.rs
+++ b/rust/lance-datafusion/src/exec.rs
@@ -313,21 +313,22 @@ impl std::fmt::Debug for LanceExecutionOptions {
     }
 }
 
-const DEFAULT_LANCE_MEM_POOL_SIZE: u64 = 100 * 1024 * 1024;
+const DEFAULT_LANCE_MEM_POOL_SIZE_PER_PARTITION: u64 = 100 * 1024 * 1024;
 const DEFAULT_LANCE_MAX_TEMP_DIRECTORY_SIZE: u64 = 100 * 1024 * 1024 * 1024; // 100GB
 
 impl LanceExecutionOptions {
     pub fn mem_pool_size(&self) -> u64 {
+        let num_partitions = self.target_partition.unwrap_or(1) as u64;
         self.mem_pool_size.unwrap_or_else(|| {
             std::env::var("LANCE_MEM_POOL_SIZE")
                 .map(|s| match s.parse::<u64>() {
                     Ok(v) => v,
                     Err(e) => {
                         warn!("Failed to parse LANCE_MEM_POOL_SIZE: {}, using default", e);
-                        DEFAULT_LANCE_MEM_POOL_SIZE
+                        DEFAULT_LANCE_MEM_POOL_SIZE_PER_PARTITION * num_partitions
                     }
                 })
-                .unwrap_or(DEFAULT_LANCE_MEM_POOL_SIZE)
+                .unwrap_or(DEFAULT_LANCE_MEM_POOL_SIZE_PER_PARTITION * num_partitions)
         })
     }
 
@@ -489,6 +490,9 @@ pub struct ExecutionSummaryCounts {
     /// Additional metrics for more detailed statistics.  These are subject to change in the future
     /// and should only be used for debugging purposes.
     pub all_counts: HashMap<String, usize>,
+    /// Additional time metrics for more detailed statistics, stored in nanoseconds.
+    /// These are subject to change in the future and should only be used for debugging purposes.
+    pub all_times: HashMap<String, usize>,
 }
 
 pub fn collect_execution_metrics(node: &dyn ExecutionPlan, counts: &mut ExecutionSummaryCounts) {
@@ -510,6 +514,13 @@ pub fn collect_execution_metrics(node: &dyn ExecutionPlan, counts: &mut Executio
                 }
             }
         }
+        for (metric_name, time) in metrics.iter_times() {
+            let existing = counts
+                .all_times
+                .entry(metric_name.as_ref().to_string())
+                .or_insert(0);
+            *existing += time.value();
+        }
         // Include gauge-based I/O metrics (some nodes record I/O as gauges)
         for (metric_name, gauge) in metrics.iter_gauges() {
             match metric_name.as_ref() {
@@ -1041,4 +1052,35 @@ mod tests {
             );
         }
     }
+
+    #[test]
+    fn test_mem_pool_size_scales_with_partitions() {
+        let default_per_partition = DEFAULT_LANCE_MEM_POOL_SIZE_PER_PARTITION;
+
+        // No partitions specified → defaults to 1 partition
+        let opts = LanceExecutionOptions::default();
+        assert_eq!(opts.mem_pool_size(), default_per_partition);
+
+        // 4 partitions → 4x the per-partition size
+        let opts = LanceExecutionOptions {
+            target_partition: Some(4),
+            ..Default::default()
+        };
+        assert_eq!(opts.mem_pool_size(), default_per_partition * 4);
+
+        // 8 partitions → 8x the per-partition size
+        let opts = LanceExecutionOptions {
+            target_partition: Some(8),
+            ..Default::default()
+        };
+        assert_eq!(opts.mem_pool_size(), default_per_partition * 8);
+
+        // Explicit mem_pool_size is not scaled
+        let opts = LanceExecutionOptions {
+            mem_pool_size: Some(50 * 1024 * 1024),
+            target_partition: Some(8),
+            ..Default::default()
+        };
+        assert_eq!(opts.mem_pool_size(), 50 * 1024 * 1024);
+    }
 }
diff --git a/rust/lance-datafusion/src/logical_expr.rs b/rust/lance-datafusion/src/logical_expr.rs
index 02775f1bbbf..3fde6cf7c82 100644
--- a/rust/lance-datafusion/src/logical_expr.rs
+++ b/rust/lance-datafusion/src/logical_expr.rs
@@ -295,7 +295,7 @@ pub fn field_path_to_expr(field_path: &str) -> Result<Expr> {
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
     use std::sync::Arc;
 
     use super::*;
diff --git a/rust/lance-datafusion/src/planner.rs b/rust/lance-datafusion/src/planner.rs
index 5d998d0ae6b..1b163963e61 100644
--- a/rust/lance-datafusion/src/planner.rs
+++ b/rust/lance-datafusion/src/planner.rs
@@ -14,6 +14,7 @@ use crate::sql::{parse_sql_expr, parse_sql_filter};
 use arrow::compute::CastOptions;
 use arrow_array::ListArray;
 use arrow_buffer::OffsetBuffer;
+use arrow_cast::cast_with_options;
 use arrow_schema::{DataType as ArrowDataType, Field, SchemaRef, TimeUnit};
 use arrow_select::concat::concat;
 use datafusion::common::DFSchema;
@@ -45,7 +46,6 @@ use datafusion::{
     scalar::ScalarValue,
 };
 use datafusion_functions::core::getfield::GetFieldFunc;
-use lance_arrow::cast::cast_with_options;
 use lance_core::datatypes::Schema;
 use lance_core::error::LanceOptionExt;
 
diff --git a/rust/lance-datafusion/src/substrait.rs b/rust/lance-datafusion/src/substrait.rs
index 8375c49abb9..1c465fcae4a 100644
--- a/rust/lance-datafusion/src/substrait.rs
+++ b/rust/lance-datafusion/src/substrait.rs
@@ -1127,4 +1127,40 @@ mod tests {
             agg.aggregates[1].schema_name()
         );
     }
+
+    // ==================== LIKE and starts_with tests ====================
+
+    #[tokio::test]
+    async fn test_substrait_roundtrip_like() {
+        use datafusion::logical_expr::Like;
+
+        let schema = Schema::new(vec![Field::new("name", DataType::Utf8, true)]);
+
+        let like_expr = Expr::Like(Like {
+            negated: false,
+            expr: Box::new(Expr::Column(Column::new_unqualified("name"))),
+            pattern: Box::new(Expr::Literal(
+                ScalarValue::Utf8(Some("test%".to_string())),
+                None,
+            )),
+            escape_char: None,
+            case_insensitive: false,
+        });
+
+        assert_substrait_roundtrip(schema, like_expr).await;
+    }
+
+    #[tokio::test]
+    async fn test_substrait_roundtrip_starts_with() {
+        use datafusion::functions::string::starts_with;
+
+        let schema = Schema::new(vec![Field::new("name", DataType::Utf8, true)]);
+
+        let starts_with_expr = starts_with().call(vec![
+            Expr::Column(Column::new_unqualified("name")),
+            Expr::Literal(ScalarValue::Utf8(Some("prefix".to_string())), None),
+        ]);
+
+        assert_substrait_roundtrip(schema, starts_with_expr).await;
+    }
 }
diff --git a/rust/lance-datafusion/src/utils.rs b/rust/lance-datafusion/src/utils.rs
index f6f28f766f0..3e1d2db2d79 100644
--- a/rust/lance-datafusion/src/utils.rs
+++ b/rust/lance-datafusion/src/utils.rs
@@ -155,6 +155,7 @@ pub fn reader_to_stream(batches: Box<dyn RecordBatchReader + Send>) -> SendableR
 pub trait MetricsExt {
     fn find_count(&self, name: &str) -> Option<Count>;
     fn iter_counts(&self) -> impl Iterator<Item = (impl AsRef<str>, &Count)>;
+    fn iter_times(&self) -> impl Iterator<Item = (impl AsRef<str>, &Time)>;
     fn iter_gauges(&self) -> impl Iterator<Item = (impl AsRef<str>, &Gauge)>;
 }
 
@@ -179,6 +180,13 @@ impl MetricsExt for MetricsSet {
         })
     }
 
+    fn iter_times(&self) -> impl Iterator<Item = (impl AsRef<str>, &Time)> {
+        self.iter().filter_map(|m| match m.value() {
+            MetricValue::Time { name, time } => Some((name, time)),
+            _ => None,
+        })
+    }
+
     fn iter_gauges(&self) -> impl Iterator<Item = (impl AsRef<str>, &Gauge)> {
         self.iter().filter_map(|m| match m.value() {
             MetricValue::Gauge { name, gauge } => Some((name, gauge)),
@@ -242,5 +250,6 @@ pub const ROWS_SCANNED_METRIC: &str = "rows_scanned";
 pub const TASK_WAIT_TIME_METRIC: &str = "task_wait_time";
 pub const DELTAS_SEARCHED_METRIC: &str = "deltas_searched";
 pub const PARTITIONS_SEARCHED_METRIC: &str = "partitions_searched";
+pub const FIND_PARTITIONS_ELAPSED_METRIC: &str = "find_partitions_elapsed";
 pub const SCALAR_INDEX_SEARCH_TIME_METRIC: &str = "search_time";
 pub const SCALAR_INDEX_SER_TIME_METRIC: &str = "ser_time";
diff --git a/rust/lance-encoding/Cargo.toml b/rust/lance-encoding/Cargo.toml
index c8f23f9b914..a12acd756e9 100644
--- a/rust/lance-encoding/Cargo.toml
+++ b/rust/lance-encoding/Cargo.toml
@@ -53,6 +53,7 @@ test-log.workspace = true
 criterion = { workspace = true }
 rand_xoshiro = { workspace = true }
 proptest.workspace = true
+serial_test.workspace = true
 
 [build-dependencies]
 prost-build.workspace = true
diff --git a/rust/lance-encoding/benches/decoder.rs b/rust/lance-encoding/benches/decoder.rs
index 939087d8a62..efabee87be6 100644
--- a/rust/lance-encoding/benches/decoder.rs
+++ b/rust/lance-encoding/benches/decoder.rs
@@ -300,7 +300,7 @@ fn bench_decode_packed_struct(c: &mut Criterion) {
     });
 }
 
-#[allow(dead_code)]
+#[cfg(target_os = "linux")]
 fn bench_decode_str_with_fixed_size_binary_encoding(c: &mut Criterion) {
     let rt = tokio::runtime::Runtime::new().unwrap();
     let mut group = c.benchmark_group("decode_primitive");
@@ -541,6 +541,7 @@ fn bench_decode_compressed_parallel(c: &mut Criterion) {
                                 false,
                                 false,
                                 rx,
+                                None,
                             )
                             .unwrap();
 
diff --git a/rust/lance-encoding/src/data.rs b/rust/lance-encoding/src/data.rs
index f2cf9c5b825..fa4ffe3021e 100644
--- a/rust/lance-encoding/src/data.rs
+++ b/rust/lance-encoding/src/data.rs
@@ -970,12 +970,8 @@ impl DataBlock {
             Self::FixedWidth(inner) => inner.data_size(),
             Self::FixedSizeList(inner) => inner.data_size(),
             Self::VariableWidth(inner) => inner.data_size(),
-            Self::Struct(_) => {
-                todo!("the data_size method for StructDataBlock is not implemented yet")
-            }
-            Self::Dictionary(_) => {
-                todo!("the data_size method for DictionaryDataBlock is not implemented yet")
-            }
+            Self::Struct(inner) => inner.children.iter().map(|child| child.data_size()).sum(),
+            Self::Dictionary(inner) => inner.indices.data_size() + inner.dictionary.data_size(),
             Self::Opaque(inner) => inner.data_size(),
         }
     }
diff --git a/rust/lance-encoding/src/decoder.rs b/rust/lance-encoding/src/decoder.rs
index ddbc3da38e3..41e20688037 100644
--- a/rust/lance-encoding/src/decoder.rs
+++ b/rust/lance-encoding/src/decoder.rs
@@ -213,6 +213,7 @@
 //!    relation to the way the data is stored.
 
 use std::collections::VecDeque;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{LazyLock, Once, OnceLock};
 use std::{ops::Range, sync::Arc};
 
@@ -226,7 +227,7 @@ use futures::{FutureExt, StreamExt};
 use lance_arrow::DataTypeExt;
 use lance_core::cache::LanceCache;
 use lance_core::datatypes::{BLOB_DESC_LANCE_FIELD, Field, Schema};
-use lance_core::utils::futures::FinallyStreamExt;
+use lance_core::utils::futures::{FinallyStreamExt, StreamOnDropExt};
 use lance_core::utils::parse::parse_env_as_bool;
 use log::{debug, trace, warn};
 use tokio::sync::mpsc::error::SendError;
@@ -1472,9 +1473,13 @@ impl BatchDecodeStream {
                     // Real decode work happens inside into_batch, which can block the current
                     // thread for a long time. By spawning it as a new task, we allow Tokio's
                     // worker threads to keep making progress.
-                    tokio::spawn(async move { next_task.into_batch(emitted_batch_size_warning) })
+                    let (batch, _data_size) =
+                        tokio::spawn(
+                            async move { next_task.into_batch(emitted_batch_size_warning) },
+                        )
                         .await
-                        .map_err(|err| Error::wrapped(err.into()))?
+                        .map_err(|err| Error::wrapped(err.into()))??;
+                    Ok(batch)
                 };
                 (task, num_rows)
             });
@@ -1660,7 +1665,7 @@ impl<T: RootDecoderType> BatchDecodeIterator<T> {
 
         self.rows_drained += to_take;
 
-        let batch = next_task.into_batch(self.emitted_batch_size_warning.clone())?;
+        let (batch, _data_size) = next_task.into_batch(self.emitted_batch_size_warning.clone())?;
 
         Ok(Some(batch))
     }
@@ -1682,6 +1687,39 @@ impl<T: RootDecoderType> RecordBatchReader for BatchDecodeIterator<T> {
     }
 }
 
+/// Estimate the number of bytes per row for a given Arrow data type.
+///
+/// For fixed-width types this is exact. For variable-width types (strings,
+/// binary, lists) a rough default is used. The estimate is used as a
+/// starting point when `batch_size_bytes` is set; a post-decode feedback
+/// loop corrects it after the first batch.
+///
+/// This estimate ignores validity bitmaps at the moment.  We can't infer
+/// their presence simply from the data_type and their impact is probably
+/// fairly negligible.
+fn estimate_bytes_per_row(data_type: &DataType) -> f64 {
+    if let Some(w) = data_type.byte_width_opt() {
+        return w as f64;
+    }
+    match data_type {
+        DataType::Boolean => 1.0 / 8.0,
+        DataType::Utf8 | DataType::Binary | DataType::LargeUtf8 | DataType::LargeBinary => 64.0,
+        DataType::Struct(fields) => fields
+            .iter()
+            .map(|f| estimate_bytes_per_row(f.data_type()))
+            .sum(),
+        DataType::List(child) | DataType::LargeList(child) => {
+            5.0 * estimate_bytes_per_row(child.data_type())
+        }
+        DataType::FixedSizeList(child, dim) => {
+            *dim as f64 * estimate_bytes_per_row(child.data_type())
+        }
+        DataType::Dictionary(_, value_type) => estimate_bytes_per_row(value_type),
+        DataType::Map(entries, _) => 5.0 * estimate_bytes_per_row(entries.data_type()),
+        _ => 64.0,
+    }
+}
+
 /// A stream that takes scheduled jobs and generates decode tasks from them.
 pub struct StructuralBatchDecodeStream {
     context: DecoderContext,
@@ -1700,6 +1738,14 @@ pub struct StructuralBatchDecodeStream {
     // - false: run `into_batch` inline, which avoids Tokio scheduling overhead and is
     //   typically better for point lookups / small takes.
     spawn_batch_decode_tasks: bool,
+    /// If set, target this many bytes per batch instead of `rows_per_batch` rows.
+    batch_size_bytes: Option<u64>,
+    /// Schema-based estimate of bytes per row, computed once at construction.
+    /// Only meaningful when `batch_size_bytes` is `Some`.
+    schema_bytes_per_row: f64,
+    /// Post-decode feedback: actual bytes-per-row measured from the most
+    /// recently decoded batch.  Zero means no feedback yet (use schema estimate).
+    bytes_per_row_feedback: Arc<AtomicU64>,
 }
 
 impl StructuralBatchDecodeStream {
@@ -1718,7 +1764,13 @@ impl StructuralBatchDecodeStream {
         num_rows: u64,
         root_decoder: StructuralStructDecoder,
         spawn_batch_decode_tasks: bool,
+        batch_size_bytes: Option<u64>,
     ) -> Self {
+        let schema_bytes_per_row = if batch_size_bytes.is_some() {
+            estimate_bytes_per_row(root_decoder.data_type()).max(1.0)
+        } else {
+            0.0
+        };
         Self {
             context: DecoderContext::new(scheduled),
             root_decoder,
@@ -1729,6 +1781,9 @@ impl StructuralBatchDecodeStream {
             scheduler_exhausted: false,
             emitted_batch_size_warning: Arc::new(Once::new()),
             spawn_batch_decode_tasks,
+            batch_size_bytes,
+            schema_bytes_per_row,
+            bytes_per_row_feedback: Arc::new(AtomicU64::new(0)),
         }
     }
 
@@ -1771,7 +1826,18 @@ impl StructuralBatchDecodeStream {
             return Ok(None);
         }
 
-        let mut to_take = self.rows_remaining.min(self.rows_per_batch as u64);
+        let mut to_take = if let Some(batch_size_bytes) = self.batch_size_bytes {
+            let feedback = self.bytes_per_row_feedback.load(Ordering::Relaxed);
+            let bpr = if feedback > 0 {
+                feedback as f64
+            } else {
+                self.schema_bytes_per_row
+            };
+            let rows = (batch_size_bytes as f64 / bpr) as u64;
+            self.rows_remaining.min(rows.max(1))
+        } else {
+            self.rows_remaining.min(self.rows_per_batch as u64)
+        };
         self.rows_remaining -= to_take;
 
         let scheduled_need = (self.rows_drained + to_take).saturating_sub(self.rows_scheduled);
@@ -1807,20 +1873,38 @@ impl StructuralBatchDecodeStream {
             let next_task = next_task.transpose().map(|next_task| {
                 let num_rows = next_task.as_ref().map(|t| t.num_rows).unwrap_or(0);
                 let emitted_batch_size_warning = slf.emitted_batch_size_warning.clone();
+                let bytes_per_row_feedback = slf.bytes_per_row_feedback.clone();
                 // Capture the per-stream policy once so every emitted batch task follows the
                 // same throughput-vs-overhead choice made by the scheduler.
                 let spawn_batch_decode_tasks = slf.spawn_batch_decode_tasks;
                 let task = async move {
                     let next_task = next_task?;
-                    if spawn_batch_decode_tasks {
+                    let (batch, data_size) = if spawn_batch_decode_tasks {
                         tokio::spawn(
                             async move { next_task.into_batch(emitted_batch_size_warning) },
                         )
                         .await
-                        .map_err(|err| Error::wrapped(err.into()))?
+                        .map_err(|err| Error::wrapped(err.into()))??
                     } else {
-                        next_task.into_batch(emitted_batch_size_warning)
+                        next_task.into_batch(emitted_batch_size_warning)?
+                    };
+                    let num_rows = batch.num_rows() as u64;
+                    if num_rows > 0 {
+                        let bpr = data_size / num_rows;
+                        let prev = bytes_per_row_feedback.load(Ordering::Relaxed);
+                        let next = if prev == 0 || bpr >= prev {
+                            // First batch or actual size is larger than estimate:
+                            // adopt immediately to avoid OOM.
+                            bpr
+                        } else {
+                            // Actual size is smaller: degrade gradually toward
+                            // the true value to avoid over-correcting on a
+                            // single anomalous batch.
+                            (prev + bpr) / 2
+                        };
+                        bytes_per_row_feedback.store(next.max(1), Ordering::Relaxed);
                     }
+                    Ok(batch)
                 };
                 (task, num_rows)
             });
@@ -1891,6 +1975,11 @@ pub struct SchedulerDecoderConfig {
     pub cache: Arc<LanceCache>,
     /// Decoder configuration
     pub decoder_config: DecoderConfig,
+    /// If set, target this many bytes per batch instead of using `batch_size` rows.
+    ///
+    /// Only supported for v2.1+ (structural) files. For v2.0 files this
+    /// option is ignored and a warning is logged.
+    pub batch_size_bytes: Option<u64>,
 }
 
 fn check_scheduler_on_drop(
@@ -1900,6 +1989,7 @@ fn check_scheduler_on_drop(
     // This is a bit weird but we create an "empty stream" that unwraps the scheduler handle (which
     // will panic if the scheduler panicked).  This let's us check if the scheduler panicked
     // when the stream finishes.
+    let abort_handle = scheduler_handle.abort_handle();
     let mut scheduler_handle = Some(scheduler_handle);
     let check_scheduler = stream::unfold((), move |_| {
         let handle = scheduler_handle.take();
@@ -1910,9 +2000,22 @@ fn check_scheduler_on_drop(
             None
         }
     });
-    stream.chain(check_scheduler).boxed()
+    stream
+        .chain(check_scheduler)
+        .on_drop(move || {
+            // Abort the scheduler task on early drop. The scheduler task holds
+            // a reference to the I/O scheduler (via config.io) which keeps the
+            // ScanScheduler alive. If the scheduler task is stuck waiting for
+            // initialization I/O (which is blocked on backpressure that will
+            // never drain because no one is consuming the stream), we need to
+            // abort it so it releases its I/O reference and allows the
+            // ScanScheduler to drop and cancel pending I/O.
+            abort_handle.abort();
+        })
+        .boxed()
 }
 
+#[allow(clippy::too_many_arguments)]
 pub fn create_decode_stream(
     schema: &Schema,
     num_rows: u64,
@@ -1921,6 +2024,7 @@ pub fn create_decode_stream(
     should_validate: bool,
     spawn_structural_batch_decode_tasks: bool,
     rx: mpsc::UnboundedReceiver<Result<DecoderMessage>>,
+    batch_size_bytes: Option<u64>,
 ) -> Result<BoxStream<'static, ReadBatchTask>> {
     if is_structural {
         let arrow_schema = ArrowSchema::from(schema);
@@ -1935,9 +2039,13 @@ pub fn create_decode_stream(
             num_rows,
             structural_decoder,
             spawn_structural_batch_decode_tasks,
+            batch_size_bytes,
         )
         .into_stream())
     } else {
+        if batch_size_bytes.is_some() {
+            warn!("batch_size_bytes is not supported for v2.0 files and will be ignored");
+        }
         let arrow_schema = ArrowSchema::from(schema);
         let root_fields = arrow_schema.fields;
 
@@ -2009,6 +2117,7 @@ fn create_scheduler_decoder(
         config.decoder_config.validate_on_decode,
         spawn_structural_batch_decode_tasks,
         rx,
+        config.batch_size_bytes,
     )?;
 
     let scheduler_handle = tokio::task::spawn(async move {
@@ -2503,13 +2612,14 @@ pub trait StructuralFieldScheduler: Send + std::fmt::Debug {
 
 /// A trait for tasks that decode data into an Arrow array
 pub trait DecodeArrayTask: Send {
-    /// Decodes the data into an Arrow array
-    fn decode(self: Box<Self>) -> Result<ArrayRef>;
+    /// Decodes the data into an Arrow array and its data size in bytes
+    fn decode(self: Box<Self>) -> Result<(ArrayRef, u64)>;
 }
 
 impl DecodeArrayTask for Box<dyn StructuralDecodeArrayTask> {
-    fn decode(self: Box<Self>) -> Result<ArrayRef> {
-        StructuralDecodeArrayTask::decode(*self).map(|decoded_array| decoded_array.array)
+    fn decode(self: Box<Self>) -> Result<(ArrayRef, u64)> {
+        StructuralDecodeArrayTask::decode(*self)
+            .map(|decoded_array| (decoded_array.array, decoded_array.data_size))
     }
 }
 
@@ -2530,25 +2640,19 @@ impl NextDecodeTask {
     // If the batch is very large this function will log a warning message
     // suggesting the user try a smaller batch size.
     #[instrument(name = "task_to_batch", level = "debug", skip_all)]
-    fn into_batch(self, emitted_batch_size_warning: Arc<Once>) -> Result<RecordBatch> {
-        let struct_arr = self.task.decode();
-        match struct_arr {
-            Ok(struct_arr) => {
-                let batch = RecordBatch::from(struct_arr.as_struct());
-                let size_bytes = batch.get_array_memory_size() as u64;
-                if size_bytes > BATCH_SIZE_BYTES_WARNING {
-                    emitted_batch_size_warning.call_once(|| {
-                        let size_mb = size_bytes / 1024 / 1024;
-                        debug!("Lance read in a single batch that contained more than {}MiB of data.  You may want to consider reducing the batch size.", size_mb);
-                    });
-                }
-                Ok(batch)
-            }
-            Err(e) => {
-                let e = Error::internal(format!("Error decoding batch: {}", e));
-                Err(e)
-            }
+    fn into_batch(self, emitted_batch_size_warning: Arc<Once>) -> Result<(RecordBatch, u64)> {
+        let (struct_arr, data_size) = self
+            .task
+            .decode()
+            .map_err(|e| Error::internal(format!("Error decoding batch: {}", e)))?;
+        let batch = RecordBatch::from(struct_arr.as_struct());
+        if data_size > BATCH_SIZE_BYTES_WARNING {
+            emitted_batch_size_warning.call_once(|| {
+                let size_mb = data_size / 1024 / 1024;
+                debug!("Lance read in a single batch that contained more than {}MiB of data.  You may want to consider reducing the batch size.", size_mb);
+            });
         }
+        Ok((batch, data_size))
     }
 }
 
@@ -2646,6 +2750,8 @@ pub struct LoadedPageShard {
 pub struct DecodedArray {
     pub array: ArrayRef,
     pub repdef: CompositeRepDefUnraveler,
+    /// The number of bytes of data in this array (excluding Arrow overhead).
+    pub data_size: u64,
 }
 
 pub trait StructuralDecodeArrayTask: std::fmt::Debug + Send {
@@ -2714,6 +2820,7 @@ pub async fn decode_batch(
         should_validate,
         spawn_structural_batch_decode_tasks,
         rx,
+        None,
     )?;
     decode_stream.next().await.unwrap().task.await
 }
@@ -2743,4 +2850,241 @@ mod tests {
         let ranges = DecodeBatchScheduler::indices_to_ranges(&indices);
         assert_eq!(ranges, vec![1..4, 5..8, 9..10]);
     }
+
+    #[test]
+    fn test_estimate_bytes_per_row() {
+        assert_eq!(estimate_bytes_per_row(&DataType::Int32), 4.0);
+        assert_eq!(estimate_bytes_per_row(&DataType::Int64), 8.0);
+        assert_eq!(estimate_bytes_per_row(&DataType::Float32), 4.0);
+        assert_eq!(estimate_bytes_per_row(&DataType::Boolean), 1.0 / 8.0);
+        assert_eq!(estimate_bytes_per_row(&DataType::Utf8), 64.0);
+        assert_eq!(estimate_bytes_per_row(&DataType::Binary), 64.0);
+        // Struct of 4 x Int32 = 16 bytes
+        let struct_type = DataType::Struct(Fields::from(vec![
+            ArrowField::new("a", DataType::Int32, false),
+            ArrowField::new("b", DataType::Int32, false),
+            ArrowField::new("c", DataType::Int32, false),
+            ArrowField::new("d", DataType::Int32, false),
+        ]));
+        assert_eq!(estimate_bytes_per_row(&struct_type), 16.0);
+    }
+
+    /// Helper: encode a batch, then decode it as a stream with optional
+    /// `batch_size_bytes`, collecting all output batches.
+    async fn decode_batches_with_byte_limit(
+        batch: &RecordBatch,
+        batch_size: u32,
+        batch_size_bytes: Option<u64>,
+    ) -> Vec<RecordBatch> {
+        use crate::encoder::{EncodingOptions, default_encoding_strategy, encode_batch};
+        use crate::version::LanceFileVersion;
+
+        let version = LanceFileVersion::V2_1;
+        let options = EncodingOptions {
+            version,
+            ..Default::default()
+        };
+        let strategy = default_encoding_strategy(version);
+        let schema = Schema::try_from(batch.schema().as_ref()).unwrap();
+        let encoded = encode_batch(batch, Arc::new(schema.clone()), strategy.as_ref(), &options)
+            .await
+            .unwrap();
+
+        let io_scheduler =
+            Arc::new(BufferScheduler::new(encoded.data.clone())) as Arc<dyn EncodingsIo>;
+        let cache = Arc::new(lance_core::cache::LanceCache::with_capacity(
+            128 * 1024 * 1024,
+        ));
+        let decoder_plugins = Arc::new(DecoderPlugins::default());
+
+        let mut decode_scheduler = DecodeBatchScheduler::try_new(
+            encoded.schema.as_ref(),
+            &encoded.top_level_columns,
+            &encoded.page_table,
+            &vec![],
+            encoded.num_rows,
+            decoder_plugins,
+            io_scheduler.clone(),
+            cache,
+            &FilterExpression::no_filter(),
+            &DecoderConfig::default(),
+        )
+        .await
+        .unwrap();
+
+        let (tx, rx) = unbounded_channel();
+        decode_scheduler.schedule_range(
+            0..encoded.num_rows,
+            &FilterExpression::no_filter(),
+            tx,
+            io_scheduler,
+        );
+
+        let mut decode_stream = create_decode_stream(
+            &encoded.schema,
+            encoded.num_rows,
+            batch_size,
+            /*is_structural=*/ true,
+            /*should_validate=*/ true,
+            /*spawn_structural_batch_decode_tasks=*/ true,
+            rx,
+            batch_size_bytes,
+        )
+        .unwrap();
+
+        let mut batches = Vec::new();
+        while let Some(task) = decode_stream.next().await {
+            batches.push(task.task.await.unwrap());
+        }
+        batches
+    }
+
+    #[tokio::test]
+    async fn test_byte_sized_batches_fixed_width() {
+        use arrow_array::Int32Array;
+
+        // 1000 rows x 4 Int32 columns = 16 bytes/row
+        let num_rows: i32 = 1000;
+        let arrays: Vec<Arc<dyn arrow_array::Array>> = (0..4)
+            .map(|col| {
+                Arc::new(Int32Array::from_iter_values(
+                    (0..num_rows).map(move |row| row * 10 + col),
+                )) as _
+            })
+            .collect();
+
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("a", DataType::Int32, false),
+            ArrowField::new("b", DataType::Int32, false),
+            ArrowField::new("c", DataType::Int32, false),
+            ArrowField::new("d", DataType::Int32, false),
+        ]));
+        let input_batch = RecordBatch::try_new(schema, arrays).unwrap();
+
+        // 16 bytes/row, batch_size_bytes=1600 => 100 rows/batch
+        let batches =
+            decode_batches_with_byte_limit(&input_batch, /*batch_size=*/ 1024, Some(1600)).await;
+
+        // Should produce 10 batches of 100 rows each
+        assert_eq!(batches.len(), 10);
+        for (i, batch) in batches.iter().enumerate() {
+            assert_eq!(
+                batch.num_rows(),
+                100,
+                "batch {i} should have 100 rows, got {}",
+                batch.num_rows()
+            );
+        }
+
+        // Verify roundtrip: concatenate and compare
+        let all_batches: Vec<&RecordBatch> = batches.iter().collect();
+        let concatenated =
+            arrow_select::concat::concat_batches(&batches[0].schema(), all_batches.iter().copied())
+                .unwrap();
+        assert_eq!(concatenated.num_rows(), num_rows as usize);
+        for col in 0..4 {
+            assert_eq!(
+                concatenated.column(col).as_ref(),
+                input_batch.column(col).as_ref(),
+                "column {col} roundtrip mismatch"
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn test_byte_sized_batches_none_unchanged() {
+        use arrow_array::Int32Array;
+
+        // Without batch_size_bytes, rows_per_batch controls batching
+        let num_rows: i32 = 1000;
+        let arrays: Vec<Arc<dyn arrow_array::Array>> = (0..2)
+            .map(|col| {
+                Arc::new(Int32Array::from_iter_values(
+                    (0..num_rows).map(move |row| row * 10 + col),
+                )) as _
+            })
+            .collect();
+
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("x", DataType::Int32, false),
+            ArrowField::new("y", DataType::Int32, false),
+        ]));
+        let input_batch = RecordBatch::try_new(schema, arrays).unwrap();
+
+        // batch_size=250, batch_size_bytes=None => 4 batches of 250 rows
+        let batches = decode_batches_with_byte_limit(&input_batch, /*batch_size=*/ 250, None).await;
+        assert_eq!(batches.len(), 4);
+        for (i, batch) in batches.iter().enumerate() {
+            assert_eq!(
+                batch.num_rows(),
+                250,
+                "batch {i} should have 250 rows, got {}",
+                batch.num_rows()
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn test_byte_sized_batches_feedback_convergence() {
+        use arrow_array::StringArray;
+
+        // Each row has a 100-byte string. Schema estimate = 64 bytes (default
+        // for Utf8), so the first batch will overshoot. The feedback loop
+        // should correct subsequent batches toward the target.
+        let num_rows = 500;
+        let value: String = "x".repeat(100);
+        let arrays: Vec<Arc<dyn arrow_array::Array>> = vec![Arc::new(StringArray::from(
+            (0..num_rows).map(|_| value.as_str()).collect::<Vec<_>>(),
+        ))];
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "s",
+            DataType::Utf8,
+            false,
+        )]));
+        let input_batch = RecordBatch::try_new(schema, arrays).unwrap();
+
+        // Target 5000 bytes/batch. At 100 bytes/row the ideal is 50 rows/batch.
+        // Schema estimate is 64 bytes/row → first batch ~78 rows (overshoot).
+        // After feedback kicks in, batches should converge to ~50 rows.
+        let target_bytes: u64 = 5000;
+        let batches = decode_batches_with_byte_limit(
+            &input_batch,
+            /*batch_size=*/ 1024,
+            Some(target_bytes),
+        )
+        .await;
+
+        // Verify all data round-trips correctly
+        let all_batches: Vec<&RecordBatch> = batches.iter().collect();
+        let concatenated =
+            arrow_select::concat::concat_batches(&batches[0].schema(), all_batches.iter().copied())
+                .unwrap();
+        assert_eq!(concatenated.num_rows(), num_rows as usize);
+        assert_eq!(
+            concatenated.column(0).as_ref(),
+            input_batch.column(0).as_ref()
+        );
+
+        // After the first batch, subsequent batches should be closer to the
+        // target. The ideal is 50 rows/batch.
+        assert!(
+            batches.len() >= 2,
+            "need at least 2 batches to test convergence"
+        );
+        // The first batch uses the schema estimate (64 bytes/row) →
+        // ~78 rows. After feedback the rows should settle near 50.
+        if batches.len() >= 3 {
+            let second_batch_rows = batches[1].num_rows();
+            let third_batch_rows = batches[2].num_rows();
+            // Both should be within 20% of the ideal (50 rows)
+            assert!(
+                (40..=60).contains(&second_batch_rows),
+                "second batch should be near 50 rows, got {second_batch_rows}"
+            );
+            assert!(
+                (40..=60).contains(&third_batch_rows),
+                "third batch should be near 50 rows, got {third_batch_rows}"
+            );
+        }
+    }
 }
diff --git a/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs b/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs
index ed62e633316..9e8e3e109ea 100644
--- a/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs
+++ b/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs
@@ -247,7 +247,11 @@ impl StructuralFixedSizeListDecodeTask {
 
 impl StructuralDecodeArrayTask for StructuralFixedSizeListDecodeTask {
     fn decode(self: Box<Self>) -> Result<DecodedArray> {
-        let DecodedArray { array, mut repdef } = self.child_task.decode()?;
+        let DecodedArray {
+            array,
+            mut repdef,
+            data_size,
+        } = self.child_task.decode()?;
         match &self.data_type {
             DataType::FixedSizeList(child_field, dimension) => {
                 let num_rows = self.num_rows as usize;
@@ -261,6 +265,7 @@ impl StructuralDecodeArrayTask for StructuralFixedSizeListDecodeTask {
                 Ok(DecodedArray {
                     array: Arc::new(fsl_array),
                     repdef,
+                    data_size,
                 })
             }
             _ => Err(Error::internal(
@@ -617,6 +622,71 @@ mod tests {
         ])
     }
 
+    fn make_fsl_of_list() -> DataType {
+        DataType::FixedSizeList(
+            Arc::new(Field::new(
+                "item",
+                DataType::List(Arc::new(Field::new("item", DataType::Int32, true))),
+                true,
+            )),
+            2,
+        )
+    }
+
+    fn make_fsl_of_large_list() -> DataType {
+        DataType::FixedSizeList(
+            Arc::new(Field::new(
+                "item",
+                DataType::LargeList(Arc::new(Field::new("item", DataType::Int32, true))),
+                true,
+            )),
+            2,
+        )
+    }
+
+    fn make_fsl_of_map() -> DataType {
+        DataType::FixedSizeList(
+            Arc::new(Field::new(
+                "item",
+                DataType::Map(
+                    Arc::new(Field::new(
+                        "entries",
+                        DataType::Struct(Fields::from(vec![
+                            Field::new("key", DataType::Utf8, false),
+                            Field::new("value", DataType::Int32, true),
+                        ])),
+                        false,
+                    )),
+                    false,
+                ),
+                true,
+            )),
+            2,
+        )
+    }
+
+    fn make_fsl_of_nested_fsl_struct() -> DataType {
+        DataType::FixedSizeList(
+            Arc::new(Field::new(
+                "item",
+                DataType::FixedSizeList(
+                    Arc::new(Field::new(
+                        "item",
+                        DataType::Struct(Fields::from(vec![Field::new(
+                            "x",
+                            DataType::Int32,
+                            true,
+                        )])),
+                        true,
+                    )),
+                    4,
+                ),
+                true,
+            )),
+            2,
+        )
+    }
+
     #[rstest]
     #[case::simple(simple_struct_fields(), 2, LanceFileVersion::V2_2)]
     #[case::nested_struct(nested_struct_fields(), 2, LanceFileVersion::V2_2)]
@@ -644,43 +714,15 @@ mod tests {
         check_specific_random(field, test_cases).await;
     }
 
-    // FSL<List> and FSL<Map> are not yet supported (blocked by repdef)
-    #[test]
-    #[should_panic(expected = "Unsupported logical type: list")]
-    fn test_fsl_list_rejected() {
-        let inner = Field::new(
-            "item",
-            DataType::List(Arc::new(Field::new("item", DataType::Int32, true))),
-            true,
-        );
-        let data_type = DataType::FixedSizeList(Arc::new(inner), 2);
-        let arrow_field = Field::new("test", data_type, true);
-        let lance_field = lance_core::datatypes::Field::try_from(&arrow_field).unwrap();
-        let _ = lance_field.data_type();
-    }
-
-    #[test]
-    #[should_panic(expected = "Unsupported logical type: map")]
-    fn test_fsl_map_rejected() {
-        let inner = Field::new(
-            "item",
-            DataType::Map(
-                Arc::new(Field::new(
-                    "entries",
-                    DataType::Struct(Fields::from(vec![
-                        Field::new("key", DataType::Utf8, false),
-                        Field::new("value", DataType::Int32, true),
-                    ])),
-                    false,
-                )),
-                false,
-            ),
-            true,
-        );
-        let data_type = DataType::FixedSizeList(Arc::new(inner), 2);
+    #[rstest]
+    #[case::list(make_fsl_of_list())]
+    #[case::large_list(make_fsl_of_large_list())]
+    #[case::map(make_fsl_of_map())]
+    #[case::nested_fsl_struct(make_fsl_of_nested_fsl_struct())]
+    fn test_unsupported_fsl_child_types_return_error(#[case] data_type: DataType) {
         let arrow_field = Field::new("test", data_type, true);
-        let lance_field = lance_core::datatypes::Field::try_from(&arrow_field).unwrap();
-        let _ = lance_field.data_type();
+        let err = lance_core::datatypes::Field::try_from(&arrow_field).unwrap_err();
+        assert!(err.to_string().contains("Unsupported data type"));
     }
 
     #[test]
diff --git a/rust/lance-encoding/src/encodings/logical/list.rs b/rust/lance-encoding/src/encodings/logical/list.rs
index 874cd83c204..01422ae5dc0 100644
--- a/rust/lance-encoding/src/encodings/logical/list.rs
+++ b/rust/lance-encoding/src/encodings/logical/list.rs
@@ -195,7 +195,11 @@ impl StructuralListDecodeTask {
 
 impl StructuralDecodeArrayTask for StructuralListDecodeTask {
     fn decode(self: Box<Self>) -> Result<DecodedArray> {
-        let DecodedArray { array, mut repdef } = self.child_task.decode()?;
+        let DecodedArray {
+            array,
+            mut repdef,
+            data_size,
+        } = self.child_task.decode()?;
         match &self.data_type {
             DataType::List(child_field) => {
                 let (offsets, validity) = repdef.unravel_offsets::<i32>()?;
@@ -209,6 +213,7 @@ impl StructuralDecodeArrayTask for StructuralListDecodeTask {
                 Ok(DecodedArray {
                     array: Arc::new(list_array),
                     repdef,
+                    data_size,
                 })
             }
             DataType::LargeList(child_field) => {
@@ -218,6 +223,7 @@ impl StructuralDecodeArrayTask for StructuralListDecodeTask {
                 Ok(DecodedArray {
                     array: Arc::new(list_array),
                     repdef,
+                    data_size,
                 })
             }
             _ => panic!("List decoder did not have a list field"),
@@ -889,4 +895,55 @@ mod tests {
         // Actual: panic at primitive.rs:1362 - assertion failed: rows_avail > 0
         check_round_trip_encoding_of_data(vec![list_array], &test_cases, HashMap::new()).await;
     }
+
+    #[rstest]
+    #[test_log::test(tokio::test)]
+    async fn test_sparse_large_string_list(
+        #[values(STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_FULLZIP)]
+        structural_encoding: &str,
+    ) {
+        // 2.5 million rows, mostly empty lists. ~100 lists have 10 short strings each.
+        let num_rows = 2_500_000u32;
+        let num_non_empty = 100u32;
+        let strings_per_list = 10;
+
+        let items_builder = StringBuilder::new();
+        let mut list_builder = ListBuilder::new(items_builder);
+
+        // Spread non-empty lists evenly across the range
+        let step = num_rows / num_non_empty;
+        let mut next_non_empty = step / 2;
+
+        for i in 0..num_rows {
+            if i == next_non_empty {
+                let vals: Vec<Option<&str>> = (0..strings_per_list)
+                    .map(|j| match j % 4 {
+                        0 => Some("a"),
+                        1 => Some("bb"),
+                        2 => Some("ccc"),
+                        _ => Some("d"),
+                    })
+                    .collect();
+                list_builder.append_value(vals);
+                next_non_empty = next_non_empty.saturating_add(step);
+            } else {
+                list_builder.append_value([] as [Option<&str>; 0]);
+            }
+        }
+        let list_array = list_builder.finish();
+
+        let mut field_metadata = HashMap::new();
+        field_metadata.insert(
+            STRUCTURAL_ENCODING_META_KEY.to_string(),
+            structural_encoding.into(),
+        );
+
+        let test_cases = TestCases::default()
+            .with_range(0..1000)
+            .with_range(0..num_rows as u64)
+            .with_indices(vec![0, (step / 2) as u64, num_rows as u64 - 1])
+            .with_max_file_version(LanceFileVersion::V2_2);
+        check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, field_metadata)
+            .await;
+    }
 }
diff --git a/rust/lance-encoding/src/encodings/logical/map.rs b/rust/lance-encoding/src/encodings/logical/map.rs
index 7ac53946063..b5172d8c189 100644
--- a/rust/lance-encoding/src/encodings/logical/map.rs
+++ b/rust/lance-encoding/src/encodings/logical/map.rs
@@ -187,7 +187,11 @@ impl StructuralMapDecodeTask {
 
 impl StructuralDecodeArrayTask for StructuralMapDecodeTask {
     fn decode(self: Box<Self>) -> Result<DecodedArray> {
-        let DecodedArray { array, mut repdef } = self.child_task.decode()?;
+        let DecodedArray {
+            array,
+            mut repdef,
+            data_size,
+        } = self.child_task.decode()?;
 
         // Decode the offsets from RepDef
         let (offsets, validity) = repdef.unravel_offsets::<i32>()?;
@@ -224,6 +228,7 @@ impl StructuralDecodeArrayTask for StructuralMapDecodeTask {
         Ok(DecodedArray {
             array: Arc::new(map_array),
             repdef,
+            data_size,
         })
     }
 }
diff --git a/rust/lance-encoding/src/encodings/logical/primitive.rs b/rust/lance-encoding/src/encodings/logical/primitive.rs
index b5de0912a32..9f6be591d13 100644
--- a/rust/lance-encoding/src/encodings/logical/primitive.rs
+++ b/rust/lance-encoding/src/encodings/logical/primitive.rs
@@ -2165,15 +2165,17 @@ impl FullZipScheduler {
     }
 
     fn create_page_load_task(
-        read_source: FullZipReadSource,
-        byte_ranges: Vec<Range<u64>>,
-        priority: u64,
+        io_future: BoxFuture<'static, Result<Vec<Bytes>>>,
         num_rows: u64,
         details: Arc<FullZipDecodeDetails>,
         bits_per_offset: u8,
     ) -> PageLoadTask {
         let load_task = async move {
-            let data = read_source.fetch(&byte_ranges, priority).await?;
+            let buffers = io_future.await?;
+            let data = buffers
+                .into_iter()
+                .map(|bytes| LanceBuffer::from_bytes(bytes, 1))
+                .collect::<VecDeque<_>>();
             Self::create_decoder(details, data, num_rows, bits_per_offset)
         }
         .boxed();
@@ -2333,14 +2335,9 @@ impl FullZipScheduler {
                 rep_index.bytes_per_value,
                 data_buf_position,
             );
-            let page_load_task = Self::create_page_load_task(
-                FullZipReadSource::Remote(io.clone()),
-                byte_ranges,
-                priority,
-                num_rows,
-                details,
-                bits_per_offset,
-            );
+            let io_future = io.submit_request(byte_ranges, priority);
+            let page_load_task =
+                Self::create_page_load_task(io_future, num_rows, details, bits_per_offset);
             return Ok(vec![page_load_task]);
         }
 
@@ -2403,10 +2400,9 @@ impl FullZipScheduler {
             })
             .collect::<Vec<_>>();
 
+        let io_future = io.submit_request(byte_ranges, self.priority);
         let page_load_task = Self::create_page_load_task(
-            FullZipReadSource::Remote(io.clone()),
-            byte_ranges,
-            self.priority,
+            io_future,
             num_rows,
             self.details.clone(),
             self.bits_per_offset,
@@ -3416,6 +3412,10 @@ impl CacheKey for FieldDataCacheKey {
     fn key(&self) -> std::borrow::Cow<'_, str> {
         self.column_index.to_string().into()
     }
+
+    fn type_name() -> &'static str {
+        "FieldData"
+    }
 }
 
 impl StructuralFieldScheduler for StructuralPrimitiveFieldScheduler {
@@ -3505,8 +3505,10 @@ impl StructuralDecodeArrayTask for StructuralCompositeDecodeArrayTask {
     fn decode(self: Box<Self>) -> Result<DecodedArray> {
         let mut arrays = Vec::with_capacity(self.tasks.len());
         let mut unravelers = Vec::with_capacity(self.tasks.len());
+        let mut data_size = 0u64;
         for task in self.tasks {
             let decoded = task.decode()?;
+            data_size += decoded.data.data_size();
             unravelers.push(decoded.repdef);
 
             let array = make_array(
@@ -3523,7 +3525,11 @@ impl StructuralDecodeArrayTask for StructuralCompositeDecodeArrayTask {
 
         let array = Self::restore_validity(array, &mut repdef);
 
-        Ok(DecodedArray { array, repdef })
+        Ok(DecodedArray {
+            array,
+            repdef,
+            data_size,
+        })
     }
 }
 
@@ -3737,6 +3743,66 @@ impl PrimitiveStructuralEncoder {
         Self::is_narrow(data_block)
     }
 
+    /// Checks if the rep/def levels are too sparse for miniblock encoding.
+    ///
+    /// Miniblock chunks are limited to ~32KiB total. Data can use up to ~16KiB,
+    /// leaving ~16KiB for both rep and def buffers combined. Each chunk has at most
+    /// MAX_MINIBLOCK_VALUES (4096) data values, but when data has many empty/null
+    /// lists, the number of rep/def levels can far exceed the number of data values
+    /// (each empty list adds a level entry with no corresponding data value).
+    ///
+    /// We estimate the compressed bits per level by computing the max value in each
+    /// buffer and taking ceil(log2(max_val + 1)) — the minimum bits needed to
+    /// bitpack each level. We then calculate the maximum number of levels that fit
+    /// in 16KiB and compare against the actual levels-to-values ratio.
+    fn repdef_too_sparse_for_miniblock(
+        repdef: &crate::repdef::SerializedRepDefs,
+        num_values: u64,
+    ) -> bool {
+        if num_values == 0 {
+            return false;
+        }
+        let num_levels = repdef
+            .repetition_levels
+            .as_ref()
+            .map(|r| r.len() as u64)
+            .max(repdef.definition_levels.as_ref().map(|d| d.len() as u64))
+            .unwrap_or(0);
+        if num_levels == 0 {
+            return false;
+        }
+
+        // Compute bits needed per level for each buffer (ceil of log2(max+1))
+        let bits_per_rep = repdef
+            .repetition_levels
+            .as_ref()
+            .and_then(|r| r.iter().max().copied())
+            .map(|max_val| u16::BITS - max_val.leading_zeros())
+            .unwrap_or(0) as u64;
+        let bits_per_def = repdef
+            .definition_levels
+            .as_ref()
+            .and_then(|d| d.iter().max().copied())
+            .map(|max_val| u16::BITS - max_val.leading_zeros())
+            .unwrap_or(0) as u64;
+
+        let bits_per_level = bits_per_rep + bits_per_def;
+        if bits_per_level == 0 {
+            return false;
+        }
+
+        // 16KiB budget for rep+def combined (half the ~32KiB chunk limit)
+        const REPDEF_BUDGET_BITS: u64 = 16 * 1024 * 8;
+        let max_levels_per_chunk = REPDEF_BUDGET_BITS / bits_per_level;
+
+        // A chunk has at most MAX_MINIBLOCK_VALUES data values. The levels-to-values
+        // ratio tells us how many levels a chunk of that size would need.
+        let levels_per_chunk =
+            (num_levels as f64 / num_values as f64) * *miniblock::MAX_MINIBLOCK_VALUES as f64;
+
+        levels_per_chunk > max_levels_per_chunk as f64
+    }
+
     fn prefers_fullzip(encoding_metadata: &HashMap<String, String>) -> bool {
         // Fullzip is the backup option so the only reason we wouldn't use it is if the
         // user specifically requested not to use it (in which case we're probably going
@@ -3798,7 +3864,7 @@ impl PrimitiveStructuralEncoder {
         rep: Option<Vec<CompressedLevelsChunk>>,
         def: Option<Vec<CompressedLevelsChunk>>,
         support_large_chunk: bool,
-    ) -> SerializedMiniBlockPage {
+    ) -> Result<SerializedMiniBlockPage> {
         let bytes_rep = rep
             .as_ref()
             .map(|rep| rep.iter().map(|r| r.data.len()).sum::<usize>())
@@ -3842,11 +3908,21 @@ impl PrimitiveStructuralEncoder {
 
             // Write the buffer lengths
             if let Some(rep) = rep.as_ref() {
-                let bytes_rep = u16::try_from(rep.data.len()).unwrap();
+                let bytes_rep = u16::try_from(rep.data.len()).map_err(|_| {
+                    Error::internal(format!(
+                        "Repetition buffer size ({} bytes) too large",
+                        rep.data.len()
+                    ))
+                })?;
                 data_buffer.extend_from_slice(&bytes_rep.to_le_bytes());
             }
             if let Some(def) = def.as_ref() {
-                let bytes_def = u16::try_from(def.data.len()).unwrap();
+                let bytes_def = u16::try_from(def.data.len()).map_err(|_| {
+                    Error::internal(format!(
+                        "Definition buffer size ({} bytes) too large",
+                        def.data.len()
+                    ))
+                })?;
                 data_buffer.extend_from_slice(&bytes_def.to_le_bytes());
             }
 
@@ -3916,11 +3992,11 @@ impl PrimitiveStructuralEncoder {
         let data_buffer = LanceBuffer::from(data_buffer);
         let metadata_buffer = LanceBuffer::from(meta_buffer);
 
-        SerializedMiniBlockPage {
+        Ok(SerializedMiniBlockPage {
             num_buffers: miniblocks.data.len() as u64,
             data: data_buffer,
             metadata: metadata_buffer,
-        }
+        })
     }
 
     /// Compresses a buffer of levels into chunks
@@ -4448,7 +4524,7 @@ impl PrimitiveStructuralEncoder {
             .map(|cd| std::mem::take(&mut cd.data));
 
         let serialized =
-            Self::serialize_miniblocks(compressed_data, rep_data, def_data, support_large_chunk);
+            Self::serialize_miniblocks(compressed_data, rep_data, def_data, support_large_chunk)?;
 
         // Metadata, Data, Dictionary, (maybe) Repetition Index
         let mut data = Vec::with_capacity(4);
@@ -5112,41 +5188,61 @@ impl PrimitiveStructuralEncoder {
                 );
             }
 
-            if let DataBlock::Dictionary(dict) = data_block {
-                log::debug!("Encoding column {} with {} items using dictionary encoding (already dictionary encoded)", column_idx, num_values);
-                let (mut indices_data_block, dictionary_data_block) = dict.into_parts();
-                // TODO: https://github.com/lancedb/lance/issues/4809
-                // If we compute stats on dictionary_data_block => panic.
-                // If we don't compute stats on indices_data_block => panic.
-                // This is messy.  Don't make me call compute_stat ever.
-                indices_data_block.compute_stat();
-                Self::encode_miniblock(
-                    column_idx,
-                    &field,
-                    compression_strategy.as_ref(),
-                    indices_data_block,
-                    repdef,
-                    row_number,
-                    Some(dictionary_data_block),
-                    num_rows,
-                    support_large_chunk,
-                )
+            // If the rep/def levels are too sparse for miniblock (e.g. many empty
+            // lists with very few values), fall back to fullzip to avoid exceeding
+            // the u16 per-chunk rep/def buffer size limit.
+            let too_sparse = Self::repdef_too_sparse_for_miniblock(&repdef, num_values);
+
+            if !too_sparse {
+                if let DataBlock::Dictionary(dict) = data_block {
+                    log::debug!("Encoding column {} with {} items using dictionary encoding (already dictionary encoded)", column_idx, num_values);
+                    let (mut indices_data_block, dictionary_data_block) = dict.into_parts();
+                    // TODO: https://github.com/lancedb/lance/issues/4809
+                    // If we compute stats on dictionary_data_block => panic.
+                    // If we don't compute stats on indices_data_block => panic.
+                    // This is messy.  Don't make me call compute_stat ever.
+                    indices_data_block.compute_stat();
+                    return Self::encode_miniblock(
+                        column_idx,
+                        &field,
+                        compression_strategy.as_ref(),
+                        indices_data_block,
+                        repdef,
+                        row_number,
+                        Some(dictionary_data_block),
+                        num_rows,
+                        support_large_chunk,
+                    );
+                }
             } else {
+                log::debug!(
+                    "Encoding column {} with {} items using full-zip layout \
+                     (rep/def too sparse for mini-block)",
+                    column_idx,
+                    num_values
+                );
+            }
+
+            {
                 // Try dictionary encoding first if applicable. If encoding aborts, fall back to the
                 // preferred structural encoding.
-                let dict_result = Self::should_dictionary_encode(&data_block, &field, version)
-                    .and_then(|budget| {
-                        log::debug!(
-                            "Encoding column {} with {} items using dictionary encoding (mini-block layout)",
-                            column_idx,
-                            num_values
-                        );
-                        dict::dictionary_encode(
-                            &data_block,
-                            budget.max_dict_entries,
-                            budget.max_encoded_size,
-                        )
-                    });
+                let dict_result = if too_sparse {
+                    None
+                } else {
+                    Self::should_dictionary_encode(&data_block, &field, version)
+                        .and_then(|budget| {
+                            log::debug!(
+                                "Encoding column {} with {} items using dictionary encoding (mini-block layout)",
+                                column_idx,
+                                num_values
+                            );
+                            dict::dictionary_encode(
+                                &data_block,
+                                budget.max_dict_entries,
+                                budget.max_encoded_size,
+                            )
+                        })
+                };
 
                 if let Some((indices_data_block, dictionary_data_block)) = dict_result {
                     Self::encode_miniblock(
@@ -5160,7 +5256,7 @@ impl PrimitiveStructuralEncoder {
                         num_rows,
                         support_large_chunk,
                     )
-                } else if Self::prefers_miniblock(&data_block, encoding_metadata.as_ref()) {
+                } else if !too_sparse && Self::prefers_miniblock(&data_block, encoding_metadata.as_ref()) {
                     log::debug!(
                         "Encoding column {} with {} items using mini-block layout",
                         column_idx,
@@ -5177,7 +5273,7 @@ impl PrimitiveStructuralEncoder {
                         num_rows,
                         support_large_chunk,
                     )
-                } else if Self::prefers_fullzip(encoding_metadata.as_ref()) {
+                } else if too_sparse || Self::prefers_fullzip(encoding_metadata.as_ref()) {
                     log::debug!(
                         "Encoding column {} with {} items using full-zip layout",
                         column_idx,
diff --git a/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs b/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs
index 6da985e9ec0..e3a4a4f4895 100644
--- a/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs
+++ b/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs
@@ -17,7 +17,19 @@ use crate::{buffer::LanceBuffer, data::DataBlock, format::pb21::CompressiveEncod
 use lance_core::Result;
 
 pub const MAX_MINIBLOCK_BYTES: u64 = 8 * 1024 - 6;
-pub const MAX_MINIBLOCK_VALUES: u64 = 4096;
+
+const DEFAULT_MAX_MINIBLOCK_VALUES: u64 = 4096;
+
+fn parse_max_miniblock_values() -> u64 {
+    let val = std::env::var("LANCE_MINIBLOCK_MAX_VALUES")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or(DEFAULT_MAX_MINIBLOCK_VALUES);
+    val.clamp(1, DEFAULT_MAX_MINIBLOCK_VALUES)
+}
+
+pub static MAX_MINIBLOCK_VALUES: std::sync::LazyLock<u64> =
+    std::sync::LazyLock::new(parse_max_miniblock_values);
 
 /// Page data that has been compressed into a series of chunks put into
 /// a single buffer.
@@ -36,10 +48,15 @@ pub struct MiniBlockCompressed {
 /// Mini-block chunks are designed to be small (just a few disk sectors)
 /// and contain a power-of-two number of values (except for the last chunk)
 ///
-/// To enforce this we limit a chunk to 4Ki values and slightly less than
+/// By default we limit a chunk to 4Ki values and slightly less than
 /// 8KiB of compressed data.  This means that even in the extreme case
 /// where we have 4 bytes of rep/def then we will have at most 24KiB of
 /// data (values, repetition, and definition) per mini-block.
+///
+/// The maximum number of values per chunk can be configured via the
+/// `LANCE_MINIBLOCK_MAX_VALUES` environment variable.  This is only
+/// useful in extremely bandwidth-limited environments; the default is
+/// appropriate for local disks and same-region cloud object storage.
 #[derive(Debug)]
 pub struct MiniBlockChunk {
     // The size in bytes of each buffer in the chunk.
@@ -54,7 +71,7 @@ pub struct MiniBlockChunk {
     // For example, 1 would mean there are 2 values in the chunk and 12 would mean there
     // are 4Ki values in the chunk.
     //
-    // This must be <= 12 (i.e. <= 4096 values)
+    // This must be <= log2(MAX_MINIBLOCK_VALUES) (i.e. <= 12 at the default of 4096)
     pub log_num_values: u8,
 }
 
@@ -87,3 +104,49 @@ pub trait MiniBlockCompressor: std::fmt::Debug + Send + Sync {
     /// used at decode time to read the data.
     fn compress(&self, page: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)>;
 }
+
+#[cfg(test)]
+mod tests {
+    use serial_test::serial;
+
+    use super::*;
+
+    #[test]
+    #[serial]
+    fn test_parse_default() {
+        unsafe { std::env::remove_var("LANCE_MINIBLOCK_MAX_VALUES") };
+        assert_eq!(parse_max_miniblock_values(), 4096);
+    }
+
+    #[test]
+    #[serial]
+    fn test_parse_custom_value() {
+        unsafe { std::env::set_var("LANCE_MINIBLOCK_MAX_VALUES", "256") };
+        assert_eq!(parse_max_miniblock_values(), 256);
+        unsafe { std::env::remove_var("LANCE_MINIBLOCK_MAX_VALUES") };
+    }
+
+    #[test]
+    #[serial]
+    fn test_parse_clamps_zero_to_one() {
+        unsafe { std::env::set_var("LANCE_MINIBLOCK_MAX_VALUES", "0") };
+        assert_eq!(parse_max_miniblock_values(), 1);
+        unsafe { std::env::remove_var("LANCE_MINIBLOCK_MAX_VALUES") };
+    }
+
+    #[test]
+    #[serial]
+    fn test_parse_clamps_above_max() {
+        unsafe { std::env::set_var("LANCE_MINIBLOCK_MAX_VALUES", "99999") };
+        assert_eq!(parse_max_miniblock_values(), DEFAULT_MAX_MINIBLOCK_VALUES);
+        unsafe { std::env::remove_var("LANCE_MINIBLOCK_MAX_VALUES") };
+    }
+
+    #[test]
+    #[serial]
+    fn test_parse_invalid_falls_back_to_default() {
+        unsafe { std::env::set_var("LANCE_MINIBLOCK_MAX_VALUES", "not_a_number") };
+        assert_eq!(parse_max_miniblock_values(), DEFAULT_MAX_MINIBLOCK_VALUES);
+        unsafe { std::env::remove_var("LANCE_MINIBLOCK_MAX_VALUES") };
+    }
+}
diff --git a/rust/lance-encoding/src/encodings/logical/struct.rs b/rust/lance-encoding/src/encodings/logical/struct.rs
index 2682090b317..eee3404abdf 100644
--- a/rust/lance-encoding/src/encodings/logical/struct.rs
+++ b/rust/lance-encoding/src/encodings/logical/struct.rs
@@ -358,6 +358,7 @@ impl StructuralDecodeArrayTask for RepDefStructDecodeTask {
             return Ok(DecodedArray {
                 array: Arc::new(StructArray::new_empty_fields(self.num_rows as usize, None)),
                 repdef: CompositeRepDefUnraveler::new(vec![]),
+                data_size: 0,
             });
         }
 
@@ -367,16 +368,19 @@ impl StructuralDecodeArrayTask for RepDefStructDecodeTask {
             .map(|task| task.decode())
             .collect::<Result<Vec<_>>>()?;
         let mut children = Vec::with_capacity(arrays.len());
+        let mut data_size = 0u64;
         let mut arrays_iter = arrays.into_iter();
         let first_array = arrays_iter.next().unwrap();
         let length = first_array.array.len();
 
         // The repdef should be identical across all children at this point
         let mut repdef = first_array.repdef;
+        data_size += first_array.data_size;
         children.push(first_array.array);
 
         for array in arrays_iter {
             debug_assert_eq!(length, array.array.len());
+            data_size += array.data_size;
             children.push(array.array);
         }
 
@@ -391,6 +395,7 @@ impl StructuralDecodeArrayTask for RepDefStructDecodeTask {
         Ok(DecodedArray {
             array: Arc::new(array),
             repdef,
+            data_size,
         })
     }
 }
@@ -493,7 +498,6 @@ pub struct StructFieldEncoder {
 }
 
 impl StructFieldEncoder {
-    #[allow(dead_code)]
     pub fn new(children: Vec<Box<dyn FieldEncoder>>, column_index: u32) -> Self {
         Self {
             children,
diff --git a/rust/lance-encoding/src/encodings/physical/binary.rs b/rust/lance-encoding/src/encodings/physical/binary.rs
index fba56420bae..d02cf2da693 100644
--- a/rust/lance-encoding/src/encodings/physical/binary.rs
+++ b/rust/lance-encoding/src/encodings/physical/binary.rs
@@ -186,7 +186,7 @@ fn search_next_offset_idx<N: OffsetSizeTrait>(
         let new_size =
             existing_bytes + N::from_usize((new_num_values + 1) * N::get_byte_width()).unwrap();
         if new_size.to_i64().unwrap() <= minichunk_size {
-            if new_num_values * 2 > MAX_MINIBLOCK_VALUES as usize {
+            if new_num_values * 2 > *MAX_MINIBLOCK_VALUES as usize {
                 // hit the max number of values limit
                 break;
             }
@@ -525,7 +525,7 @@ impl BlockDecompressor for BinaryBlockDecompressor {
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
     use arrow_array::{
         ArrayRef, StringArray,
         builder::{LargeStringBuilder, StringBuilder},
diff --git a/rust/lance-encoding/src/encodings/physical/rle.rs b/rust/lance-encoding/src/encodings/physical/rle.rs
index 31eab63ed5c..88e27bf954f 100644
--- a/rust/lance-encoding/src/encodings/physical/rle.rs
+++ b/rust/lance-encoding/src/encodings/physical/rle.rs
@@ -205,7 +205,7 @@ impl RleEncoder {
         let type_size = std::mem::size_of::<T>();
 
         let chunk_start = offset * type_size;
-        let max_by_count = MAX_MINIBLOCK_VALUES as usize;
+        let max_by_count = *MAX_MINIBLOCK_VALUES as usize;
         let max_values = values_remaining.min(max_by_count);
         let chunk_end = chunk_start + max_values * type_size;
 
@@ -239,7 +239,7 @@ impl RleEncoder {
             2 => 7, // 128
             _ => 6, // 64
         };
-        let max_checkpoint_log2 = (values_remaining.min(MAX_MINIBLOCK_VALUES as usize))
+        let max_checkpoint_log2 = (values_remaining.min(*MAX_MINIBLOCK_VALUES as usize))
             .next_power_of_two()
             .ilog2();
         let mut checkpoint_log2 = min_checkpoint_log2;
@@ -718,7 +718,7 @@ mod tests {
                     assert!(chunk.log_num_values > 0);
                     let chunk_values = 1u64 << chunk.log_num_values;
                     assert!(chunk_values.is_power_of_two());
-                    assert!(chunk_values <= MAX_MINIBLOCK_VALUES);
+                    assert!(chunk_values <= *MAX_MINIBLOCK_VALUES);
                 } else {
                     assert_eq!(chunk.log_num_values, 0);
                 }
diff --git a/rust/lance-encoding/src/encodings/physical/value.rs b/rust/lance-encoding/src/encodings/physical/value.rs
index 48f1b01886c..c49bbd3efbd 100644
--- a/rust/lance-encoding/src/encodings/physical/value.rs
+++ b/rust/lance-encoding/src/encodings/physical/value.rs
@@ -38,7 +38,7 @@ impl ValueEncoder {
         // If the type is so wide that we can't even fit 2 values we shouldn't be here
         assert!(size_bytes < MAX_MINIBLOCK_BYTES);
 
-        while 2 * size_bytes < MAX_MINIBLOCK_BYTES && 2 * num_vals <= MAX_MINIBLOCK_VALUES {
+        while 2 * size_bytes < MAX_MINIBLOCK_BYTES && 2 * num_vals <= *MAX_MINIBLOCK_VALUES {
             log_num_vals += 1;
             size_bytes *= 2;
             num_vals *= 2;
@@ -749,7 +749,7 @@ impl PerValueCompressor for ValueEncoder {
 
 // public tests module because we share the PRIMITIVE_TYPES constant with fixed_size_list
 #[cfg(test)]
-pub(crate) mod tests {
+mod tests {
     use std::{
         collections::HashMap,
         sync::{Arc, LazyLock},
diff --git a/rust/lance-encoding/src/previous/encoder.rs b/rust/lance-encoding/src/previous/encoder.rs
index 9188c8ca4be..9c314ae97e9 100644
--- a/rust/lance-encoding/src/previous/encoder.rs
+++ b/rust/lance-encoding/src/previous/encoder.rs
@@ -639,7 +639,7 @@ impl ArrayEncodingStrategy for CoreArrayEncodingStrategy {
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
     use crate::constants::{COMPRESSION_LEVEL_META_KEY, COMPRESSION_META_KEY};
     use crate::previous::encoder::{
         ArrayEncodingStrategy, CoreArrayEncodingStrategy, check_dict_encoding,
diff --git a/rust/lance-encoding/src/previous/encodings/logical/binary.rs b/rust/lance-encoding/src/previous/encodings/logical/binary.rs
index 29b04c68a25..00715f64511 100644
--- a/rust/lance-encoding/src/previous/encodings/logical/binary.rs
+++ b/rust/lance-encoding/src/previous/encodings/logical/binary.rs
@@ -169,17 +169,18 @@ impl BinaryArrayDecoder {
 }
 
 impl DecodeArrayTask for BinaryArrayDecoder {
-    fn decode(self: Box<Self>) -> Result<ArrayRef> {
+    fn decode(self: Box<Self>) -> Result<(ArrayRef, u64)> {
         let data_type = self.data_type;
-        let arr = self.inner.decode()?;
-        match data_type {
-            DataType::Binary => Ok(Self::from_list_array::<BinaryType>(arr.as_list::<i32>())),
-            DataType::LargeBinary => Ok(Self::from_list_array::<LargeBinaryType>(
-                arr.as_list::<i64>(),
-            )),
-            DataType::Utf8 => Ok(Self::from_list_array::<Utf8Type>(arr.as_list::<i32>())),
-            DataType::LargeUtf8 => Ok(Self::from_list_array::<LargeUtf8Type>(arr.as_list::<i64>())),
+        let (arr, _) = self.inner.decode()?;
+        let result = match data_type {
+            DataType::Binary => Self::from_list_array::<BinaryType>(arr.as_list::<i32>()),
+            DataType::LargeBinary => Self::from_list_array::<LargeBinaryType>(arr.as_list::<i64>()),
+            DataType::Utf8 => Self::from_list_array::<Utf8Type>(arr.as_list::<i32>()),
+            DataType::LargeUtf8 => Self::from_list_array::<LargeUtf8Type>(arr.as_list::<i64>()),
             _ => panic!("Binary decoder does not support this data type"),
-        }
+        };
+        // data_size is only tracked in the v2.1 structural decode path; the legacy
+        // v2.0 path does not need it so we return 0.
+        Ok((result, 0))
     }
 }
diff --git a/rust/lance-encoding/src/previous/encodings/logical/blob.rs b/rust/lance-encoding/src/previous/encodings/logical/blob.rs
index 460b6e2bbea..b54c7f89e5b 100644
--- a/rust/lance-encoding/src/previous/encodings/logical/blob.rs
+++ b/rust/lance-encoding/src/previous/encodings/logical/blob.rs
@@ -77,7 +77,7 @@ impl SchedulingJob for BlobFieldSchedulingJob<'_> {
                     .await
                     .unwrap();
                 let descriptions_task = decoder.drain(decoder.num_rows()).unwrap();
-                descriptions_task.task.decode()
+                descriptions_task.task.decode().map(|(arr, _)| arr)
             }
             .boxed();
             let decoder = Box::new(BlobFieldDecoder {
@@ -254,7 +254,7 @@ impl BlobArrayDecodeTask {
 }
 
 impl DecodeArrayTask for BlobArrayDecodeTask {
-    fn decode(self: Box<Self>) -> Result<ArrayRef> {
+    fn decode(self: Box<Self>) -> Result<(ArrayRef, u64)> {
         let num_bytes = self.bytes.iter().map(|b| b.len()).sum::<usize>();
         let offsets = self
             .bytes
@@ -272,11 +272,12 @@ impl DecodeArrayTask for BlobArrayDecodeTask {
             buffer.extend_from_slice(&bytes);
         }
         let data_buf = Buffer::from_vec(buffer);
-        Ok(Arc::new(LargeBinaryArray::new(
-            offsets,
-            data_buf,
-            self.validity,
-        )))
+        // data_size is only tracked in the v2.1 structural decode path; the legacy
+        // v2.0 path does not need it so we return 0.
+        Ok((
+            Arc::new(LargeBinaryArray::new(offsets, data_buf, self.validity)),
+            0,
+        ))
     }
 }
 
@@ -386,7 +387,7 @@ impl FieldEncoder for BlobFieldEncoder {
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
     use std::{
         collections::HashMap,
         sync::{Arc, LazyLock},
diff --git a/rust/lance-encoding/src/previous/encodings/logical/list.rs b/rust/lance-encoding/src/previous/encodings/logical/list.rs
index 785a5e2632d..83f93cb22e6 100644
--- a/rust/lance-encoding/src/previous/encodings/logical/list.rs
+++ b/rust/lance-encoding/src/previous/encodings/logical/list.rs
@@ -330,7 +330,7 @@ async fn indirect_schedule_task(
     // pages.  We can use a dummy receiver to match the decoder API
     offsets_decoder.wait_for_loaded(num_offsets - 1).await?;
     let decode_task = offsets_decoder.drain(num_offsets)?;
-    let offsets = decode_task.task.decode()?;
+    let (offsets, _) = decode_task.task.decode()?;
 
     let (item_ranges, offsets, validity) =
         decode_offsets(offsets.as_ref(), &list_requests, null_offset_adjustment);
@@ -615,13 +615,13 @@ struct ListDecodeTask {
 }
 
 impl DecodeArrayTask for ListDecodeTask {
-    fn decode(self: Box<Self>) -> Result<ArrayRef> {
+    fn decode(self: Box<Self>) -> Result<(ArrayRef, u64)> {
         let items = self
             .items
             .map(|items| {
                 // When we run the indirect I/O we wrap things in a struct array with a single field
                 // named "item".  We can unwrap that now.
-                let wrapped_items = items.decode()?;
+                let (wrapped_items, _) = items.decode()?;
                 Result::Ok(wrapped_items.as_struct().column(0).clone())
             })
             .unwrap_or_else(|| Ok(new_empty_array(self.items_field.data_type())))?;
@@ -640,33 +640,36 @@ impl DecodeArrayTask for ListDecodeTask {
         };
         let min_offset = UInt64Array::new_scalar(offsets.value(0));
         let offsets = arrow_arith::numeric::sub(&offsets, &min_offset)?;
-        match &self.offset_type {
+        let array: ArrayRef = match &self.offset_type {
             DataType::Int32 => {
                 let offsets = arrow_cast::cast(&offsets, &DataType::Int32)?;
                 let offsets_i32 = offsets.as_primitive::<Int32Type>();
                 let offsets = OffsetBuffer::new(offsets_i32.values().clone());
 
-                Ok(Arc::new(ListArray::try_new(
+                Arc::new(ListArray::try_new(
                     self.items_field.clone(),
                     offsets,
                     items,
                     validity,
-                )?))
+                )?)
             }
             DataType::Int64 => {
                 let offsets = arrow_cast::cast(&offsets, &DataType::Int64)?;
                 let offsets_i64 = offsets.as_primitive::<Int64Type>();
                 let offsets = OffsetBuffer::new(offsets_i64.values().clone());
 
-                Ok(Arc::new(LargeListArray::try_new(
+                Arc::new(LargeListArray::try_new(
                     self.items_field.clone(),
                     offsets,
                     items,
                     validity,
-                )?))
+                )?)
             }
             _ => panic!("ListDecodeTask with data type that is not i32 or i64"),
-        }
+        };
+        // data_size is only tracked in the v2.1 structural decode path; the legacy
+        // v2.0 path does not need it so we return 0.
+        Ok((array, 0))
     }
 }
 
diff --git a/rust/lance-encoding/src/previous/encodings/logical/primitive.rs b/rust/lance-encoding/src/previous/encodings/logical/primitive.rs
index 794a45bf783..d1debf3ef33 100644
--- a/rust/lance-encoding/src/previous/encodings/logical/primitive.rs
+++ b/rust/lance-encoding/src/previous/encodings/logical/primitive.rs
@@ -288,7 +288,7 @@ struct PrimitiveFieldDecodeTask {
 }
 
 impl DecodeArrayTask for PrimitiveFieldDecodeTask {
-    fn decode(self: Box<Self>) -> Result<ArrayRef> {
+    fn decode(self: Box<Self>) -> Result<(ArrayRef, u64)> {
         let block = self
             .physical_decoder
             .decode(self.rows_to_skip, self.rows_to_take)?;
@@ -313,10 +313,12 @@ impl DecodeArrayTask for PrimitiveFieldDecodeTask {
                         .data_type(dict.data_type().clone())
                         .build()?,
                 );
-                return Ok(new_array);
+                return Ok((new_array, 0));
             }
         }
-        Ok(array)
+        // data_size is only tracked in the v2.1 structural decode path; the legacy
+        // v2.0 path does not need it so we return 0.
+        Ok((array, 0))
     }
 }
 
diff --git a/rust/lance-encoding/src/previous/encodings/logical/struct.rs b/rust/lance-encoding/src/previous/encodings/logical/struct.rs
index f26bf118dd9..b117f74ce2b 100644
--- a/rust/lance-encoding/src/previous/encodings/logical/struct.rs
+++ b/rust/lance-encoding/src/previous/encodings/logical/struct.rs
@@ -56,11 +56,13 @@ struct EmptyStructDecodeTask {
 }
 
 impl DecodeArrayTask for EmptyStructDecodeTask {
-    fn decode(self: Box<Self>) -> Result<ArrayRef> {
-        Ok(Arc::new(StructArray::new_empty_fields(
-            self.num_rows as usize,
-            None,
-        )))
+    fn decode(self: Box<Self>) -> Result<(ArrayRef, u64)> {
+        // data_size is only tracked in the v2.1 structural decode path; the legacy
+        // v2.0 path does not need it so we return 0.
+        Ok((
+            Arc::new(StructArray::new_empty_fields(self.num_rows as usize, None)),
+            0,
+        ))
     }
 }
 
@@ -580,7 +582,7 @@ impl CompositeDecodeTask {
         let arrays = self
             .tasks
             .into_iter()
-            .map(|task| task.decode())
+            .map(|task| task.decode().map(|(arr, _)| arr))
             .collect::<Result<Vec<_>>>()?;
         let array_refs = arrays.iter().map(|arr| arr.as_ref()).collect::<Vec<_>>();
         // TODO: If this is a primitive column we should be able to avoid this
@@ -599,16 +601,17 @@ struct SimpleStructDecodeTask {
 }
 
 impl DecodeArrayTask for SimpleStructDecodeTask {
-    fn decode(self: Box<Self>) -> Result<ArrayRef> {
+    fn decode(self: Box<Self>) -> Result<(ArrayRef, u64)> {
         let child_arrays = self
             .children
             .into_iter()
             .map(|child| child.decode())
             .collect::<Result<Vec<_>>>()?;
-        Ok(Arc::new(StructArray::try_new(
-            self.child_fields,
-            child_arrays,
-            None,
-        )?))
+        // data_size is only tracked in the v2.1 structural decode path; the legacy
+        // v2.0 path does not need it so we return 0.
+        Ok((
+            Arc::new(StructArray::try_new(self.child_fields, child_arrays, None)?),
+            0,
+        ))
     }
 }
diff --git a/rust/lance-encoding/src/previous/encodings/physical/binary.rs b/rust/lance-encoding/src/previous/encodings/physical/binary.rs
index fe1e3a9bd3a..0bd4d96b45e 100644
--- a/rust/lance-encoding/src/previous/encodings/physical/binary.rs
+++ b/rust/lance-encoding/src/previous/encodings/physical/binary.rs
@@ -57,7 +57,7 @@ impl IndicesNormalizer {
         }
     }
 
-    fn extend(&mut self, new_indices: &PrimitiveArray<UInt64Type>, is_start: bool) {
+    fn extend(&mut self, new_indices: &PrimitiveArray<UInt64Type>, is_start: bool) -> Result<()> {
         let mut last = *self.indices.last().unwrap();
         if is_start {
             let (is_valid, val) = self.normalize(new_indices.value(0));
@@ -66,14 +66,25 @@ impl IndicesNormalizer {
             last += val;
         }
         let mut prev = self.normalize(*new_indices.values().first().unwrap()).1;
-        for w in new_indices.values().windows(2) {
+        for (i, w) in new_indices.values().windows(2).enumerate() {
             let (is_valid, val) = self.normalize(w[1]);
-            let next = val - prev + last;
+            let next = match val.checked_sub(prev) {
+                Some(delta) => delta + last,
+                None => {
+                    return Err(lance_core::Error::invalid_input(format!(
+                        "corrupt binary page: normalized offset {} is less than previous offset {} \
+                             at index {}, null_adjustment={}, raw values were [{}, {}]. \
+                             This usually indicates the file data has been corrupted.",
+                        val, prev, i, self.null_adjustment, w[0], w[1]
+                    )));
+                }
+            };
             self.indices.push(next);
             self.validity.append(is_valid);
             prev = val;
             last = next;
         }
+        Ok(())
     }
 
     fn into_parts(mut self) -> (Vec<u64>, BooleanBuffer) {
@@ -109,7 +120,7 @@ impl BinaryPageScheduler {
             PrimitiveFieldDecoder::new_from_data(decoder, DataType::UInt64, num_rows, false);
         let drained_task = primitive_wrapper.drain(num_rows)?;
         let indices_decode_task = drained_task.task;
-        indices_decode_task.decode()
+        indices_decode_task.decode().map(|(arr, _)| arr)
     }
 }
 
@@ -201,11 +212,12 @@ impl PageScheduler for BinaryPageScheduler {
                 let last = indices_builder
                     .normalize(*curr_indices.values().last().unwrap())
                     .1;
+
                 if first != last {
                     bytes_ranges.push(first..last);
                 }
 
-                indices_builder.extend(&curr_indices, row_start == 0);
+                indices_builder.extend(&curr_indices, row_start == 0)?;
             }
 
             let (indices, validity) = indices_builder.into_parts();
diff --git a/rust/lance-encoding/src/previous/encodings/physical/dictionary.rs b/rust/lance-encoding/src/previous/encodings/physical/dictionary.rs
index 759ea5a4429..a4c01ff9ef5 100644
--- a/rust/lance-encoding/src/previous/encodings/physical/dictionary.rs
+++ b/rust/lance-encoding/src/previous/encodings/physical/dictionary.rs
@@ -101,7 +101,7 @@ impl PageScheduler for DictionaryPageScheduler {
                 // Decode all items
                 let drained_task = primitive_wrapper.drain(copy_size)?;
                 let items_decode_task = drained_task.task;
-                let decoded_dict = items_decode_task.decode()?;
+                let (decoded_dict, _) = items_decode_task.decode()?;
 
                 let indices_decoder: Box<dyn PrimitivePageDecoder> = indices_page_decoder.await?;
 
@@ -403,7 +403,7 @@ impl ArrayEncoder for DictionaryEncoder {
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
 
     use arrow_array::{
         ArrayRef, DictionaryArray, StringArray, UInt8Array,
diff --git a/rust/lance-encoding/src/previous/encodings/physical/packed_struct.rs b/rust/lance-encoding/src/previous/encodings/physical/packed_struct.rs
index 9b151afebd7..0f3a5fc3832 100644
--- a/rust/lance-encoding/src/previous/encodings/physical/packed_struct.rs
+++ b/rust/lance-encoding/src/previous/encodings/physical/packed_struct.rs
@@ -256,7 +256,7 @@ impl ArrayEncoder for PackedStructEncoder {
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
 
     use arrow_array::{ArrayRef, Int32Array, StructArray, UInt8Array, UInt64Array};
     use arrow_schema::{DataType, Field, Fields};
diff --git a/rust/lance-encoding/src/repdef.rs b/rust/lance-encoding/src/repdef.rs
index 5530f78a722..fc11e25f931 100644
--- a/rust/lance-encoding/src/repdef.rs
+++ b/rust/lance-encoding/src/repdef.rs
@@ -817,6 +817,10 @@ impl RepDefBuilder {
     /// Registers a nullable validity bitmap
     pub fn add_validity_bitmap(&mut self, validity: NullBuffer) {
         self.check_validity_len(validity.len());
+        if validity.null_count() == 0 {
+            self.add_no_null(validity.len());
+            return;
+        }
         self.repdefs.push(RawRepDef::Validity(ValidityDesc {
             num_values: validity.len(),
             validity: Some(validity.into_inner()),
@@ -1225,7 +1229,7 @@ impl RepDefUnraveler {
     }
 
     pub fn is_all_valid(&self) -> bool {
-        self.def_meaning[self.current_layer].is_all_valid()
+        self.def_levels.is_none() || self.def_meaning[self.current_layer].is_all_valid()
     }
 
     /// If the current level is a repetition layer then this returns the number of lists
@@ -1400,15 +1404,14 @@ impl RepDefUnraveler {
     }
 
     pub fn skip_validity(&mut self) {
-        debug_assert!(
-            self.def_meaning[self.current_layer] == DefinitionInterpretation::AllValidItem
-        );
+        debug_assert!(self.is_all_valid());
         self.current_layer += 1;
     }
 
     /// Unravels a layer of validity from the definition levels
     pub fn unravel_validity(&mut self, validity: &mut BooleanBufferBuilder) {
-        if self.def_meaning[self.current_layer] == DefinitionInterpretation::AllValidItem {
+        let meaning = self.def_meaning[self.current_layer];
+        if meaning == DefinitionInterpretation::AllValidItem || self.def_levels.is_none() {
             self.current_layer += 1;
             validity.append_n(self.num_items as usize, true);
             return;
@@ -2834,6 +2837,26 @@ mod tests {
         assert_eq!([0, 0, 0, 3, 1, 1, 2, 1, 0, 0, 1], *def);
     }
 
+    #[test]
+    fn test_all_valid_validity_bitmap_serializes_as_no_null() {
+        let mut from_bitmap = RepDefBuilder::default();
+        from_bitmap.add_validity_bitmap(validity(&[true, true, true, true]));
+
+        let mut from_no_null = RepDefBuilder::default();
+        from_no_null.add_no_null(4);
+
+        let from_bitmap = RepDefBuilder::serialize(vec![from_bitmap]);
+        let from_no_null = RepDefBuilder::serialize(vec![from_no_null]);
+
+        assert!(from_bitmap.repetition_levels.is_none());
+        assert!(from_bitmap.definition_levels.is_none());
+        assert_eq!(from_bitmap.def_meaning, from_no_null.def_meaning);
+        assert_eq!(
+            from_bitmap.max_visible_level,
+            from_no_null.max_visible_level
+        );
+    }
+
     #[test]
     fn test_slicer() {
         let mut builder = RepDefBuilder::default();
@@ -3184,4 +3207,31 @@ mod tests {
             )
         );
     }
+
+    #[test]
+    fn test_mixed_unraveler_nullable_without_def_levels() {
+        // A page can keep nullable layer metadata even when all definition levels are 0
+        // and no definition buffer needs to be materialized. This should decode as all-valid.
+        let mut unraveler = CompositeRepDefUnraveler::new(vec![
+            RepDefUnraveler::new(
+                None,
+                Some(vec![0, 1, 0, 1]),
+                vec![DefinitionInterpretation::NullableItem].into(),
+                4,
+            ),
+            RepDefUnraveler::new(
+                None,
+                None,
+                vec![DefinitionInterpretation::NullableItem].into(),
+                4,
+            ),
+        ]);
+
+        assert_eq!(
+            unraveler.unravel_validity(8),
+            Some(validity(&[
+                true, false, true, false, true, true, true, true
+            ]))
+        );
+    }
 }
diff --git a/rust/lance-encoding/src/testing.rs b/rust/lance-encoding/src/testing.rs
index 7016c5be7a4..176083d6d64 100644
--- a/rust/lance-encoding/src/testing.rs
+++ b/rust/lance-encoding/src/testing.rs
@@ -218,6 +218,7 @@ async fn test_decode(
         /*should_validate=*/ true,
         /*spawn_structural_batch_decode_tasks=*/ is_structural_encoding,
         rx,
+        /*batch_size_bytes=*/ None,
     )
     .unwrap();
 
diff --git a/rust/lance-encoding/src/version.rs b/rust/lance-encoding/src/version.rs
index 14b1ee21448..41640d0bf2d 100644
--- a/rust/lance-encoding/src/version.rs
+++ b/rust/lance-encoding/src/version.rs
@@ -27,11 +27,11 @@ pub enum LanceFileVersion {
     //
     /// The legacy (0.1) format
     Legacy,
-    #[default]
     V2_0,
+    #[default]
+    V2_1,
     /// The latest stable release (also the default version for new datasets)
     Stable,
-    V2_1,
     V2_2,
     /// The latest unstable release
     Next,
@@ -42,7 +42,7 @@ impl LanceFileVersion {
     /// Convert Stable or Next to the actual version
     pub fn resolve(&self) -> Self {
         match self {
-            Self::Stable => Self::V2_0,
+            Self::Stable => Self::default(),
             Self::Next => Self::V2_3,
             _ => *self,
         }
diff --git a/rust/lance-file/src/previous/reader.rs b/rust/lance-file/src/previous/reader.rs
index 863aca1afc6..9e1fc175d04 100644
--- a/rust/lance-file/src/previous/reader.rs
+++ b/rust/lance-file/src/previous/reader.rs
@@ -83,12 +83,19 @@ impl<'a, T> StringCacheKey<'a, T> {
     }
 }
 
-impl<T> CacheKey for StringCacheKey<'_, T> {
+impl<T: 'static> CacheKey for StringCacheKey<'_, T> {
     type ValueType = T;
 
     fn key(&self) -> Cow<'_, str> {
         self.key.into()
     }
+
+    fn type_name() -> &'static str {
+        // This is a private, crate-internal key that is only instantiated with
+        // a single concrete T within one build, so std::any::type_name is fine
+        // here — there is no cross-crate collision risk.
+        std::any::type_name::<T>()
+    }
 }
 
 impl FileReader {
@@ -238,7 +245,7 @@ impl FileReader {
         loader: F,
     ) -> Result<Arc<T>>
     where
-        F: Fn(&str) -> Fut,
+        F: Fn(&str) -> Fut + Send + Sync,
         Fut: Future<Output = Result<T>> + Send,
     {
         if let Some(cache) = cache {
diff --git a/rust/lance-file/src/reader.rs b/rust/lance-file/src/reader.rs
index 29c1aa3ccc0..eeafd54a2ee 100644
--- a/rust/lance-file/src/reader.rs
+++ b/rust/lance-file/src/reader.rs
@@ -103,17 +103,70 @@ pub struct CachedFileMetadata {
     pub num_footer_bytes: u64,
     pub major_version: u16,
     pub minor_version: u16,
+    /// The actual total file size in bytes, as reported by the object store.
+    pub file_size_bytes: u64,
+}
+
+impl CachedFileMetadata {
+    /// Total file size in bytes.
+    pub fn file_size(&self) -> u64 {
+        self.file_size_bytes
+    }
 }
 
 impl DeepSizeOf for CachedFileMetadata {
-    // TODO: include size for `column_metadatas` and `column_infos`.
     fn deep_size_of_children(&self, context: &mut Context) -> usize {
-        self.file_schema.deep_size_of_children(context)
-            + self
-                .file_buffers
-                .iter()
-                .map(|file_buffer| file_buffer.deep_size_of_children(context))
-                .sum::<usize>()
+        let schema_size = self.file_schema.deep_size_of_children(context);
+
+        let buffers_size: usize = self
+            .file_buffers
+            .iter()
+            .map(|fb| fb.deep_size_of_children(context))
+            .sum();
+
+        // column_metadatas is Vec<pbfile::ColumnMetadata> (protobuf generated,
+        // does not implement DeepSizeOf). We use prost::Message::encoded_len()
+        // as a proxy for in-memory size. The decoded representation is typically
+        // several times larger than the wire format due to heap-allocated
+        // repeated/string/bytes fields, so we apply a 4x multiplier.
+        let column_metadatas_size: usize = self
+            .column_metadatas
+            .iter()
+            .map(|cm| cm.encoded_len() * 4)
+            .sum::<usize>()
+            + std::mem::size_of_val(self.column_metadatas.as_slice());
+
+        // column_infos is Vec<Arc<ColumnInfo>>. Each ColumnInfo contains
+        // page_infos (with protobuf PageEncoding), buffer offsets, and a
+        // column-level ColumnEncoding protobuf.
+        let column_infos_size: usize = self
+            .column_infos
+            .iter()
+            .map(|ci| {
+                let pages_size: usize = ci
+                    .page_infos
+                    .iter()
+                    .map(|pi| {
+                        let enc_size = match &pi.encoding {
+                            lance_encoding::decoder::PageEncoding::Legacy(e) => e.encoded_len() * 4,
+                            lance_encoding::decoder::PageEncoding::Structural(e) => {
+                                e.encoded_len() * 4
+                            }
+                        };
+                        enc_size
+                            + std::mem::size_of_val(pi.buffer_offsets_and_sizes.as_ref())
+                            + std::mem::size_of::<u64>() * 2 // num_rows + priority
+                    })
+                    .sum();
+                pages_size
+                    + std::mem::size_of_val(ci.buffer_offsets_and_sizes.as_ref())
+                    + ci.encoding.encoded_len() * 4
+                    + std::mem::size_of::<u32>() // index
+                    + std::mem::size_of::<usize>() * 2 // Arc overhead
+            })
+            .sum();
+
+        schema_size + buffers_size + column_metadatas_size + column_infos_size
     }
 }
 
@@ -333,6 +386,13 @@ pub struct FileReaderOptions {
     /// will be read in multiple chunks to control memory usage.
     /// Default: 8MB (DEFAULT_READ_CHUNK_SIZE)
     pub read_chunk_size: u64,
+    /// If set, the reader will produce batches whose total size in bytes
+    /// is approximately this value, overriding the row-based `batch_size`.
+    ///
+    /// This can be set at the dataset level (via `ReadParams::file_reader_options`)
+    /// to provide a default for all scans, or at the scanner level (via
+    /// `Scanner::batch_size_bytes`) to override per scan.
+    pub batch_size_bytes: Option<u64>,
 }
 
 impl Default for FileReaderOptions {
@@ -340,11 +400,12 @@ impl Default for FileReaderOptions {
         Self {
             decoder_config: DecoderConfig::default(),
             read_chunk_size: DEFAULT_READ_CHUNK_SIZE,
+            batch_size_bytes: None,
         }
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct FileReader {
     scheduler: Arc<dyn EncodingsIo>,
     // The default projection to be applied to all reads
@@ -653,6 +714,7 @@ impl FileReader {
             file_buffers: gbo_table,
             major_version: footer.major_version,
             minor_version: footer.minor_version,
+            file_size_bytes: file_len,
         })
     }
 
@@ -861,6 +923,7 @@ impl FileReader {
         projection: ReaderProjection,
         filter: FilterExpression,
         decoder_config: DecoderConfig,
+        batch_size_bytes: Option<u64>,
     ) -> Result<BoxStream<'static, ReadBatchTask>> {
         debug!(
             "Reading range {:?} with batch_size {} from file with {} rows and {} columns into schema with {} columns",
@@ -877,6 +940,7 @@ impl FileReader {
             decoder_plugins,
             io,
             decoder_config,
+            batch_size_bytes,
         };
 
         let requested_rows = RequestedRows::Ranges(vec![range]);
@@ -910,6 +974,7 @@ impl FileReader {
             projection,
             filter,
             self.options.decoder_config.clone(),
+            self.options.batch_size_bytes,
         )
     }
 
@@ -924,6 +989,7 @@ impl FileReader {
         projection: ReaderProjection,
         filter: FilterExpression,
         decoder_config: DecoderConfig,
+        batch_size_bytes: Option<u64>,
     ) -> Result<BoxStream<'static, ReadBatchTask>> {
         debug!(
             "Taking {} rows spread across range {}..{} with batch_size {} from columns {:?}",
@@ -940,6 +1006,7 @@ impl FileReader {
             decoder_plugins,
             io,
             decoder_config,
+            batch_size_bytes,
         };
 
         let requested_rows = RequestedRows::Indices(indices);
@@ -971,6 +1038,7 @@ impl FileReader {
             projection,
             FilterExpression::no_filter(),
             self.options.decoder_config.clone(),
+            self.options.batch_size_bytes,
         )
     }
 
@@ -985,6 +1053,7 @@ impl FileReader {
         projection: ReaderProjection,
         filter: FilterExpression,
         decoder_config: DecoderConfig,
+        batch_size_bytes: Option<u64>,
     ) -> Result<BoxStream<'static, ReadBatchTask>> {
         let num_rows = ranges.iter().map(|r| r.end - r.start).sum::<u64>();
         debug!(
@@ -1003,6 +1072,7 @@ impl FileReader {
             decoder_plugins,
             io,
             decoder_config,
+            batch_size_bytes,
         };
 
         let requested_rows = RequestedRows::Ranges(ranges);
@@ -1034,6 +1104,7 @@ impl FileReader {
             projection,
             filter,
             self.options.decoder_config.clone(),
+            self.options.batch_size_bytes,
         )
     }
 
@@ -1181,6 +1252,7 @@ impl FileReader {
             decoder_plugins: self.decoder_plugins.clone(),
             io: self.scheduler.clone(),
             decoder_config: self.options.decoder_config.clone(),
+            batch_size_bytes: self.options.batch_size_bytes,
         };
 
         let requested_rows = RequestedRows::Indices(indices);
@@ -1220,6 +1292,7 @@ impl FileReader {
             decoder_plugins: self.decoder_plugins.clone(),
             io: self.scheduler.clone(),
             decoder_config: self.options.decoder_config.clone(),
+            batch_size_bytes: self.options.batch_size_bytes,
         };
 
         let requested_rows = RequestedRows::Ranges(ranges);
@@ -1259,6 +1332,7 @@ impl FileReader {
             decoder_plugins: self.decoder_plugins.clone(),
             io: self.scheduler.clone(),
             decoder_config: self.options.decoder_config.clone(),
+            batch_size_bytes: self.options.batch_size_bytes,
         };
 
         let requested_rows = RequestedRows::Ranges(vec![range]);
@@ -1532,7 +1606,7 @@ impl EncodedBatchReaderExt for EncodedBatch {
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
     use std::{collections::BTreeMap, pin::Pin, sync::Arc};
 
     use arrow_array::{
@@ -2259,4 +2333,68 @@ pub mod tests {
         let buf = file_reader.read_global_buffer(1).await.unwrap();
         assert_eq!(buf, test_bytes);
     }
+
+    #[rstest]
+    #[tokio::test]
+    async fn test_deep_size_of_includes_column_metadata(
+        #[values(
+            LanceFileVersion::V2_0,
+            LanceFileVersion::V2_1,
+            LanceFileVersion::V2_2,
+            LanceFileVersion::V2_3
+        )]
+        version: LanceFileVersion,
+    ) {
+        // Regression test: CachedFileMetadata::deep_size_of must account for
+        // column_metadatas and column_infos, otherwise the moka cache weigher
+        // dramatically underestimates entry sizes and never evicts, causing
+        // unbounded memory growth on random-access workloads.
+        use deepsize::DeepSizeOf;
+
+        let fs = FsFixture::default();
+        let _written = create_some_file(&fs, version).await;
+        let cache = test_cache();
+        let file_scheduler = fs
+            .scheduler
+            .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let file_reader = FileReader::try_open(
+            file_scheduler,
+            None,
+            Arc::<DecoderPlugins>::default(),
+            &cache,
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+
+        let metadata = file_reader.metadata();
+        let deep_size = metadata.deep_size_of();
+
+        // The file has multiple columns (score, location, categories, binary,
+        // maybe large_bin). The deep_size_of must be substantially more than
+        // just the schema — it should include column_metadatas + column_infos.
+        // A naive implementation that ignores these fields reports < 1 KB;
+        // a correct one should report at least several KB for this test file.
+        assert!(
+            deep_size > 1024,
+            "deep_size_of ({deep_size}) is suspiciously small — \
+             column_metadatas and column_infos may not be accounted for"
+        );
+
+        // Verify column_metadatas is non-empty (sanity check).
+        assert!(
+            !metadata.column_metadatas.is_empty(),
+            "Expected non-empty column_metadatas"
+        );
+
+        // Verify the size scales with the number of columns: a file with more
+        // columns should have a larger deep_size_of.
+        let num_columns = metadata.column_metadatas.len();
+        assert!(
+            deep_size > num_columns * 50,
+            "deep_size_of ({deep_size}) should scale with column count ({num_columns})"
+        );
+    }
 }
diff --git a/rust/lance-index/Cargo.toml b/rust/lance-index/Cargo.toml
index 515c58cd53a..18968f77b01 100644
--- a/rust/lance-index/Cargo.toml
+++ b/rust/lance-index/Cargo.toml
@@ -45,6 +45,7 @@ lance-file.workspace = true
 lance-geo = { workspace = true, optional = true }
 lance-io.workspace = true
 lance-linalg.workspace = true
+lance-tokenizer.workspace = true
 lance-table.workspace = true
 libm.workspace = true
 log.workspace = true
@@ -60,14 +61,12 @@ serde_json.workspace = true
 serde.workspace = true
 snafu.workspace = true
 smallvec = "1.15"
-tantivy.workspace = true
-lindera = { workspace = true, optional = true }
-lindera-tantivy = { workspace = true, optional = true }
 tokio.workspace = true
 tracing.workspace = true
 tempfile.workspace = true
 crossbeam-queue.workspace = true
 bytes.workspace = true
+chrono.workspace = true
 uuid.workspace = true
 twox-hash = "2.0"
 async-channel = "2.3.1"
@@ -81,6 +80,7 @@ approx.workspace = true
 criterion.workspace = true
 env_logger = "0.11.6"
 geo-traits.workspace = true
+libc.workspace = true
 lance-datagen.workspace = true
 lance-testing.workspace = true
 test-log.workspace = true
@@ -90,8 +90,10 @@ chrono.workspace = true
 [features]
 geo = ["dep:lance-geo", "lance-geo/geo", "dep:geoarrow-array", "dep:geoarrow-schema", "dep:geo-types"]
 protoc = ["dep:protobuf-src"]
-tokenizer-lindera = ["dep:lindera", "dep:lindera-tantivy"]
-tokenizer-jieba = ["dep:jieba-rs"]
+jieba-rs = ["tokenizer-jieba"]
+lindera = ["tokenizer-lindera"]
+tokenizer-lindera = ["lance-tokenizer/tokenizer-lindera"]
+tokenizer-jieba = ["dep:jieba-rs", "lance-tokenizer/tokenizer-jieba"]
 
 [build-dependencies]
 prost-build.workspace = true
diff --git a/rust/lance-index/benches/find_partitions.rs b/rust/lance-index/benches/find_partitions.rs
index eb36f59f3f1..bade4e3bfee 100644
--- a/rust/lance-index/benches/find_partitions.rs
+++ b/rust/lance-index/benches/find_partitions.rs
@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-mod sq;
-
 use arrow_array::Float32Array;
 use arrow_array::{FixedSizeListArray, types::Float32Type};
 use lance_arrow::FixedSizeListArrayExt;
diff --git a/rust/lance-index/benches/hnsw.rs b/rust/lance-index/benches/hnsw.rs
index a2731d71eb8..f1e22c5acc8 100644
--- a/rust/lance-index/benches/hnsw.rs
+++ b/rust/lance-index/benches/hnsw.rs
@@ -14,11 +14,13 @@ use lance_arrow::FixedSizeListArrayExt;
 use lance_index::vector::v3::subindex::IvfSubIndex;
 #[cfg(target_os = "linux")]
 use pprof::criterion::{Output, PProfProfiler};
+use rayon::ThreadPoolBuilder;
 
 use lance_core::ROW_ID_FIELD;
 use lance_index::vector::{
     flat::storage::FlatFloatStorage,
     hnsw::builder::{HNSW, HnswBuildParams, HnswQueryParams},
+    pq::{PQBuildParams, ProductQuantizer},
     quantizer::Quantization,
     sq::{ScalarQuantizer, builder::SQBuildParams},
     storage::StorageBuilder,
@@ -64,7 +66,10 @@ fn bench_hnsw(c: &mut Criterion) {
         })
     });
 
-    let hnsw = HNSW::index_vectors(vectors.as_ref(), HnswBuildParams::default()).unwrap();
+    let search_build_pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap();
+    let hnsw = search_build_pool
+        .install(|| HNSW::index_vectors(vectors.as_ref(), HnswBuildParams::default()))
+        .unwrap();
     c.bench_function(format!("search_hnsw{TOTAL}x{DIMENSION}").as_str(), |b| {
         b.to_async(&rt).iter(|| async {
             let uids: HashSet<u32> = hnsw
@@ -154,7 +159,10 @@ fn bench_hnsw_sq(c: &mut Criterion) {
         },
     );
 
-    let hnsw = HNSW::index_vectors(vectors.as_ref(), HnswBuildParams::default()).unwrap();
+    let search_build_pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap();
+    let hnsw = search_build_pool
+        .install(|| HNSW::index_vectors(vectors.as_ref(), HnswBuildParams::default()))
+        .unwrap();
     c.bench_function(format!("search_hnsw_sq{TOTAL}x{DIMENSION}").as_str(), |b| {
         b.to_async(&rt).iter(|| async {
             let uids: HashSet<u32> = hnsw
@@ -180,6 +188,102 @@ fn bench_hnsw_sq(c: &mut Criterion) {
     });
 }
 
+fn bench_hnsw_pq(c: &mut Criterion) {
+    const DIMENSION: usize = 128;
+    const TOTAL: usize = 100_000;
+    const SEED: [u8; 32] = [42; 32];
+    const K: usize = 100;
+
+    let rt = tokio::runtime::Runtime::new().unwrap();
+
+    let data = generate_random_array_with_seed::<Float32Type>(TOTAL * DIMENSION, SEED);
+    let fsl = FixedSizeListArray::try_new_from_values(data, DIMENSION as i32).unwrap();
+    let quantizer = <ProductQuantizer as Quantization>::build(
+        &fsl,
+        DistanceType::L2,
+        &PQBuildParams::new(16, 8),
+    )
+    .unwrap();
+
+    let schema = Arc::new(Schema::new(vec![
+        Field::new(
+            "vector",
+            DataType::FixedSizeList(
+                Field::new_list_field(DataType::Float32, true).into(),
+                DIMENSION as i32,
+            ),
+            true,
+        ),
+        ROW_ID_FIELD.clone(),
+    ]));
+    let row_ids = UInt64Array::from_iter_values((0..TOTAL).map(|v| v as u64));
+    let batch =
+        RecordBatch::try_new(schema, vec![Arc::new(fsl.clone()), Arc::new(row_ids)]).unwrap();
+    let pq_storage = StorageBuilder::new("vector".to_owned(), DistanceType::L2, quantizer, None)
+        .unwrap()
+        .build(vec![batch])
+        .unwrap();
+    let vectors = Arc::new(pq_storage);
+
+    let query = fsl.value(0);
+    c.bench_function(
+        format!("create_hnsw_pq({TOTAL}x{DIMENSION})").as_str(),
+        |b| {
+            b.to_async(&rt).iter(|| async {
+                let hnsw =
+                    HNSW::index_vectors(vectors.as_ref(), HnswBuildParams::default()).unwrap();
+                let uids: HashSet<u32> = hnsw
+                    .search_basic(
+                        query.clone(),
+                        K,
+                        &HnswQueryParams {
+                            ef: 300,
+                            lower_bound: None,
+                            upper_bound: None,
+                            dist_q_c: 0.0,
+                        },
+                        None,
+                        vectors.as_ref(),
+                    )
+                    .unwrap()
+                    .iter()
+                    .map(|node| node.id)
+                    .collect();
+
+                assert_eq!(uids.len(), K);
+            })
+        },
+    );
+
+    let search_build_pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap();
+    let hnsw = search_build_pool
+        .install(|| HNSW::index_vectors(vectors.as_ref(), HnswBuildParams::default()))
+        .unwrap();
+    c.bench_function(format!("search_hnsw_pq{TOTAL}x{DIMENSION}").as_str(), |b| {
+        b.to_async(&rt).iter(|| async {
+            let uids: HashSet<u32> = hnsw
+                .search_basic(
+                    query.clone(),
+                    K,
+                    &HnswQueryParams {
+                        ef: 300,
+                        lower_bound: None,
+                        upper_bound: None,
+                        dist_q_c: 0.0,
+                    },
+                    None,
+                    vectors.as_ref(),
+                )
+                .unwrap()
+                .iter()
+                .map(|node| node.id)
+                .collect();
+
+            assert_eq!(uids.len(), K);
+        })
+    });
+}
+
 #[cfg(target_os = "linux")]
 criterion_group!(
     name=benches;
@@ -187,7 +291,7 @@ criterion_group!(
         .measurement_time(Duration::from_secs(10))
         .sample_size(10)
         .with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
-    targets = bench_hnsw, bench_hnsw_sq);
+    targets = bench_hnsw, bench_hnsw_sq, bench_hnsw_pq);
 
 // Non-linux version does not support pprof.
 #[cfg(not(target_os = "linux"))]
@@ -196,6 +300,6 @@ criterion_group!(
     config = Criterion::default()
         .measurement_time(Duration::from_secs(10))
         .sample_size(10);
-    targets = bench_hnsw, bench_hnsw_sq);
+    targets = bench_hnsw, bench_hnsw_sq, bench_hnsw_pq);
 
 criterion_main!(benches);
diff --git a/rust/lance-index/benches/inverted.rs b/rust/lance-index/benches/inverted.rs
index d43e3c001af..0647cca0c14 100644
--- a/rust/lance-index/benches/inverted.rs
+++ b/rust/lance-index/benches/inverted.rs
@@ -15,7 +15,7 @@ use itertools::Itertools;
 use lance_core::ROW_ID;
 use lance_core::cache::LanceCache;
 use lance_index::prefilter::NoFilter;
-use lance_index::scalar::inverted::lance_tokenizer::DocType;
+use lance_index::scalar::inverted::document_tokenizer::DocType;
 use lance_index::scalar::inverted::query::{FtsSearchParams, Operator, Tokens};
 use lance_index::scalar::inverted::{InvertedIndex, InvertedIndexBuilder};
 use lance_index::scalar::lance_format::LanceIndexStore;
@@ -34,15 +34,22 @@ fn bench_inverted(c: &mut Criterion) {
 
     let rt = tokio::runtime::Builder::new_multi_thread().build().unwrap();
 
-    let tempdir = tempfile::tempdir().unwrap();
-    let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
-    let store = rt.block_on(async {
-        Arc::new(LanceIndexStore::new(
-            Arc::new(ObjectStore::local()),
-            index_dir,
-            Arc::new(LanceCache::no_cache()),
-        ))
-    });
+    let make_store = |path: &std::path::Path| {
+        let index_dir = Path::from_filesystem_path(path).unwrap();
+        rt.block_on(async {
+            Arc::new(LanceIndexStore::new(
+                Arc::new(ObjectStore::local()),
+                index_dir,
+                Arc::new(LanceCache::no_cache()),
+            ))
+        })
+    };
+    let indexing_tempdir = tempfile::tempdir().unwrap();
+    let indexing_store = make_store(indexing_tempdir.path());
+    let indexing_with_positions_tempdir = tempfile::tempdir().unwrap();
+    let indexing_with_positions_store = make_store(indexing_with_positions_tempdir.path());
+    let phrase_search_tempdir = tempfile::tempdir().unwrap();
+    let phrase_search_store = make_store(phrase_search_tempdir.path());
 
     let row_id_col = Arc::new(UInt64Array::from(
         (0..TOTAL).map(|i| i as u64).collect_vec(),
@@ -90,13 +97,54 @@ fn bench_inverted(c: &mut Criterion) {
             let mut builder =
                 InvertedIndexBuilder::new(InvertedIndexParams::default().with_position(false));
             black_box({
-                builder.update(stream, store.as_ref()).await.unwrap();
+                builder
+                    .update(stream, indexing_store.as_ref(), None)
+                    .await
+                    .unwrap();
                 builder
             });
         })
     });
+
+    c.bench_function(
+        format!("invert_indexing_with_positions({TOTAL})").as_str(),
+        |b| {
+            b.to_async(&rt).iter(|| async {
+                let stream = RecordBatchStreamAdapter::new(
+                    batch.schema(),
+                    stream::iter(vec![Ok(batch.clone())]),
+                );
+                let stream = Box::pin(stream);
+                let mut builder =
+                    InvertedIndexBuilder::new(InvertedIndexParams::default().with_position(true));
+                black_box({
+                    builder
+                        .update(stream, indexing_with_positions_store.as_ref(), None)
+                        .await
+                        .unwrap();
+                    builder
+                });
+            })
+        },
+    );
+
+    rt.block_on(async {
+        let stream =
+            RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch.clone())]));
+        let stream = Box::pin(stream);
+        let mut builder =
+            InvertedIndexBuilder::new(InvertedIndexParams::default().with_position(true));
+        builder
+            .update(stream, phrase_search_store.as_ref(), None)
+            .await
+            .unwrap();
+    });
     let invert_index = rt
-        .block_on(InvertedIndex::load(store, None, &LanceCache::no_cache()))
+        .block_on(InvertedIndex::load(
+            phrase_search_store,
+            None,
+            &LanceCache::no_cache(),
+        ))
         .unwrap();
 
     let params = FtsSearchParams::new().with_limit(Some(10));
@@ -147,6 +195,44 @@ fn bench_inverted(c: &mut Criterion) {
             }
         })
     });
+
+    let phrase_params = FtsSearchParams::new()
+        .with_limit(Some(10))
+        .with_phrase_slop(Some(0));
+    let phrase_pairs = sample_words
+        .windows(2)
+        .map(|pair| {
+            Arc::new(Tokens::new(
+                pair.iter().map(|s| s.to_string()).collect(),
+                DocType::Text,
+            ))
+        })
+        .collect_vec();
+    let mut phrase_query_idx = 0usize;
+
+    c.bench_function(format!("invert_phrase_search({TOTAL})").as_str(), |b| {
+        b.to_async(&rt).iter(|| {
+            let query = phrase_pairs[phrase_query_idx % phrase_pairs.len()].clone();
+            phrase_query_idx = phrase_query_idx.wrapping_add(1);
+            let invert_index = invert_index.clone();
+            let params = phrase_params.clone();
+            let no_filter = no_filter.clone();
+            async move {
+                black_box(
+                    invert_index
+                        .bm25_search(
+                            query,
+                            params.clone().into(),
+                            Operator::And,
+                            no_filter.clone(),
+                            Arc::new(NoOpMetricsCollector),
+                        )
+                        .await
+                        .unwrap(),
+                );
+            }
+        })
+    });
 }
 
 #[cfg(target_os = "linux")]
diff --git a/rust/lance-index/benches/kmeans.rs b/rust/lance-index/benches/kmeans.rs
index e250f59e21c..ad04adb27e8 100644
--- a/rust/lance-index/benches/kmeans.rs
+++ b/rust/lance-index/benches/kmeans.rs
@@ -7,7 +7,6 @@ use arrow_array::FixedSizeListArray;
 use criterion::{Criterion, criterion_group, criterion_main};
 
 use lance_arrow::FixedSizeListArrayExt;
-use lance_index::vector::flat::storage::FlatFloatStorage;
 use lance_index::vector::utils::SimpleIndex;
 #[cfg(target_os = "linux")]
 use pprof::criterion::{Output, PProfProfiler};
@@ -66,9 +65,13 @@ fn bench_train(c: &mut Criterion) {
         });
 
         if k * dimension as usize >= 1_000_000 {
-            let index =
-                SimpleIndex::try_new(FlatFloatStorage::new(centroids.clone(), DistanceType::L2))
-                    .unwrap();
+            let index = SimpleIndex::may_train_index(
+                centroids.values().clone(),
+                dimension as usize,
+                DistanceType::L2,
+            )
+            .unwrap()
+            .unwrap();
             group.bench_function("with_index", |b| {
                 b.iter(|| {
                     KMeansAlgoFloat::<Float32Type>::compute_membership_and_loss(
diff --git a/rust/lance-index/benches/sq.rs b/rust/lance-index/benches/sq.rs
index 2e0828e2117..84b5a304723 100644
--- a/rust/lance-index/benches/sq.rs
+++ b/rust/lance-index/benches/sq.rs
@@ -65,7 +65,6 @@ fn create_sq_batch(row_id_range: Range<u64>, dim: usize) -> RecordBatch {
     batch.drop_column("vector").unwrap()
 }
 
-#[allow(dead_code)]
 pub fn bench_storage(c: &mut Criterion) {
     let mut rng = rand::rng();
 
diff --git a/rust/lance-index/src/frag_reuse.rs b/rust/lance-index/src/frag_reuse.rs
index e122540f653..490d21a7132 100644
--- a/rust/lance-index/src/frag_reuse.rs
+++ b/rust/lance-index/src/frag_reuse.rs
@@ -390,7 +390,7 @@ impl Index for FragReuseIndex {
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
 
     use super::*;
 
diff --git a/rust/lance-index/src/mem_wal.rs b/rust/lance-index/src/mem_wal.rs
index 5bafc370eb4..cbc123b8881 100644
--- a/rust/lance-index/src/mem_wal.rs
+++ b/rust/lance-index/src/mem_wal.rs
@@ -17,8 +17,8 @@ use crate::{Index, IndexType};
 
 pub const MEM_WAL_INDEX_NAME: &str = "__lance_mem_wal";
 
-/// Type alias for region identifier (UUID v4).
-pub type RegionId = Uuid;
+/// Type alias for shard identifier (UUID v4).
+pub type ShardId = Uuid;
 
 /// A flushed MemTable generation and its storage location.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
@@ -45,10 +45,10 @@ impl From<pb::FlushedGeneration> for FlushedGeneration {
     }
 }
 
-/// A region's merged generation, used in MemWalIndexDetails.
+/// A shard's merged generation, used in MemWalIndexDetails.
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash, Serialize, Deserialize)]
 pub struct MergedGeneration {
-    pub region_id: Uuid,
+    pub shard_id: Uuid,
     pub generation: u64,
 }
 
@@ -59,9 +59,9 @@ impl DeepSizeOf for MergedGeneration {
 }
 
 impl MergedGeneration {
-    pub fn new(region_id: Uuid, generation: u64) -> Self {
+    pub fn new(shard_id: Uuid, generation: u64) -> Self {
         Self {
-            region_id,
+            shard_id,
             generation,
         }
     }
@@ -70,7 +70,7 @@ impl MergedGeneration {
 impl From<&MergedGeneration> for pb::MergedGeneration {
     fn from(mg: &MergedGeneration) -> Self {
         Self {
-            region_id: Some((&mg.region_id).into()),
+            shard_id: Some((&mg.shard_id).into()),
             generation: mg.generation,
         }
     }
@@ -80,13 +80,13 @@ impl TryFrom<pb::MergedGeneration> for MergedGeneration {
     type Error = Error;
 
     fn try_from(mg: pb::MergedGeneration) -> lance_core::Result<Self> {
-        let region_id = mg
-            .region_id
+        let shard_id = mg
+            .shard_id
             .as_ref()
             .map(Uuid::try_from)
-            .ok_or_else(|| Error::invalid_input("Missing region_id in MergedGeneration"))??;
+            .ok_or_else(|| Error::invalid_input("Missing shard_id in MergedGeneration"))??;
         Ok(Self {
-            region_id,
+            shard_id,
             generation: mg.generation,
         })
     }
@@ -108,12 +108,12 @@ impl IndexCatchupProgress {
         }
     }
 
-    /// Get the caught up generation for a specific region.
-    /// Returns None if the region is not present (assumed fully caught up).
-    pub fn caught_up_generation_for_region(&self, region_id: &Uuid) -> Option<u64> {
+    /// Get the caught up generation for a specific shard.
+    /// Returns None if the shard is not present (assumed fully caught up).
+    pub fn caught_up_generation_for_shard(&self, shard_id: &Uuid) -> Option<u64> {
         self.caught_up_generations
             .iter()
-            .find(|mg| &mg.region_id == region_id)
+            .find(|mg| &mg.shard_id == shard_id)
             .map(|mg| mg.generation)
     }
 }
@@ -146,13 +146,13 @@ impl TryFrom<pb::IndexCatchupProgress> for IndexCatchupProgress {
     }
 }
 
-/// Region manifest containing epoch-based fencing and WAL state.
-/// Each region has exactly one active writer at any time.
+/// Shard manifest containing epoch-based fencing and WAL state.
+/// Each shard has exactly one active writer at any time.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct RegionManifest {
-    pub region_id: Uuid,
+pub struct ShardManifest {
+    pub shard_id: Uuid,
     pub version: u64,
-    pub region_spec_id: u32,
+    pub shard_spec_id: u32,
     pub writer_epoch: u64,
     /// The most recent WAL entry position (0-based) flushed to a MemTable.
     /// Recovery replays from `replay_after_wal_entry_position + 1`.
@@ -163,18 +163,18 @@ pub struct RegionManifest {
     pub flushed_generations: Vec<FlushedGeneration>,
 }
 
-impl DeepSizeOf for RegionManifest {
+impl DeepSizeOf for ShardManifest {
     fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
         self.flushed_generations.deep_size_of_children(context)
     }
 }
 
-impl From<&RegionManifest> for pb::RegionManifest {
-    fn from(rm: &RegionManifest) -> Self {
+impl From<&ShardManifest> for pb::ShardManifest {
+    fn from(rm: &ShardManifest) -> Self {
         Self {
-            region_id: Some((&rm.region_id).into()),
+            shard_id: Some((&rm.shard_id).into()),
             version: rm.version,
-            region_spec_id: rm.region_spec_id,
+            shard_spec_id: rm.shard_spec_id,
             writer_epoch: rm.writer_epoch,
             replay_after_wal_entry_position: rm.replay_after_wal_entry_position,
             wal_entry_position_last_seen: rm.wal_entry_position_last_seen,
@@ -184,19 +184,19 @@ impl From<&RegionManifest> for pb::RegionManifest {
     }
 }
 
-impl TryFrom<pb::RegionManifest> for RegionManifest {
+impl TryFrom<pb::ShardManifest> for ShardManifest {
     type Error = Error;
 
-    fn try_from(rm: pb::RegionManifest) -> lance_core::Result<Self> {
-        let region_id = rm
-            .region_id
+    fn try_from(rm: pb::ShardManifest) -> lance_core::Result<Self> {
+        let shard_id = rm
+            .shard_id
             .as_ref()
             .map(Uuid::try_from)
-            .ok_or_else(|| Error::invalid_input("Missing region_id in RegionManifest"))??;
+            .ok_or_else(|| Error::invalid_input("Missing shard_id in ShardManifest"))??;
         Ok(Self {
-            region_id,
+            shard_id,
             version: rm.version,
-            region_spec_id: rm.region_spec_id,
+            shard_spec_id: rm.shard_spec_id,
             writer_epoch: rm.writer_epoch,
             replay_after_wal_entry_position: rm.replay_after_wal_entry_position,
             wal_entry_position_last_seen: rm.wal_entry_position_last_seen,
@@ -210,9 +210,9 @@ impl TryFrom<pb::RegionManifest> for RegionManifest {
     }
 }
 
-/// Region field definition.
+/// Shard field definition.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
-pub struct RegionField {
+pub struct ShardField {
     pub field_id: String,
     pub source_ids: Vec<i32>,
     pub transform: Option<String>,
@@ -221,8 +221,8 @@ pub struct RegionField {
     pub parameters: HashMap<String, String>,
 }
 
-impl From<&RegionField> for pb::RegionField {
-    fn from(rf: &RegionField) -> Self {
+impl From<&ShardField> for pb::ShardField {
+    fn from(rf: &ShardField) -> Self {
         Self {
             field_id: rf.field_id.clone(),
             source_ids: rf.source_ids.clone(),
@@ -234,8 +234,8 @@ impl From<&RegionField> for pb::RegionField {
     }
 }
 
-impl From<pb::RegionField> for RegionField {
-    fn from(rf: pb::RegionField) -> Self {
+impl From<pb::ShardField> for ShardField {
+    fn from(rf: pb::ShardField) -> Self {
         Self {
             field_id: rf.field_id,
             source_ids: rf.source_ids,
@@ -247,15 +247,15 @@ impl From<pb::RegionField> for RegionField {
     }
 }
 
-/// Region spec definition.
+/// Shard spec definition.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
-pub struct RegionSpec {
+pub struct ShardSpec {
     pub spec_id: u32,
-    pub fields: Vec<RegionField>,
+    pub fields: Vec<ShardField>,
 }
 
-impl From<&RegionSpec> for pb::RegionSpec {
-    fn from(rs: &RegionSpec) -> Self {
+impl From<&ShardSpec> for pb::ShardSpec {
+    fn from(rs: &ShardSpec) -> Self {
         Self {
             spec_id: rs.spec_id,
             fields: rs.fields.iter().map(|f| f.into()).collect(),
@@ -263,11 +263,11 @@ impl From<&RegionSpec> for pb::RegionSpec {
     }
 }
 
-impl From<pb::RegionSpec> for RegionSpec {
-    fn from(rs: pb::RegionSpec) -> Self {
+impl From<pb::ShardSpec> for ShardSpec {
+    fn from(rs: pb::ShardSpec) -> Self {
         Self {
             spec_id: rs.spec_id,
-            fields: rs.fields.into_iter().map(RegionField::from).collect(),
+            fields: rs.fields.into_iter().map(ShardField::from).collect(),
         }
     }
 }
@@ -276,9 +276,9 @@ impl From<pb::RegionSpec> for RegionSpec {
 #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
 pub struct MemWalIndexDetails {
     pub snapshot_ts_millis: i64,
-    pub num_regions: u32,
+    pub num_shards: u32,
     pub inline_snapshots: Option<Vec<u8>>,
-    pub region_specs: Vec<RegionSpec>,
+    pub shard_specs: Vec<ShardSpec>,
     pub maintained_indexes: Vec<String>,
     pub merged_generations: Vec<MergedGeneration>,
     pub index_catchup: Vec<IndexCatchupProgress>,
@@ -288,9 +288,9 @@ impl From<&MemWalIndexDetails> for pb::MemWalIndexDetails {
     fn from(details: &MemWalIndexDetails) -> Self {
         Self {
             snapshot_ts_millis: details.snapshot_ts_millis,
-            num_regions: details.num_regions,
+            num_shards: details.num_shards,
             inline_snapshots: details.inline_snapshots.clone(),
-            region_specs: details.region_specs.iter().map(|rs| rs.into()).collect(),
+            shard_specs: details.shard_specs.iter().map(|rs| rs.into()).collect(),
             maintained_indexes: details.maintained_indexes.clone(),
             merged_generations: details
                 .merged_generations
@@ -308,12 +308,12 @@ impl TryFrom<pb::MemWalIndexDetails> for MemWalIndexDetails {
     fn try_from(details: pb::MemWalIndexDetails) -> lance_core::Result<Self> {
         Ok(Self {
             snapshot_ts_millis: details.snapshot_ts_millis,
-            num_regions: details.num_regions,
+            num_shards: details.num_shards,
             inline_snapshots: details.inline_snapshots,
-            region_specs: details
-                .region_specs
+            shard_specs: details
+                .shard_specs
                 .into_iter()
-                .map(RegionSpec::from)
+                .map(ShardSpec::from)
                 .collect(),
             maintained_indexes: details.maintained_indexes,
             merged_generations: details
@@ -341,29 +341,29 @@ impl MemWalIndex {
         Self { details }
     }
 
-    pub fn merged_generation_for_region(&self, region_id: &Uuid) -> Option<u64> {
+    pub fn merged_generation_for_shard(&self, shard_id: &Uuid) -> Option<u64> {
         self.details
             .merged_generations
             .iter()
-            .find(|mg| &mg.region_id == region_id)
+            .find(|mg| &mg.shard_id == shard_id)
             .map(|mg| mg.generation)
     }
 
-    /// Get the caught up generation for a specific index and region.
+    /// Get the caught up generation for a specific index and shard.
     /// Returns None if the index is not tracked (assumed fully caught up).
-    pub fn index_caught_up_generation(&self, index_name: &str, region_id: &Uuid) -> Option<u64> {
+    pub fn index_caught_up_generation(&self, index_name: &str, shard_id: &Uuid) -> Option<u64> {
         self.details
             .index_catchup
             .iter()
             .find(|icp| icp.index_name == index_name)
-            .and_then(|icp| icp.caught_up_generation_for_region(region_id))
+            .and_then(|icp| icp.caught_up_generation_for_shard(shard_id))
     }
 
-    /// Check if an index is fully caught up for a region.
-    /// Returns true if the index covers all merged data for the region.
-    pub fn is_index_caught_up(&self, index_name: &str, region_id: &Uuid) -> bool {
-        let merged_gen = self.merged_generation_for_region(region_id).unwrap_or(0);
-        let caught_up_gen = self.index_caught_up_generation(index_name, region_id);
+    /// Check if an index is fully caught up for a shard.
+    /// Returns true if the index covers all merged data for the shard.
+    pub fn is_index_caught_up(&self, index_name: &str, shard_id: &Uuid) -> bool {
+        let merged_gen = self.merged_generation_for_shard(shard_id).unwrap_or(0);
+        let caught_up_gen = self.index_caught_up_generation(index_name, shard_id);
 
         // If not tracked in index_catchup, assumed fully caught up
         caught_up_gen.is_none_or(|generation| generation >= merged_gen)
@@ -372,9 +372,9 @@ impl MemWalIndex {
 
 #[derive(Serialize)]
 struct MemWalStatistics {
-    num_regions: u32,
+    num_shards: u32,
     num_merged_generations: usize,
-    num_region_specs: usize,
+    num_shard_specs: usize,
     num_maintained_indexes: usize,
     num_index_catchup_entries: usize,
 }
@@ -397,9 +397,9 @@ impl Index for MemWalIndex {
 
     fn statistics(&self) -> lance_core::Result<serde_json::Value> {
         let stats = MemWalStatistics {
-            num_regions: self.details.num_regions,
+            num_shards: self.details.num_shards,
             num_merged_generations: self.details.merged_generations.len(),
-            num_region_specs: self.details.region_specs.len(),
+            num_shard_specs: self.details.shard_specs.len(),
             num_maintained_indexes: self.details.maintained_indexes.len(),
             num_index_catchup_entries: self.details.index_catchup.len(),
         };
diff --git a/rust/lance-index/src/optimize.rs b/rust/lance-index/src/optimize.rs
index 68092c28ea5..5b6269a1a99 100644
--- a/rust/lance-index/src/optimize.rs
+++ b/rust/lance-index/src/optimize.rs
@@ -1,6 +1,9 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use std::collections::HashMap;
+use std::sync::Arc;
+
 /// Options for optimizing all indices.
 #[non_exhaustive]
 #[derive(Debug, Clone, Default)]
@@ -33,6 +36,13 @@ pub struct OptimizeOptions {
     ///
     /// NOTE: this option is only supported for v3 vector indices.
     pub retrain: bool,
+
+    /// Transaction properties to store with this commit.
+    ///
+    /// These key-value pairs are stored in the transaction file
+    /// and can be read later to identify the source of the commit
+    /// (e.g., job_id for tracking completed index jobs).
+    pub transaction_properties: Option<Arc<HashMap<String, String>>>,
 }
 
 impl OptimizeOptions {
@@ -61,6 +71,7 @@ impl OptimizeOptions {
             num_indices_to_merge: None,
             index_names: None,
             retrain: true,
+            ..Default::default()
         }
     }
 
@@ -73,4 +84,10 @@ impl OptimizeOptions {
         self.index_names = Some(names);
         self
     }
+
+    /// Set transaction properties to store in the commit manifest.
+    pub fn transaction_properties(mut self, properties: HashMap<String, String>) -> Self {
+        self.transaction_properties = Some(Arc::new(properties));
+        self
+    }
 }
diff --git a/rust/lance-index/src/progress.rs b/rust/lance-index/src/progress.rs
index 4ac664c7623..b02ec4280ba 100644
--- a/rust/lance-index/src/progress.rs
+++ b/rust/lance-index/src/progress.rs
@@ -5,16 +5,18 @@ use async_trait::async_trait;
 use lance_core::Result;
 use std::sync::Arc;
 
-/// Progress callback for index building.
+/// Progress callback for index building and distributed index finalization.
 ///
-/// Called at stage boundaries during index construction. Stages are sequential:
-/// `stage_complete` is always called before the next `stage_start`, so only one
-/// stage is active at a time. Stage names are index-type-specific (e.g.
-/// "train_ivf", "shuffle", "build_partitions" for vector indices; "load_data",
-/// "build_pages" for scalar indices).
+/// Called at stage boundaries during index construction. For a single logical stream, stages are
+/// sequential: `stage_complete` is always called before the next `stage_start`, so only one stage
+/// is active at a time. Callers that orchestrate independent sub-builds in parallel may prefix
+/// stage names (for example `segment_plan[0]/merge_partitions`) to represent separate logical
+/// streams. Stage names are index-type-specific (e.g. "train_ivf", "shuffle", "merge_partitions"
+/// for vector indices; "load_data", "build_pages" for scalar indices; merge/finalization stages
+/// for distributed index construction).
 ///
-/// Methods take `&self` to allow concurrent calls from within a single stage.
-/// Implementations must be thread-safe.
+/// Methods take `&self` to allow concurrent calls from within a single stage. Implementations
+/// must be thread-safe.
 #[async_trait]
 pub trait IndexBuildProgress: std::fmt::Debug + Sync + Send {
     /// A named stage has started.
diff --git a/rust/lance-index/src/scalar.rs b/rust/lance-index/src/scalar.rs
index 753f1445d76..dbce2ec1aa7 100644
--- a/rust/lance-index/src/scalar.rs
+++ b/rust/lance-index/src/scalar.rs
@@ -7,6 +7,7 @@ use arrow::buffer::{OffsetBuffer, ScalarBuffer};
 use arrow_array::{BooleanArray, ListArray, RecordBatch, UInt64Array};
 use arrow_schema::{Field, Schema};
 use async_trait::async_trait;
+use bytes::Bytes;
 use datafusion::functions::string::contains::ContainsFunc;
 use datafusion::functions_nested::array_has;
 use datafusion::physical_plan::SendableRecordBatchStream;
@@ -27,6 +28,7 @@ use serde::Serialize;
 use crate::metrics::MetricsCollector;
 use crate::scalar::registry::TrainingCriteria;
 use crate::{Index, IndexParams, IndexType};
+pub use lance_table::format::IndexFile;
 
 pub mod bitmap;
 pub mod bloomfilter;
@@ -172,6 +174,12 @@ pub trait IndexWriter: Send {
     ///
     /// E.g. if this is the third time this is called this method will return 2
     async fn write_record_batch(&mut self, batch: RecordBatch) -> Result<u64>;
+    /// Adds a global buffer and returns its index.
+    async fn add_global_buffer(&mut self, _data: Bytes) -> Result<u32> {
+        Err(Error::not_supported(
+            "global buffers are not supported by this index writer",
+        ))
+    }
     /// Finishes writing the file and closes the file
     async fn finish(&mut self) -> Result<()>;
     /// Finishes writing the file and closes the file with additional metadata
@@ -183,6 +191,12 @@ pub trait IndexWriter: Send {
 pub trait IndexReader: Send + Sync {
     /// Read the n-th record batch from the file
     async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result<RecordBatch>;
+    /// Reads a global buffer by index.
+    async fn read_global_buffer(&self, _index: u32) -> Result<Bytes> {
+        Err(Error::not_supported(
+            "global buffers are not supported by this index reader",
+        ))
+    }
     /// Read the range of rows from the file.
     /// If projection is Some, only return the columns in the projection,
     /// nested columns like Some(&["x.y"]) are not supported.
@@ -208,6 +222,7 @@ pub trait IndexReader: Send + Sync {
 #[async_trait]
 pub trait IndexStore: std::fmt::Debug + Send + Sync + DeepSizeOf {
     fn as_any(&self) -> &dyn Any;
+    fn clone_arc(&self) -> Arc<dyn IndexStore>;
 
     /// Suggested I/O parallelism for the store
     fn io_parallelism(&self) -> usize;
@@ -229,6 +244,12 @@ pub trait IndexStore: std::fmt::Debug + Send + Sync + DeepSizeOf {
 
     /// Delete an index file (used in the tmp spill store to keep tmp size down)
     async fn delete_index_file(&self, name: &str) -> Result<()>;
+
+    /// List all files in the index directory with their sizes.
+    ///
+    /// Returns a list of (relative_path, size_bytes) tuples.
+    /// Used to capture file metadata after index creation/modification.
+    async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>>;
 }
 
 /// Different scalar indices may support different kinds of queries
@@ -360,6 +381,9 @@ pub enum SargableQuery {
     FullTextSearch(FullTextSearchQuery),
     /// Retrieve all row ids where the value is null
     IsNull(),
+    /// Retrieve all row ids where the value matches LIKE 'prefix%' pattern
+    /// This is used for both explicit LIKE expressions and starts_with() function calls
+    LikePrefix(ScalarValue),
 }
 
 impl AnyQuery for SargableQuery {
@@ -408,6 +432,9 @@ impl AnyQuery for SargableQuery {
             Self::Equals(val) => {
                 format!("{} = {}", col, val)
             }
+            Self::LikePrefix(prefix) => {
+                format!("{} LIKE '{}%'", col, prefix)
+            }
         }
     }
 
@@ -460,6 +487,16 @@ impl AnyQuery for SargableQuery {
             )),
             Self::IsNull() => col_expr.is_null(),
             Self::Equals(value) => col_expr.eq(Expr::Literal(value.clone(), None)),
+            Self::LikePrefix(prefix) => {
+                let pattern = match prefix {
+                    ScalarValue::Utf8(Some(s)) => ScalarValue::Utf8(Some(format!("{}%", s))),
+                    ScalarValue::LargeUtf8(Some(s)) => {
+                        ScalarValue::LargeUtf8(Some(format!("{}%", s)))
+                    }
+                    other => other.clone(),
+                };
+                col_expr.like(Expr::Literal(pattern, None))
+            }
         }
     }
 
@@ -787,6 +824,11 @@ pub struct CreatedIndex {
     ///
     /// This can be used to determine if a reader is able to load the index.
     pub index_version: u32,
+    /// List of files and their sizes for this index
+    ///
+    /// This enables skipping HEAD calls when opening indices and provides
+    /// visibility into index storage size via describe_indices().
+    pub files: Option<Vec<IndexFile>>,
 }
 
 /// The criteria that specifies how to update an index
@@ -807,10 +849,13 @@ pub struct UpdateCriteria {
 /// - stable row IDs: use exact row-id membership instead
 #[derive(Debug, Clone)]
 pub enum OldIndexDataFilter {
-    /// Keep old rows whose row-address fragment is in this bitmap.
+    /// Keeps track of which fragments are still valid and which are no longer valid.
     ///
     /// This is valid for address-style row IDs.
-    Fragments(RoaringBitmap),
+    Fragments {
+        to_keep: RoaringBitmap,
+        to_remove: RoaringBitmap,
+    },
     /// Keep old rows whose row IDs are in this exact allow-list.
     ///
     /// This is required for stable row IDs, where row IDs are opaque and
@@ -822,9 +867,9 @@ impl OldIndexDataFilter {
     /// Build a boolean mask that keeps only row IDs selected by this filter.
     pub fn filter_row_ids(&self, row_ids: &UInt64Array) -> BooleanArray {
         match self {
-            Self::Fragments(valid_fragments) => row_ids
+            Self::Fragments { to_keep, .. } => row_ids
                 .iter()
-                .map(|id| id.map(|id| valid_fragments.contains((id >> 32) as u32)))
+                .map(|id| id.map(|id| to_keep.contains((id >> 32) as u32)))
                 .collect(),
             Self::RowIds(valid_row_ids) => row_ids
                 .iter()
@@ -850,6 +895,62 @@ impl UpdateCriteria {
     }
 }
 
+/// Compute the lexicographically next prefix by incrementing the last character's code point.
+/// Returns None if no valid upper bound exists.
+///
+/// This is used for LIKE prefix queries to convert `LIKE 'foo%'` to range `[foo, fop)`.
+///
+/// # UTF-8 and Unicode Handling
+///
+/// This function operates on Unicode code points (characters), not bytes. Since UTF-8
+/// byte ordering is identical to Unicode code point ordering, incrementing a character's
+/// code point produces the correct lexicographic successor for byte-wise string comparison.
+///
+/// If incrementing the last character would overflow or land in the surrogate range
+/// (U+D800-U+DFFF), we try incrementing the previous character, and so on.
+///
+/// Examples:
+/// - `"foo"` → `Some("fop")`
+/// - `"café"` → `Some("cafê")`  (é U+00E9 → ê U+00EA)
+/// - `"abc中"` → `Some("abc丮")` (中 U+4E2D → 丮 U+4E2E)
+/// - `"cafÿ"` → `Some("cafĀ")` (ÿ U+00FF → Ā U+0100)
+pub fn compute_next_prefix(prefix: &str) -> Option<String> {
+    if prefix.is_empty() {
+        return None;
+    }
+
+    let chars: Vec<char> = prefix.chars().collect();
+
+    // Try incrementing characters from right to left
+    for i in (0..chars.len()).rev() {
+        if let Some(next_char) = next_unicode_char(chars[i]) {
+            let mut result: String = chars[..i].iter().collect();
+            result.push(next_char);
+            return Some(result);
+        }
+        // This character cannot be incremented (e.g., U+10FFFF), try previous
+    }
+
+    // All characters were at maximum value
+    None
+}
+
+/// Get the next valid Unicode scalar value after the given character.
+/// Skips the surrogate range (U+D800-U+DFFF) which is not valid in UTF-8.
+fn next_unicode_char(c: char) -> Option<char> {
+    let cp = c as u32;
+    let next_cp = cp.checked_add(1)?;
+
+    // Skip surrogate range (U+D800-U+DFFF)
+    let next_cp = if (0xD800..=0xDFFF).contains(&next_cp) {
+        0xE000
+    } else {
+        next_cp
+    };
+
+    char::from_u32(next_cp)
+}
+
 /// A trait for a scalar index, a structure that can determine row ids that satisfy scalar queries
 #[async_trait]
 pub trait ScalarIndex: Send + Sync + std::fmt::Debug + Index + DeepSizeOf {
diff --git a/rust/lance-index/src/scalar/bitmap.rs b/rust/lance-index/src/scalar/bitmap.rs
index 86931d8a64f..dd490265e0c 100644
--- a/rust/lance-index/src/scalar/bitmap.rs
+++ b/rust/lance-index/src/scalar/bitmap.rs
@@ -13,6 +13,7 @@ use arrow::array::BinaryBuilder;
 use arrow_array::{Array, BinaryArray, RecordBatch, UInt64Array, new_null_array};
 use arrow_schema::{DataType, Field, Schema};
 use async_trait::async_trait;
+use bytes::Bytes;
 use datafusion::physical_plan::SendableRecordBatchStream;
 use datafusion_common::ScalarValue;
 use deepsize::DeepSizeOf;
@@ -128,6 +129,10 @@ impl CacheKey for BitmapKey {
     fn key(&self) -> std::borrow::Cow<'_, str> {
         format!("{}", self.value.0).into()
     }
+
+    fn type_name() -> &'static str {
+        "Bitmap"
+    }
 }
 
 impl BitmapIndex {
@@ -288,6 +293,30 @@ impl BitmapIndex {
 
         Ok(Arc::new(bitmap))
     }
+
+    pub(crate) fn value_type(&self) -> &DataType {
+        &self.value_type
+    }
+
+    /// Loads the current bitmap index into an in-memory value-to-row-id map.
+    pub(crate) async fn load_bitmap_index_state(
+        &self,
+    ) -> Result<HashMap<ScalarValue, RowAddrTreeMap>> {
+        let mut state = HashMap::new();
+
+        for key in self.index_map.keys() {
+            let bitmap = self.load_bitmap(key, None).await?;
+            state.insert(key.0.clone(), (*bitmap).clone());
+        }
+
+        if !self.null_map.is_empty() {
+            let existing_null = new_null_array(&self.value_type, 1);
+            let existing_null = ScalarValue::try_from_array(existing_null.as_ref(), 0)?;
+            state.insert(existing_null, (*self.null_map).clone());
+        }
+
+        Ok(state)
+    }
 }
 
 impl DeepSizeOf for BitmapIndex {
@@ -537,6 +566,11 @@ impl ScalarIndex for BitmapIndex {
                     "full text search is not supported for bitmap indexes".into(),
                 ));
             }
+            SargableQuery::LikePrefix(_) => {
+                return Err(Error::not_supported_source(
+                    "LIKE prefix queries are not supported for bitmap indexes".into(),
+                ));
+            }
         };
 
         let selection = NullableRowAddrSet::new(row_ids, null_row_ids.unwrap_or_default());
@@ -553,39 +587,15 @@ impl ScalarIndex for BitmapIndex {
         mapping: &HashMap<u64, Option<u64>>,
         dest_store: &dyn IndexStore,
     ) -> Result<CreatedIndex> {
-        let mut state = HashMap::new();
-
-        for key in self.index_map.keys() {
-            let bitmap = self.load_bitmap(key, None).await?;
-            let remapped_bitmap =
-                RowAddrTreeMap::from_iter(bitmap.row_addrs().unwrap().filter_map(|addr| {
-                    let addr_as_u64 = u64::from(addr);
-                    mapping
-                        .get(&addr_as_u64)
-                        .copied()
-                        .unwrap_or(Some(addr_as_u64))
-                }));
-            state.insert(key.0.clone(), remapped_bitmap);
-        }
-
-        if !self.null_map.is_empty() {
-            let remapped_null =
-                RowAddrTreeMap::from_iter(self.null_map.row_addrs().unwrap().filter_map(|addr| {
-                    let addr_as_u64 = u64::from(addr);
-                    mapping
-                        .get(&addr_as_u64)
-                        .copied()
-                        .unwrap_or(Some(addr_as_u64))
-                }));
-            state.insert(ScalarValue::try_from(&self.value_type)?, remapped_null);
-        }
-
-        BitmapIndexPlugin::write_bitmap_index(state, dest_store, &self.value_type).await?;
+        let state = self.load_bitmap_index_state().await?;
+        let remapped_state = BitmapIndexPlugin::remap_bitmap_state(state, mapping);
+        BitmapIndexPlugin::write_bitmap_index(remapped_state, dest_store, &self.value_type).await?;
 
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::BitmapIndexDetails::default())
                 .unwrap(),
             index_version: BITMAP_INDEX_VERSION,
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -596,31 +606,18 @@ impl ScalarIndex for BitmapIndex {
         dest_store: &dyn IndexStore,
         _old_data_filter: Option<super::OldIndexDataFilter>,
     ) -> Result<CreatedIndex> {
-        let mut state = HashMap::new();
-
-        // Initialize builder with existing data
-        for key in self.index_map.keys() {
-            let bitmap = self.load_bitmap(key, None).await?;
-            state.insert(key.0.clone(), (*bitmap).clone());
-        }
-
-        if !self.null_map.is_empty() {
-            let ex_null = new_null_array(&self.value_type, 1);
-            let ex_null = ScalarValue::try_from_array(ex_null.as_ref(), 0)?;
-            state.insert(ex_null, (*self.null_map).clone());
-        }
-
-        BitmapIndexPlugin::do_train_bitmap_index(new_data, state, dest_store).await?;
+        BitmapIndexPlugin::streaming_build_and_write(new_data, Some(self), dest_store).await?;
 
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::BitmapIndexDetails::default())
                 .unwrap(),
             index_version: BITMAP_INDEX_VERSION,
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
     fn update_criteria(&self) -> UpdateCriteria {
-        UpdateCriteria::only_new_data(TrainingCriteria::new(TrainingOrdering::None).with_row_id())
+        UpdateCriteria::only_new_data(TrainingCriteria::new(TrainingOrdering::Values).with_row_id())
     }
 
     fn derive_index_params(&self) -> Result<ScalarIndexParams> {
@@ -628,6 +625,79 @@ impl ScalarIndex for BitmapIndex {
     }
 }
 
+/// Buffers serialized (key, bitmap) pairs and flushes them as record batches
+/// to the index file, respecting the MAX_BITMAP_ARRAY_LENGTH limit.
+struct BitmapBatchWriter {
+    file: Box<dyn super::IndexWriter>,
+    keys: Vec<ScalarValue>,
+    serialized: Vec<Vec<u8>>,
+    bytes: usize,
+    num_bitmaps: usize,
+}
+
+impl BitmapBatchWriter {
+    fn new(file: Box<dyn super::IndexWriter>) -> Self {
+        Self {
+            file,
+            keys: Vec::new(),
+            serialized: Vec::new(),
+            bytes: 0,
+            num_bitmaps: 0,
+        }
+    }
+
+    /// Serialize and buffer a single (key, bitmap) pair, flushing the current
+    /// batch to disk if adding it would exceed MAX_BITMAP_ARRAY_LENGTH.
+    async fn emit(&mut self, key: ScalarValue, bitmap: &RowAddrTreeMap) -> Result<()> {
+        let mut buf = Vec::new();
+        bitmap.serialize_into(&mut buf).unwrap();
+        let size = buf.len();
+
+        if self.bytes + size > MAX_BITMAP_ARRAY_LENGTH {
+            self.flush().await?;
+        }
+
+        self.keys.push(key);
+        self.serialized.push(buf);
+        self.bytes += size;
+        self.num_bitmaps += 1;
+        Ok(())
+    }
+
+    /// Write the current batch to disk.
+    async fn flush(&mut self) -> Result<()> {
+        if self.keys.is_empty() {
+            return Ok(());
+        }
+        let keys_array =
+            ScalarValue::iter_to_array(self.keys.drain(..).collect::<Vec<_>>().into_iter())
+                .unwrap();
+        let total_size: usize = self.serialized.iter().map(|b| b.len()).sum();
+        let mut binary_builder = BinaryBuilder::with_capacity(self.serialized.len(), total_size);
+        for b in self.serialized.drain(..) {
+            binary_builder.append_value(&b);
+        }
+        let bitmaps_array = Arc::new(binary_builder.finish()) as Arc<dyn Array>;
+        let batch = BitmapIndexPlugin::get_batch_from_arrays(keys_array, bitmaps_array)?;
+        self.file.write_record_batch(batch).await?;
+        self.bytes = 0;
+        Ok(())
+    }
+
+    /// Flush any remaining data, write index statistics, and finalize the file.
+    async fn finish(mut self) -> Result<()> {
+        self.flush().await?;
+        let stats_json = serde_json::to_string(&BitmapStatistics {
+            num_bitmaps: self.num_bitmaps,
+        })
+        .map_err(|e| Error::internal(format!("failed to serialize bitmap statistics: {e}")))?;
+        let mut metadata = HashMap::new();
+        metadata.insert(INDEX_STATS_METADATA_KEY.to_string(), stats_json);
+        self.file.finish_with_metadata(metadata).await?;
+        Ok(())
+    }
+}
+
 #[derive(Debug, Default)]
 pub struct BitmapIndexPlugin;
 
@@ -650,6 +720,24 @@ impl BitmapIndexPlugin {
         state: HashMap<ScalarValue, RowAddrTreeMap>,
         index_store: &dyn IndexStore,
         value_type: &DataType,
+    ) -> Result<()> {
+        Self::write_bitmap_index_with_extras(
+            state,
+            index_store,
+            value_type,
+            HashMap::new(),
+            Vec::new(),
+        )
+        .await
+    }
+
+    /// Writes a bitmap index and attaches extra metadata and global buffers.
+    pub(crate) async fn write_bitmap_index_with_extras(
+        state: HashMap<ScalarValue, RowAddrTreeMap>,
+        index_store: &dyn IndexStore,
+        value_type: &DataType,
+        mut metadata: HashMap<String, String>,
+        global_buffers: Vec<(String, Bytes)>,
     ) -> Result<()> {
         let num_bitmaps = state.len();
         let schema = Arc::new(Schema::new(vec![
@@ -661,6 +749,11 @@ impl BitmapIndexPlugin {
             .new_index_file(BITMAP_LOOKUP_NAME, schema)
             .await?;
 
+        for (metadata_key, data) in global_buffers {
+            let buffer_idx = bitmap_index_file.add_global_buffer(data).await?;
+            metadata.insert(metadata_key, buffer_idx.to_string());
+        }
+
         let mut cur_keys = Vec::new();
         let mut cur_bitmaps = Vec::new();
         let mut cur_bytes = 0;
@@ -707,7 +800,6 @@ impl BitmapIndexPlugin {
         // Finish file with metadata that allows lightweight statistics reads
         let stats_json = serde_json::to_string(&BitmapStatistics { num_bitmaps })
             .map_err(|e| Error::internal(format!("failed to serialize bitmap statistics: {e}")))?;
-        let mut metadata = HashMap::new();
         metadata.insert(INDEX_STATS_METADATA_KEY.to_string(), stats_json);
 
         bitmap_index_file.finish_with_metadata(metadata).await?;
@@ -715,11 +807,11 @@ impl BitmapIndexPlugin {
         Ok(())
     }
 
-    async fn do_train_bitmap_index(
+    /// Builds bitmap index state from a `(value, row_id)` stream without writing it.
+    pub(crate) async fn build_bitmap_index_state(
         mut data_source: SendableRecordBatchStream,
         mut state: HashMap<ScalarValue, RowAddrTreeMap>,
-        index_store: &dyn IndexStore,
-    ) -> Result<()> {
+    ) -> Result<(HashMap<ScalarValue, RowAddrTreeMap>, DataType)> {
         let value_type = data_source.schema().field(0).data_type().clone();
         while let Some(batch) = data_source.try_next().await? {
             let values = batch.column_by_name(VALUE_COLUMN_NAME).expect_ok()?;
@@ -735,17 +827,198 @@ impl BitmapIndexPlugin {
             }
         }
 
-        Self::write_bitmap_index(state, index_store, &value_type).await
+        Ok((state, value_type))
     }
 
     pub async fn train_bitmap_index(
         data: SendableRecordBatchStream,
         index_store: &dyn IndexStore,
     ) -> Result<()> {
-        // mapping from item to list of the row ids where it is present
-        let dictionary: HashMap<ScalarValue, RowAddrTreeMap> = HashMap::new();
+        Self::streaming_build_and_write(data, None, index_store).await
+    }
 
-        Self::do_train_bitmap_index(data, dictionary, index_store).await
+    /// Builds and writes a bitmap index in a streaming fashion from value-sorted
+    /// input. Only one value's bitmap is in memory at a time, reducing peak memory
+    /// from O(unique_values * avg_bitmap) to O(largest_single_bitmap).
+    ///
+    /// If `old_index` is provided, its existing bitmaps are merged with the new
+    /// data via a sorted merge-join (the old index_map is a BTreeMap, already
+    /// sorted by value).
+    async fn streaming_build_and_write(
+        mut data_source: SendableRecordBatchStream,
+        old_index: Option<&BitmapIndex>,
+        index_store: &dyn IndexStore,
+    ) -> Result<()> {
+        let value_type = data_source.schema().field(0).data_type().clone();
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("keys", value_type.clone(), true),
+            Field::new("bitmaps", DataType::Binary, true),
+        ]));
+
+        let index_file = index_store
+            .new_index_file(BITMAP_LOOKUP_NAME, schema)
+            .await?;
+        let mut writer = BitmapBatchWriter::new(index_file);
+
+        // Collect old index keys (already in memory as BTreeMap keys — this is
+        // just a Vec of references, not a copy of the bitmaps themselves).
+        let old_keys: Vec<OrderableScalarValue> = old_index
+            .map(|idx| idx.index_map.keys().cloned().collect())
+            .unwrap_or_default();
+        let mut old_pos: usize = 0;
+
+        // Current value being accumulated from the new data stream.
+        let mut current_key: Option<ScalarValue> = None;
+        let mut current_bitmap = RowAddrTreeMap::default();
+        // Track whether we emitted a null bitmap (old index stores nulls
+        // separately in null_map, not in index_map).
+        let mut emitted_null = false;
+
+        while let Some(batch) = data_source.try_next().await? {
+            let values = batch.column_by_name(VALUE_COLUMN_NAME).expect_ok()?;
+            let row_ids = batch.column_by_name(ROW_ID).expect_ok()?;
+            debug_assert_eq!(row_ids.data_type(), &DataType::UInt64);
+            let row_id_column = row_ids.as_any().downcast_ref::<UInt64Array>().unwrap();
+
+            for i in 0..values.len() {
+                let row_id = row_id_column.value(i);
+                let key = ScalarValue::try_from_array(values.as_ref(), i)?;
+
+                match &current_key {
+                    Some(cur) if *cur == key => {
+                        current_bitmap.insert(row_id);
+                    }
+                    _ => {
+                        // Value changed — flush the previous run.
+                        if let Some(prev_key) = current_key.take() {
+                            let mut prev_bitmap = std::mem::take(&mut current_bitmap);
+                            Self::finish_run(
+                                prev_key,
+                                &mut prev_bitmap,
+                                old_index,
+                                &old_keys,
+                                &mut old_pos,
+                                &mut emitted_null,
+                                &mut writer,
+                            )
+                            .await?;
+                        }
+                        current_key = Some(key);
+                        current_bitmap = RowAddrTreeMap::default();
+                        current_bitmap.insert(row_id);
+                    }
+                }
+            }
+        }
+
+        // Flush the last accumulated run from new data.
+        if let Some(last_key) = current_key.take() {
+            let mut last_bitmap = std::mem::take(&mut current_bitmap);
+            Self::finish_run(
+                last_key,
+                &mut last_bitmap,
+                old_index,
+                &old_keys,
+                &mut old_pos,
+                &mut emitted_null,
+                &mut writer,
+            )
+            .await?;
+        }
+
+        // Emit any remaining old-only entries.
+        if let Some(idx) = old_index {
+            while old_pos < old_keys.len() {
+                let old_bitmap = idx.load_bitmap(&old_keys[old_pos], None).await?;
+                writer
+                    .emit(old_keys[old_pos].0.clone(), &old_bitmap)
+                    .await?;
+                old_pos += 1;
+            }
+        }
+
+        // Emit old null bitmap if we didn't already merge it with new nulls.
+        if !emitted_null
+            && let Some(idx) = old_index
+            && !idx.null_map.is_empty()
+        {
+            let null_key = new_null_array(&value_type, 1);
+            let null_key = ScalarValue::try_from_array(null_key.as_ref(), 0)?;
+            writer.emit(null_key, &idx.null_map).await?;
+        }
+
+        writer.finish().await?;
+
+        Ok(())
+    }
+
+    /// Flush a completed value-run from the new data stream, emitting any
+    /// old-only entries that sort before it and merging the old bitmap if the
+    /// key exists in both old and new.
+    async fn finish_run(
+        key: ScalarValue,
+        bitmap: &mut RowAddrTreeMap,
+        old_index: Option<&BitmapIndex>,
+        old_keys: &[OrderableScalarValue],
+        old_pos: &mut usize,
+        emitted_null: &mut bool,
+        writer: &mut BitmapBatchWriter,
+    ) -> Result<()> {
+        if key.is_null() {
+            // Null values are stored separately in the old index's null_map.
+            if let Some(idx) = old_index
+                && !idx.null_map.is_empty()
+            {
+                *bitmap |= &*idx.null_map;
+            }
+            *emitted_null = true;
+            writer.emit(key, bitmap).await?;
+        } else if let Some(idx) = old_index {
+            let orderable = OrderableScalarValue(key.clone());
+
+            // Emit old-only entries that sort before this key.
+            while *old_pos < old_keys.len() && old_keys[*old_pos] < orderable {
+                let old_bitmap = idx.load_bitmap(&old_keys[*old_pos], None).await?;
+                writer
+                    .emit(old_keys[*old_pos].0.clone(), &old_bitmap)
+                    .await?;
+                *old_pos += 1;
+            }
+
+            // If the old index also has this key, merge its bitmap.
+            if *old_pos < old_keys.len() && old_keys[*old_pos] == orderable {
+                let old_bitmap = idx.load_bitmap(&old_keys[*old_pos], None).await?;
+                *bitmap |= &*old_bitmap;
+                *old_pos += 1;
+            }
+
+            writer.emit(key, bitmap).await?;
+        } else {
+            writer.emit(key, bitmap).await?;
+        }
+        Ok(())
+    }
+
+    /// Remaps every bitmap in a materialized bitmap-index state using row-id mappings.
+    pub(crate) fn remap_bitmap_state(
+        state: HashMap<ScalarValue, RowAddrTreeMap>,
+        mapping: &HashMap<u64, Option<u64>>,
+    ) -> HashMap<ScalarValue, RowAddrTreeMap> {
+        state
+            .into_iter()
+            .map(|(key, bitmap)| {
+                let remapped_bitmap =
+                    RowAddrTreeMap::from_iter(bitmap.row_addrs().unwrap().filter_map(|addr| {
+                        let addr_as_u64 = u64::from(addr);
+                        mapping
+                            .get(&addr_as_u64)
+                            .copied()
+                            .unwrap_or(Some(addr_as_u64))
+                    }));
+                (key, remapped_bitmap)
+            })
+            .collect()
     }
 }
 
@@ -766,7 +1039,7 @@ impl ScalarIndexPlugin for BitmapIndexPlugin {
             ));
         }
         Ok(Box::new(DefaultTrainingRequest::new(
-            TrainingCriteria::new(TrainingOrdering::None).with_row_id(),
+            TrainingCriteria::new(TrainingOrdering::Values).with_row_id(),
         )))
     }
 
@@ -805,6 +1078,7 @@ impl ScalarIndexPlugin for BitmapIndexPlugin {
             index_details: prost_types::Any::from_msg(&pbold::BitmapIndexDetails::default())
                 .unwrap(),
             index_version: BITMAP_INDEX_VERSION,
+            files: Some(index_store.list_files_with_sizes().await?),
         })
     }
 
@@ -837,12 +1111,28 @@ impl ScalarIndexPlugin for BitmapIndexPlugin {
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
     use super::*;
     use crate::metrics::NoOpMetricsCollector;
     use crate::scalar::lance_format::LanceIndexStore;
     use arrow_array::{RecordBatch, StringArray, UInt64Array, record_batch};
     use arrow_schema::{DataType, Field, Schema};
+
+    /// Sort a (value, row_id) RecordBatch by the value column so that unit tests
+    /// match the ordering the production scanner applies via TrainingOrdering::Values.
+    fn sort_batch_by_value(batch: &RecordBatch) -> RecordBatch {
+        use arrow::compute::SortOptions;
+        let values = batch.column(0);
+        let row_ids = batch.column(1);
+        let options = SortOptions {
+            descending: false,
+            nulls_first: true,
+        };
+        let indices = arrow::compute::sort_to_indices(values, Some(options), None).unwrap();
+        let sorted_values = arrow::compute::take(values.as_ref(), &indices, None).unwrap();
+        let sorted_row_ids = arrow::compute::take(row_ids.as_ref(), &indices, None).unwrap();
+        RecordBatch::try_new(batch.schema(), vec![sorted_values, sorted_row_ids]).unwrap()
+    }
     use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
     use futures::stream;
     use lance_core::utils::mask::RowSetOps;
@@ -882,6 +1172,7 @@ pub mod tests {
         )
         .unwrap();
 
+        let batch = sort_batch_by_value(&batch);
         let stream = stream::once(async move { Ok(batch) });
         let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream));
 
@@ -1156,6 +1447,7 @@ pub mod tests {
         )
         .unwrap();
 
+        let batch = sort_batch_by_value(&batch);
         let stream = stream::once(async move { Ok(batch) });
         let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream));
 
diff --git a/rust/lance-index/src/scalar/bloomfilter.rs b/rust/lance-index/src/scalar/bloomfilter.rs
index b72c8806fc4..7fcaa3aad82 100644
--- a/rust/lance-index/src/scalar/bloomfilter.rs
+++ b/rust/lance-index/src/scalar/bloomfilter.rs
@@ -464,6 +464,7 @@ impl ScalarIndex for BloomFilterIndex {
             index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default())
                 .unwrap(),
             index_version: BLOOMFILTER_INDEX_VERSION,
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -1070,6 +1071,7 @@ impl ScalarIndexPlugin for BloomFilterIndexPlugin {
             index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default())
                 .unwrap(),
             index_version: BLOOMFILTER_INDEX_VERSION,
+            files: Some(index_store.list_files_with_sizes().await?),
         })
     }
 
diff --git a/rust/lance-index/src/scalar/bloomfilter/sbbf.rs b/rust/lance-index/src/scalar/bloomfilter/sbbf.rs
index 50574768050..cbb4eb76b12 100644
--- a/rust/lance-index/src/scalar/bloomfilter/sbbf.rs
+++ b/rust/lance-index/src/scalar/bloomfilter/sbbf.rs
@@ -316,7 +316,6 @@ impl Sbbf {
     }
 
     /// Write the bitset in serialized form to the writer
-    #[allow(dead_code)]
     pub fn write_bitset<W: Write>(&self, mut writer: W) -> Result<()> {
         for block in &self.blocks {
             writer
@@ -336,19 +335,16 @@ impl Sbbf {
     }
 
     /// Get the number of blocks in this filter
-    #[allow(dead_code)]
     pub fn num_blocks(&self) -> usize {
         self.blocks.len()
     }
 
     /// Get the size in bytes of this filter
-    #[allow(dead_code)]
     pub fn size_bytes(&self) -> usize {
         self.blocks.len() * 32
     }
 
     /// Return the total in memory size of this bloom filter in bytes
-    #[allow(dead_code)]
     pub fn estimated_memory_size(&self) -> usize {
         self.blocks.capacity() * std::mem::size_of::<Block>()
     }
@@ -456,7 +452,6 @@ impl SbbfBuilder {
     }
 
     /// Set the number of bytes directly
-    #[allow(dead_code)]
     pub fn log2_num_bytes(mut self, log2_num_bytes: u8) -> Self {
         self.log2_num_bytes = Some(log2_num_bytes);
         self
diff --git a/rust/lance-index/src/scalar/btree.rs b/rust/lance-index/src/scalar/btree.rs
index 9930c96793e..1df740e815f 100644
--- a/rust/lance-index/src/scalar/btree.rs
+++ b/rust/lance-index/src/scalar/btree.rs
@@ -13,10 +13,12 @@ use std::{
 use super::{
     AnyQuery, BuiltinIndexType, IndexReader, IndexStore, IndexWriter, MetricsCollector,
     OldIndexDataFilter, SargableQuery, ScalarIndex, ScalarIndexParams, SearchResult,
+    compute_next_prefix,
 };
 use crate::{Index, IndexType};
 use crate::{
     frag_reuse::FragReuseIndex,
+    progress::{IndexBuildProgress, noop_progress},
     scalar::{
         CreatedIndex, UpdateCriteria,
         expression::{SargableQueryParser, ScalarQueryParser},
@@ -989,6 +991,10 @@ impl CacheKey for BTreePageKey {
     fn key(&self) -> std::borrow::Cow<'_, str> {
         format!("page-{}", self.page_number).into()
     }
+
+    fn type_name() -> &'static str {
+        "BTreePage"
+    }
 }
 
 /// Note: this is very similar to the IVF index except we store the IVF part in a btree
@@ -1501,6 +1507,39 @@ impl ScalarIndex for BTreeIndex {
                 ));
             }
             SargableQuery::IsNull() => self.page_lookup.pages_null(),
+            SargableQuery::LikePrefix(prefix) => {
+                // Convert LikePrefix to a range query: [prefix, next_prefix)
+                match prefix {
+                    ScalarValue::Utf8(Some(s)) => {
+                        let start = Bound::Included(OrderableScalarValue(prefix.clone()));
+                        let end = match compute_next_prefix(s) {
+                            Some(next) => {
+                                Bound::Excluded(OrderableScalarValue(ScalarValue::Utf8(Some(next))))
+                            }
+                            None => Bound::Unbounded,
+                        };
+                        self.page_lookup
+                            .pages_between((start.as_ref(), end.as_ref()))
+                    }
+                    ScalarValue::LargeUtf8(Some(s)) => {
+                        let start = Bound::Included(OrderableScalarValue(prefix.clone()));
+                        let end = match compute_next_prefix(s) {
+                            Some(next) => Bound::Excluded(OrderableScalarValue(
+                                ScalarValue::LargeUtf8(Some(next)),
+                            )),
+                            None => Bound::Unbounded,
+                        };
+                        self.page_lookup
+                            .pages_between((start.as_ref(), end.as_ref()))
+                    }
+                    _ => {
+                        // Conservative: return all pages for non-string types
+                        // This is consistent with ZoneMap behavior
+                        self.page_lookup
+                            .pages_between((Bound::Unbounded, Bound::Unbounded))
+                    }
+                }
+            }
         };
 
         // For non-IsNull queries, also include null pages so that null row IDs
@@ -1621,13 +1660,21 @@ impl ScalarIndex for BTreeIndex {
             let lookup_files = (0..num_parts)
                 .map(|part_id| part_lookup_file_path((part_id as u64) << 32))
                 .collect::<Vec<_>>();
-            merge_metadata_files(dest_store, &page_files, &lookup_files, None).await?;
+            merge_metadata_files(
+                dest_store,
+                &page_files,
+                &lookup_files,
+                None,
+                noop_progress(),
+            )
+            .await?;
         }
 
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::BTreeIndexDetails::default())
                 .unwrap(),
             index_version: BTREE_INDEX_VERSION,
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -1648,6 +1695,7 @@ impl ScalarIndex for BTreeIndex {
             index_details: prost_types::Any::from_msg(&pbold::BTreeIndexDetails::default())
                 .unwrap(),
             index_version: BTREE_INDEX_VERSION,
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -1885,6 +1933,7 @@ pub async fn merge_index_files(
     index_dir: &Path,
     store: Arc<dyn IndexStore>,
     batch_readhead: Option<usize>,
+    progress: Arc<dyn IndexBuildProgress>,
 ) -> Result<()> {
     // List all partition page / lookup files in the index directory
     let (part_page_files, part_lookup_files) =
@@ -1894,6 +1943,7 @@ pub async fn merge_index_files(
         &part_page_files,
         &part_lookup_files,
         batch_readhead,
+        progress,
     )
     .await
 }
@@ -1949,6 +1999,7 @@ async fn merge_metadata_files(
     part_page_files: &[String],
     part_lookup_files: &[String],
     batch_readhead: Option<usize>,
+    progress: Arc<dyn IndexBuildProgress>,
 ) -> Result<()> {
     if part_lookup_files.is_empty() || part_page_files.is_empty() {
         return Err(Error::internal(
@@ -2022,6 +2073,7 @@ async fn merge_metadata_files(
             metadata,
             batch_size,
             batch_readhead,
+            progress,
         )
         .await
     } else {
@@ -2034,6 +2086,7 @@ async fn merge_metadata_files(
             metadata,
             batch_size,
             batch_readhead,
+            progress,
         )
         .await
     }
@@ -2075,6 +2128,7 @@ async fn merge_range_partitioned_lookups(
     mut metadata: HashMap<String, String>,
     batch_size: u64,
     batch_readhead: Option<usize>,
+    progress: Arc<dyn IndexBuildProgress>,
 ) -> Result<()> {
     let sorted_part_lookup_files = sort_files_by_partition_id(part_lookup_files)?;
     let mut lookup_file = store
@@ -2085,7 +2139,15 @@ async fn merge_range_partitioned_lookups(
     let mut pages_per_file: Vec<(u64, u32)> = Vec::with_capacity(sorted_part_lookup_files.len());
     let mut num_pages_written = 0u32;
 
-    for (part_id, part_lookup_file) in sorted_part_lookup_files {
+    progress
+        .stage_start(
+            "merge_lookups",
+            Some(sorted_part_lookup_files.len() as u64),
+            "files",
+        )
+        .await?;
+
+    for (idx, (part_id, part_lookup_file)) in sorted_part_lookup_files.into_iter().enumerate() {
         let lookup_reader = store.open_index_file(&part_lookup_file).await?;
         let reader_stream = IndexReaderStream::new(lookup_reader.clone(), batch_size).await;
         let mut stream = reader_stream.buffered(batch_readhead.unwrap_or(1)).boxed();
@@ -2096,6 +2158,9 @@ async fn merge_range_partitioned_lookups(
         }
         pages_per_file.push((part_id, lookup_reader.num_rows() as u32));
         num_pages_written += lookup_reader.num_rows() as u32;
+        progress
+            .stage_progress("merge_lookups", idx as u64 + 1)
+            .await?;
     }
 
     metadata.insert(RANGE_PARTITIONED_META_KEY.to_string(), "true".to_string());
@@ -2105,6 +2170,7 @@ async fn merge_range_partitioned_lookups(
     );
 
     lookup_file.finish_with_metadata(metadata).await?;
+    progress.stage_complete("merge_lookups").await?;
 
     // In this mode, we only clean up lookup files, and page files are untouched.
     cleanup_partition_files(store, part_lookup_files, &[]).await;
@@ -2126,6 +2192,7 @@ async fn merge_pages_and_lookups(
     metadata: HashMap<String, String>,
     batch_size: u64,
     batch_readhead: Option<usize>,
+    progress: Arc<dyn IndexBuildProgress>,
 ) -> Result<()> {
     // Create a new global page file
     let partition_id = extract_partition_id(part_lookup_files[0].as_str())?;
@@ -2137,7 +2204,7 @@ async fn merge_pages_and_lookups(
     let mut page_file = store
         .new_index_file(BTREE_PAGES_NAME, arrow_schema.clone())
         .await?;
-
+    progress.stage_start("merge_pages", None, "pages").await?;
     let lookup_entries = merge_pages(
         part_lookup_files,
         page_files_map,
@@ -2146,9 +2213,11 @@ async fn merge_pages_and_lookups(
         &mut page_file,
         arrow_schema.clone(),
         batch_readhead,
+        progress.clone(),
     )
     .await?;
     page_file.finish().await?;
+    progress.stage_complete("merge_pages").await?;
 
     let lookup_batch = RecordBatch::try_new(
         lookup_schema.clone(),
@@ -2168,8 +2237,13 @@ async fn merge_pages_and_lookups(
     let mut lookup_file = store
         .new_index_file(BTREE_LOOKUP_NAME, lookup_schema)
         .await?;
+    progress
+        .stage_start("write_lookup_file", Some(1), "files")
+        .await?;
     lookup_file.write_record_batch(lookup_batch).await?;
     lookup_file.finish_with_metadata(metadata).await?;
+    progress.stage_progress("write_lookup_file", 1).await?;
+    progress.stage_complete("write_lookup_file").await?;
 
     // After successfully writing the merged files, delete all partition files
     // Only perform deletion after files are successfully written, ensuring debug information is not lost in case of failure
@@ -2200,6 +2274,7 @@ fn add_offset_to_page_idx(batch: &RecordBatch, offset: u32) -> Result<RecordBatc
 
 /// Merge pages using Datafusion's SortPreservingMergeExec
 /// which implements a K-way merge algorithm with fixed-size output batches
+#[allow(clippy::too_many_arguments)]
 async fn merge_pages(
     part_lookup_files: &[String],
     page_files_map: &HashMap<u64, &String>,
@@ -2208,6 +2283,7 @@ async fn merge_pages(
     page_file: &mut Box<dyn IndexWriter>,
     arrow_schema: Arc<Schema>,
     batch_readhead: Option<usize>,
+    progress: Arc<dyn IndexBuildProgress>,
 ) -> Result<Vec<(ScalarValue, ScalarValue, u32, u32)>> {
     let mut lookup_entries = Vec::new();
     let mut page_idx = 0u32;
@@ -2291,6 +2367,9 @@ async fn merge_pages(
 
         lookup_entries.push((min_val, max_val, null_count, page_idx));
         page_idx += 1;
+        progress
+            .stage_progress("merge_pages", page_idx as u64)
+            .await?;
     }
 
     Ok(lookup_entries)
@@ -2586,6 +2665,7 @@ impl ScalarIndexPlugin for BTreeIndexPlugin {
             index_details: prost_types::Any::from_msg(&pbold::BTreeIndexDetails::default())
                 .unwrap(),
             index_version: BTREE_INDEX_VERSION,
+            files: Some(index_store.list_files_with_sizes().await?),
         })
     }
 
@@ -2625,6 +2705,7 @@ mod tests {
     use object_store::path::Path;
 
     use crate::metrics::LocalMetricsCollector;
+    use crate::progress::{IndexBuildProgress, noop_progress};
     use crate::{
         metrics::NoOpMetricsCollector,
         scalar::{
@@ -2638,6 +2719,12 @@ mod tests {
         DEFAULT_BTREE_BATCH_SIZE, OrderableScalarValue, part_lookup_file_path,
         part_page_data_file_path, train_btree_index,
     };
+
+    lance_testing::define_stage_event_progress!(
+        RecordingProgress,
+        IndexBuildProgress,
+        lance_core::Result<()>
+    );
     #[test]
     fn test_scalar_value_size() {
         let size_of_i32 = OrderableScalarValue(ScalarValue::Int32(Some(0))).deep_size_of();
@@ -2810,6 +2897,203 @@ mod tests {
         assert_eq!(metrics.parts_loaded.load(Ordering::Relaxed), 1);
     }
 
+    #[tokio::test]
+    async fn test_like_prefix_search() {
+        use arrow::datatypes::DataType;
+        use arrow_array::StringArray;
+
+        let tmpdir = TempObjDir::default();
+        let test_store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // Create string data with various prefixes
+        let values = vec![
+            "apple",
+            "app",
+            "application",
+            "banana",
+            "band",
+            "test_ns$table1",
+            "test_ns$table2",
+            "test_ns2$table1",
+            "test",
+            "testing",
+        ];
+        let row_ids: Vec<u64> = (0..values.len() as u64).collect();
+
+        let schema = Arc::new(arrow::datatypes::Schema::new(vec![
+            arrow::datatypes::Field::new("value", DataType::Utf8, false),
+            arrow::datatypes::Field::new("_rowid", DataType::UInt64, false),
+        ]));
+
+        let batch = arrow::record_batch::RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(values.clone())),
+                Arc::new(arrow_array::UInt64Array::from(row_ids)),
+            ],
+        )
+        .unwrap();
+
+        let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema,
+            stream::once(async { Ok(batch) }),
+        ));
+
+        train_btree_index(stream, test_store.as_ref(), 100, None, None)
+            .await
+            .unwrap();
+
+        let index = BTreeIndex::load(test_store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        // Test LikePrefix for "app" - should match "apple", "app", "application" (row ids 0, 1, 2)
+        let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("app".to_string())));
+        let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
+
+        match &result {
+            SearchResult::Exact(row_ids) => {
+                let ids: Vec<u64> = row_ids
+                    .true_rows()
+                    .row_addrs()
+                    .unwrap()
+                    .map(u64::from)
+                    .collect();
+                assert!(ids.contains(&0), "Should contain row 0 (apple)");
+                assert!(ids.contains(&1), "Should contain row 1 (app)");
+                assert!(ids.contains(&2), "Should contain row 2 (application)");
+                assert!(!ids.contains(&3), "Should not contain row 3 (banana)");
+            }
+            _ => panic!("Expected Exact result"),
+        }
+
+        // Test LikePrefix for "test_ns$" - should match "test_ns$table1", "test_ns$table2" (row ids 5, 6)
+        let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("test_ns$".to_string())));
+        let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
+
+        match &result {
+            SearchResult::Exact(row_ids) => {
+                let ids: Vec<u64> = row_ids
+                    .true_rows()
+                    .row_addrs()
+                    .unwrap()
+                    .map(u64::from)
+                    .collect();
+                assert!(ids.contains(&5), "Should contain row 5 (test_ns$table1)");
+                assert!(ids.contains(&6), "Should contain row 6 (test_ns$table2)");
+                assert!(
+                    !ids.contains(&7),
+                    "Should not contain row 7 (test_ns2$table1)"
+                );
+            }
+            _ => panic!("Expected Exact result"),
+        }
+
+        // Test LikePrefix for "test" - should match "test", "testing", "test_ns$table1", "test_ns$table2", "test_ns2$table1"
+        let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("test".to_string())));
+        let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
+
+        match &result {
+            SearchResult::Exact(row_ids) => {
+                let ids: Vec<u64> = row_ids
+                    .true_rows()
+                    .row_addrs()
+                    .unwrap()
+                    .map(u64::from)
+                    .collect();
+                assert!(
+                    ids.contains(&5),
+                    "Should contain row 5 (test_ns$table1): {:?}",
+                    ids
+                );
+                assert!(
+                    ids.contains(&6),
+                    "Should contain row 6 (test_ns$table2): {:?}",
+                    ids
+                );
+                assert!(
+                    ids.contains(&7),
+                    "Should contain row 7 (test_ns2$table1): {:?}",
+                    ids
+                );
+                assert!(ids.contains(&8), "Should contain row 8 (test): {:?}", ids);
+                assert!(
+                    ids.contains(&9),
+                    "Should contain row 9 (testing): {:?}",
+                    ids
+                );
+            }
+            _ => panic!("Expected Exact result"),
+        }
+    }
+
+    #[tokio::test]
+    async fn test_like_prefix_search_large_utf8() {
+        use arrow::datatypes::DataType;
+        use arrow_array::LargeStringArray;
+
+        let tmpdir = TempObjDir::default();
+        let test_store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let values = vec!["apple", "app", "application", "banana"];
+        let row_ids: Vec<u64> = (0..values.len() as u64).collect();
+
+        let schema = Arc::new(arrow::datatypes::Schema::new(vec![
+            arrow::datatypes::Field::new("value", DataType::LargeUtf8, false),
+            arrow::datatypes::Field::new("_rowid", DataType::UInt64, false),
+        ]));
+
+        let batch = arrow::record_batch::RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(LargeStringArray::from(values)),
+                Arc::new(arrow_array::UInt64Array::from(row_ids)),
+            ],
+        )
+        .unwrap();
+
+        let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema,
+            stream::once(async { Ok(batch) }),
+        ));
+
+        train_btree_index(stream, test_store.as_ref(), 100, None, None)
+            .await
+            .unwrap();
+
+        let index = BTreeIndex::load(test_store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        // Test LikePrefix with LargeUtf8
+        let query = SargableQuery::LikePrefix(ScalarValue::LargeUtf8(Some("app".to_string())));
+        let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
+
+        match &result {
+            SearchResult::Exact(row_ids) => {
+                let ids: Vec<u64> = row_ids
+                    .true_rows()
+                    .row_addrs()
+                    .unwrap()
+                    .map(u64::from)
+                    .collect();
+                assert!(ids.contains(&0), "Should contain row 0 (apple)");
+                assert!(ids.contains(&1), "Should contain row 1 (app)");
+                assert!(ids.contains(&2), "Should contain row 2 (application)");
+                assert!(!ids.contains(&3), "Should not contain row 3 (banana)");
+            }
+            _ => panic!("Expected Exact result"),
+        }
+    }
+
     #[tokio::test]
     async fn test_fragment_btree_index_consistency() {
         // Setup stores for both indexes
@@ -2906,15 +3190,50 @@ mod tests {
             part_lookup_file_path(2 << 32),
         ];
 
+        let progress = Arc::new(RecordingProgress::default());
         super::merge_metadata_files(
             fragment_store.as_ref(),
             &part_page_files,
             &part_lookup_files,
             Option::from(1usize),
+            progress.clone(),
         )
         .await
         .unwrap();
 
+        let tags = progress
+            .recorded_events()
+            .iter()
+            .map(|(kind, stage, _)| format!("{kind}:{stage}"))
+            .collect::<Vec<_>>();
+        let merge_start = tags
+            .iter()
+            .position(|e| e == "start:merge_pages")
+            .expect("missing merge_pages start");
+        let merge_complete = tags
+            .iter()
+            .position(|e| e == "complete:merge_pages")
+            .expect("missing merge_pages complete");
+        let lookup_start = tags
+            .iter()
+            .position(|e| e == "start:write_lookup_file")
+            .expect("missing write_lookup_file start");
+        let lookup_complete = tags
+            .iter()
+            .position(|e| e == "complete:write_lookup_file")
+            .expect("missing write_lookup_file complete");
+        assert!(merge_start < merge_complete);
+        assert!(merge_complete < lookup_start);
+        assert!(lookup_start < lookup_complete);
+        assert!(
+            tags.iter().any(|e| e == "progress:merge_pages"),
+            "expected merge_pages progress callbacks"
+        );
+        assert!(
+            tags.iter().any(|e| e == "progress:write_lookup_file"),
+            "expected write_lookup_file progress callbacks"
+        );
+
         // Load both indexes
         let full_index = BTreeIndex::load(full_store.clone(), None, &LanceCache::no_cache())
             .await
@@ -3120,6 +3439,7 @@ mod tests {
             &part_page_files,
             &part_lookup_files,
             Option::from(1usize),
+            noop_progress(),
         )
         .await
         .unwrap();
@@ -3687,6 +4007,7 @@ mod tests {
             &part_page_files,
             &part_lookup_files,
             Option::from(1usize),
+            noop_progress(),
         )
         .await
         .unwrap();
@@ -4004,6 +4325,7 @@ mod tests {
             &part_page_files,
             &part_lookup_files,
             Option::from(1usize),
+            noop_progress(),
         )
         .await
         .unwrap();
diff --git a/rust/lance-index/src/scalar/expression.rs b/rust/lance-index/src/scalar/expression.rs
index f452ee3858e..e6d04f031f0 100644
--- a/rust/lance-index/src/scalar/expression.rs
+++ b/rust/lance-index/src/scalar/expression.rs
@@ -14,7 +14,7 @@ use async_trait::async_trait;
 use datafusion_common::ScalarValue;
 use datafusion_expr::{
     Between, BinaryExpr, Expr, Operator, ReturnFieldArgs, ScalarUDF,
-    expr::{InList, ScalarFunction},
+    expr::{InList, Like, ScalarFunction},
 };
 use tokio::try_join;
 
@@ -112,6 +112,29 @@ pub trait ScalarQueryParser: std::fmt::Debug + Send + Sync {
         args: &[Expr],
     ) -> Option<IndexedExpression>;
 
+    /// Visit a LIKE expression
+    ///
+    /// Returns an IndexedExpression if the index can accelerate LIKE expressions.
+    /// For prefix patterns (e.g., "foo%"):
+    /// - ZoneMaps prune zones based on min/max statistics
+    /// - BTrees use range query conversion `[prefix, next_prefix)`
+    ///
+    /// For patterns with wildcards in the middle (e.g., "foo%bar%"), the leading prefix
+    /// can still be used for pruning, with the full pattern as a refine expression.
+    ///
+    /// # Arguments
+    /// * `column` - The column name
+    /// * `like` - The full LIKE expression (for constructing refine_expr if needed)
+    /// * `pattern` - The LIKE pattern as ScalarValue (e.g., "foo%")
+    fn visit_like(
+        &self,
+        _column: &str,
+        _like: &Like,
+        _pattern: &ScalarValue,
+    ) -> Option<IndexedExpression> {
+        None
+    }
+
     /// Visits a potential reference to a column
     ///
     /// This function is a little different from the other visitors.  It is used to test if a potential
@@ -212,6 +235,16 @@ impl ScalarQueryParser for MultiQueryParser {
             .iter()
             .find_map(|parser| parser.visit_scalar_function(column, data_type, func, args))
     }
+    fn visit_like(
+        &self,
+        column: &str,
+        like: &Like,
+        pattern: &ScalarValue,
+    ) -> Option<IndexedExpression> {
+        self.parsers
+            .iter()
+            .find_map(|parser| parser.visit_like(column, like, pattern))
+    }
     /// TODO(low-priority): This is maybe not quite right.  We should filter down the list of parsers based
     /// on those that consider the reference valid.  Instead what we are doing is checking all parsers if any one
     /// parser considers the reference valid.
@@ -340,13 +373,201 @@ impl ScalarQueryParser for SargableQueryParser {
 
     fn visit_scalar_function(
         &self,
-        _: &str,
-        _: &DataType,
-        _: &ScalarUDF,
-        _: &[Expr],
+        column: &str,
+        _data_type: &DataType,
+        func: &ScalarUDF,
+        args: &[Expr],
     ) -> Option<IndexedExpression> {
+        // Handle starts_with(col, 'prefix') -> convert to LikePrefix query
+        if func.name() == "starts_with" && args.len() == 2 {
+            // Extract the prefix from the second argument
+            let prefix = match &args[1] {
+                Expr::Literal(ScalarValue::Utf8(Some(s)), _) => ScalarValue::Utf8(Some(s.clone())),
+                Expr::Literal(ScalarValue::LargeUtf8(Some(s)), _) => {
+                    ScalarValue::LargeUtf8(Some(s.clone()))
+                }
+                _ => return None,
+            };
+
+            let query = SargableQuery::LikePrefix(prefix);
+            return Some(IndexedExpression::index_query_with_recheck(
+                column.to_string(),
+                self.index_name.clone(),
+                Arc::new(query),
+                self.needs_recheck,
+            ));
+        }
+
         None
     }
+
+    fn visit_like(
+        &self,
+        column: &str,
+        like: &Like,
+        pattern: &ScalarValue,
+    ) -> Option<IndexedExpression> {
+        // Case-insensitive LIKE (ILIKE) cannot be efficiently pruned with zone maps
+        if like.case_insensitive {
+            return None;
+        }
+
+        // Extract the pattern string
+        let pattern_str = match pattern {
+            ScalarValue::Utf8(Some(s)) => s.as_str(),
+            ScalarValue::LargeUtf8(Some(s)) => s.as_str(),
+            _ => return None,
+        };
+
+        // Try to extract a prefix from the LIKE pattern
+        let (prefix, needs_refine) = extract_like_leading_prefix(pattern_str, like.escape_char)?;
+
+        // Create the prefix ScalarValue with the same type as the pattern
+        let prefix_value = match pattern {
+            ScalarValue::Utf8(_) => ScalarValue::Utf8(Some(prefix)),
+            ScalarValue::LargeUtf8(_) => ScalarValue::LargeUtf8(Some(prefix)),
+            _ => return None,
+        };
+
+        let query = SargableQuery::LikePrefix(prefix_value);
+        let scalar_query = Some(ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: column.to_string(),
+            index_name: self.index_name.clone(),
+            query: Arc::new(query),
+            needs_recheck: self.needs_recheck,
+        }));
+
+        // If the pattern has wildcards beyond simple prefix, add refine expression
+        let refine_expr = if needs_refine {
+            Some(Expr::Like(like.clone()))
+        } else {
+            None
+        };
+
+        Some(IndexedExpression {
+            scalar_query,
+            refine_expr,
+        })
+    }
+}
+
+/// Extract the leading literal prefix from a LIKE pattern.
+///
+/// Returns `Some((prefix, needs_refine))` where:
+/// - `prefix` is the leading literal portion before any wildcards
+/// - `needs_refine` is true if the pattern has wildcards beyond a simple trailing `%`
+///
+/// Returns `None` if the pattern starts with a wildcard (no leading literal).
+///
+/// Examples:
+/// - "foo%" -> Some(("foo", false)) - pure prefix, no recheck needed
+/// - "foo%bar%" -> Some(("foo", true)) - can use prefix for pruning, needs recheck
+/// - "foo_bar%" -> Some(("foo", true)) - _ is a wildcard, needs recheck
+/// - "foo\%bar%" with escape '\' -> Some(("foo%bar", false)) - escaped %, pure prefix
+/// - "%foo" -> None - starts with wildcard, cannot prune
+/// - "foo" -> None - no wildcard at all, use equality instead
+fn extract_like_leading_prefix(pattern: &str, escape_char: Option<char>) -> Option<(String, bool)> {
+    let chars: Vec<char> = pattern.chars().collect();
+    let len = chars.len();
+
+    if len == 0 {
+        return None;
+    }
+
+    // DataFusion's starts_with simplification escapes special characters with backslash
+    // but doesn't set escape_char. Use backslash as default escape character.
+    // Pattern: starts_with(col, 'test_ns$') -> col LIKE 'test\_ns$%' (escape_char: None)
+    // See: https://github.com/apache/datafusion/issues/XXXX
+    let effective_escape_char = escape_char.or(Some('\\'));
+
+    // Helper to check if a character at position i is escaped
+    let is_escaped = |i: usize| -> bool {
+        if let Some(esc) = effective_escape_char {
+            if i > 0 && chars[i - 1] == esc {
+                // Check if the escape char itself is escaped
+                if i >= 2 && chars[i - 2] == esc {
+                    false // Escape was escaped, so this char is NOT escaped
+                } else {
+                    true // This char is escaped
+                }
+            } else {
+                false
+            }
+        } else {
+            // No escape character defined - nothing can be escaped
+            false
+        }
+    };
+
+    // Pattern must contain at least one unescaped wildcard
+    let has_wildcard = chars.iter().enumerate().any(|(i, &c)| {
+        if c != '%' && c != '_' {
+            return false;
+        }
+        !is_escaped(i)
+    });
+
+    if !has_wildcard {
+        return None; // No wildcards, should use equality
+    }
+
+    // Check if pattern starts with an unescaped wildcard
+    if chars[0] == '%' || chars[0] == '_' {
+        return None; // Starts with wildcard, cannot prune
+    }
+
+    // Extract the leading literal prefix (everything before first unescaped wildcard)
+    let mut prefix = String::new();
+    let mut i = 0;
+    let mut found_wildcard = false;
+
+    while i < len {
+        let c = chars[i];
+
+        // Check for escape character (using effective escape char which may be inferred)
+        if let Some(esc) = effective_escape_char
+            && c == esc
+            && i + 1 < len
+        {
+            let next = chars[i + 1];
+            if next == '%' || next == '_' || next == esc {
+                // Escaped character - add the literal character
+                prefix.push(next);
+                i += 2;
+                continue;
+            }
+        }
+
+        // Check for unescaped wildcard
+        if c == '%' || c == '_' {
+            found_wildcard = true;
+            break;
+        }
+
+        prefix.push(c);
+        i += 1;
+    }
+
+    if prefix.is_empty() {
+        return None;
+    }
+
+    // Check if pattern is just a simple prefix (ends with single % and nothing after)
+    let needs_refine = if found_wildcard && i < len {
+        // Check if we're at a % wildcard
+        if chars[i] == '%' && i + 1 == len {
+            // Pattern is "prefix%" - pure prefix match, no refine needed
+            false
+        } else {
+            // Pattern has more after first wildcard, or has _ wildcard
+            true
+        }
+    } else {
+        // No wildcard found (shouldn't happen due to earlier check)
+        false
+    };
+
+    Some((prefix, needs_refine))
 }
 
 /// A parser for bloom filter indices that only support equals, is_null, and is_in operations
@@ -1660,6 +1881,21 @@ fn visit_scalar_fn(
     query_parser.visit_scalar_function(&col, &data_type, &scalar_fn.func, &scalar_fn.args)
 }
 
+fn visit_like_expr(
+    like: &Like,
+    index_info: &dyn IndexInformationProvider,
+) -> Option<IndexedExpression> {
+    let (column, _, query_parser) = maybe_indexed_column(&like.expr, index_info)?;
+
+    // Extract the pattern as a ScalarValue
+    let pattern = match like.pattern.as_ref() {
+        Expr::Literal(scalar, _) => scalar.clone(),
+        _ => return None,
+    };
+
+    query_parser.visit_like(&column, like, &pattern)
+}
+
 fn visit_node(
     expr: &Expr,
     index_info: &dyn IndexInformationProvider,
@@ -1683,6 +1919,14 @@ fn visit_node(
         Expr::Not(expr) => visit_not(expr.as_ref(), index_info, depth),
         Expr::BinaryExpr(binary_expr) => visit_binary_expr(binary_expr, index_info, depth),
         Expr::ScalarFunction(scalar_fn) => Ok(visit_scalar_fn(scalar_fn, index_info)),
+        Expr::Like(like) => {
+            if like.negated {
+                // NOT LIKE cannot be efficiently pruned with zone maps
+                Ok(None)
+            } else {
+                Ok(visit_like_expr(like, index_info))
+            }
+        }
         _ => Ok(None),
     }
 }
@@ -2450,4 +2694,308 @@ mod tests {
             NullableIndexExprResult::AtLeast(_)
         ));
     }
+
+    #[test]
+    fn test_extract_like_leading_prefix() {
+        // Simple prefix patterns (no recheck needed)
+        assert_eq!(
+            extract_like_leading_prefix("foo%", None),
+            Some(("foo".to_string(), false))
+        );
+        assert_eq!(
+            extract_like_leading_prefix("abc%", None),
+            Some(("abc".to_string(), false))
+        );
+
+        // Patterns with wildcards in the middle (need recheck)
+        assert_eq!(
+            extract_like_leading_prefix("foo%bar%", None),
+            Some(("foo".to_string(), true))
+        );
+        assert_eq!(
+            extract_like_leading_prefix("foo_bar%", None),
+            Some(("foo".to_string(), true))
+        );
+        assert_eq!(
+            extract_like_leading_prefix("foo%bar", None),
+            Some(("foo".to_string(), true))
+        );
+        assert_eq!(
+            extract_like_leading_prefix("foo_", None),
+            Some(("foo".to_string(), true))
+        );
+
+        // Not prefix patterns (starts with wildcard)
+        assert_eq!(extract_like_leading_prefix("%foo", None), None);
+        assert_eq!(extract_like_leading_prefix("_foo%", None), None);
+        assert_eq!(extract_like_leading_prefix("%", None), None);
+
+        // No wildcard at all (should use equality)
+        assert_eq!(extract_like_leading_prefix("foo", None), None);
+
+        // With escape character
+        assert_eq!(
+            extract_like_leading_prefix(r"foo\%bar%", Some('\\')),
+            Some(("foo%bar".to_string(), false))
+        );
+        assert_eq!(
+            extract_like_leading_prefix(r"foo\_bar%", Some('\\')),
+            Some(("foo_bar".to_string(), false))
+        );
+        assert_eq!(
+            extract_like_leading_prefix(r"foo\\bar%", Some('\\')),
+            Some(("foo\\bar".to_string(), false))
+        );
+
+        // Escaped trailing % is not a wildcard (no wildcards)
+        assert_eq!(extract_like_leading_prefix(r"foo\%", Some('\\')), None);
+
+        // With backslash as default escape (for DataFusion starts_with compatibility):
+        // "foo\%" means escaped %, no wildcard -> None (should use equality)
+        assert_eq!(extract_like_leading_prefix(r"foo\%", None), None);
+        // "foo\bar%" - \b is not a valid escape sequence, so \ and b are literals, % is wildcard
+        assert_eq!(
+            extract_like_leading_prefix(r"foo\bar%", None),
+            Some(("foo\\bar".to_string(), false))
+        );
+
+        // Empty pattern
+        assert_eq!(extract_like_leading_prefix("", None), None);
+
+        // Mixed escaped and unescaped
+        assert_eq!(
+            extract_like_leading_prefix(r"foo\%bar%baz%", Some('\\')),
+            Some(("foo%bar".to_string(), true))
+        );
+    }
+
+    #[test]
+    fn test_like_expression_parsing() {
+        // Test that LIKE expressions are parsed correctly with refine_expr for complex patterns
+
+        let index_info = MockIndexInfoProvider::new(vec![(
+            "color",
+            ColInfo::new(
+                DataType::Utf8,
+                Box::new(SargableQueryParser::new("color_idx".to_string(), false)),
+            ),
+        )]);
+
+        // Simple prefix pattern: LIKE 'foo%' -> LikePrefix("foo"), no refine_expr
+        let schema = Schema::new(vec![Field::new("color", DataType::Utf8, false)]);
+        let df_schema: DFSchema = schema.try_into().unwrap();
+        let ctx = get_session_context(&LanceExecutionOptions::default());
+        let state = ctx.state();
+
+        let expr = state
+            .create_logical_expr("color LIKE 'foo%'", &df_schema)
+            .unwrap();
+        let result = apply_scalar_indices(expr, &index_info).unwrap();
+
+        assert!(result.scalar_query.is_some(), "Should have scalar_query");
+        assert!(
+            result.refine_expr.is_none(),
+            "Simple prefix should not need refine_expr"
+        );
+
+        // Extract the query and verify it's LikePrefix
+        if let Some(ScalarIndexExpr::Query(search)) = &result.scalar_query {
+            let query = search.query.as_any().downcast_ref::<SargableQuery>();
+            assert!(query.is_some(), "Query should be SargableQuery");
+            match query.unwrap() {
+                SargableQuery::LikePrefix(prefix) => {
+                    assert_eq!(prefix, &ScalarValue::Utf8(Some("foo".to_string())));
+                }
+                _ => panic!("Expected LikePrefix query"),
+            }
+        } else {
+            panic!("Expected Query variant");
+        }
+
+        // Complex pattern: LIKE 'foo%bar%' -> LikePrefix("foo"), with refine_expr
+        let expr = state
+            .create_logical_expr("color LIKE 'foo%bar%'", &df_schema)
+            .unwrap();
+        let result = apply_scalar_indices(expr, &index_info).unwrap();
+
+        assert!(result.scalar_query.is_some(), "Should have scalar_query");
+        assert!(
+            result.refine_expr.is_some(),
+            "Complex pattern should have refine_expr"
+        );
+
+        // Verify the query is still LikePrefix("foo")
+        if let Some(ScalarIndexExpr::Query(search)) = &result.scalar_query {
+            let query = search.query.as_any().downcast_ref::<SargableQuery>();
+            assert!(query.is_some(), "Query should be SargableQuery");
+            match query.unwrap() {
+                SargableQuery::LikePrefix(prefix) => {
+                    assert_eq!(prefix, &ScalarValue::Utf8(Some("foo".to_string())));
+                }
+                _ => panic!("Expected LikePrefix query"),
+            }
+        }
+
+        // Verify the refine_expr is the original LIKE expression
+        let refine = result.refine_expr.unwrap();
+        match refine {
+            Expr::Like(like) => {
+                assert!(!like.negated);
+                assert!(!like.case_insensitive);
+                if let Expr::Literal(ScalarValue::Utf8(Some(pattern)), _) = like.pattern.as_ref() {
+                    assert_eq!(pattern, "foo%bar%");
+                } else {
+                    panic!("Expected Utf8 literal pattern");
+                }
+            }
+            _ => panic!("Expected Like expression in refine_expr"),
+        }
+
+        // Pattern starting with wildcard: LIKE '%foo' -> no index, only refine
+        let expr = state
+            .create_logical_expr("color LIKE '%foo'", &df_schema)
+            .unwrap();
+        let result = apply_scalar_indices(expr, &index_info).unwrap();
+
+        assert!(
+            result.scalar_query.is_none(),
+            "Pattern starting with wildcard should not use index"
+        );
+        assert!(result.refine_expr.is_some(), "Should fall back to refine");
+    }
+
+    #[test]
+    fn test_starts_with_with_underscore_after_optimization() {
+        // Test that starts_with with underscore in prefix works correctly after DataFusion optimization
+        // DataFusion simplifies starts_with(col, 'test_ns$') to col LIKE 'test_ns$%'
+        // The underscore in the prefix should NOT be treated as a wildcard!
+        let index_info = MockIndexInfoProvider::new(vec![(
+            "object_id",
+            ColInfo::new(
+                DataType::Utf8,
+                Box::new(SargableQueryParser::new("object_id_idx".to_string(), false)),
+            ),
+        )]);
+
+        let schema = Schema::new(vec![Field::new("object_id", DataType::Utf8, false)]);
+        let df_schema: DFSchema = schema.try_into().unwrap();
+        let ctx = get_session_context(&LanceExecutionOptions::default());
+        let state = ctx.state();
+
+        // Create the expression with starts_with containing underscore
+        let expr = state
+            .create_logical_expr("starts_with(object_id, 'test_ns$')", &df_schema)
+            .unwrap();
+
+        // Apply DataFusion simplification (this may convert starts_with to LIKE)
+        let props = ExecutionProps::new().with_query_execution_start_time(Utc::now());
+        let simplify_context = SimplifyContext::new(&props).with_schema(Arc::new(df_schema));
+        let simplifier =
+            datafusion::optimizer::simplify_expressions::ExprSimplifier::new(simplify_context);
+        let simplified_expr = simplifier.simplify(expr).unwrap();
+
+        // Apply scalar indices
+        let result = apply_scalar_indices(simplified_expr, &index_info).unwrap();
+
+        // The prefix should be "test_ns$", NOT "test"
+        // This test documents the current (potentially broken) behavior
+        if let Some(ScalarIndexExpr::Query(search)) = &result.scalar_query {
+            let query = search
+                .query
+                .as_any()
+                .downcast_ref::<SargableQuery>()
+                .unwrap();
+            match query {
+                SargableQuery::LikePrefix(prefix) => {
+                    let prefix_str = match prefix {
+                        ScalarValue::Utf8(Some(s)) => s.clone(),
+                        _ => panic!("Expected Utf8 prefix"),
+                    };
+                    // Verify the prefix is correctly extracted with underscore as literal
+                    assert_eq!(
+                        prefix_str, "test_ns$",
+                        "Prefix should be 'test_ns$', not 'test' (underscore should not be a wildcard)"
+                    );
+                }
+                _ => panic!("Expected LikePrefix query"),
+            }
+        } else {
+            // If no scalar query, it means the pattern was not recognized
+            panic!("Expected scalar_query to be present");
+        }
+    }
+
+    #[test]
+    fn test_starts_with_to_like_conversion() {
+        // Test that starts_with(col, 'prefix') is converted to LikePrefix query
+        let index_info = MockIndexInfoProvider::new(vec![(
+            "color",
+            ColInfo::new(
+                DataType::Utf8,
+                Box::new(SargableQueryParser::new("color_idx".to_string(), false)),
+            ),
+        )]);
+
+        let schema = Schema::new(vec![Field::new("color", DataType::Utf8, false)]);
+        let df_schema: DFSchema = schema.try_into().unwrap();
+        let ctx = get_session_context(&LanceExecutionOptions::default());
+        let state = ctx.state();
+
+        // starts_with(color, 'foo') should be converted to LikePrefix("foo")
+        let expr = state
+            .create_logical_expr("starts_with(color, 'foo')", &df_schema)
+            .unwrap();
+        let result = apply_scalar_indices(expr, &index_info).unwrap();
+
+        assert!(
+            result.scalar_query.is_some(),
+            "starts_with should use index"
+        );
+        assert!(
+            result.refine_expr.is_none(),
+            "Pure prefix starts_with should not need refine_expr"
+        );
+
+        // Extract the query and verify it's LikePrefix
+        if let Some(ScalarIndexExpr::Query(search)) = &result.scalar_query {
+            let query = search.query.as_any().downcast_ref::<SargableQuery>();
+            assert!(query.is_some(), "Query should be SargableQuery");
+            match query.unwrap() {
+                SargableQuery::LikePrefix(prefix) => {
+                    assert_eq!(prefix, &ScalarValue::Utf8(Some("foo".to_string())));
+                }
+                _ => panic!("Expected LikePrefix query"),
+            }
+        } else {
+            panic!("Expected Query variant");
+        }
+
+        // Both starts_with and LIKE 'prefix%' should produce the same LikePrefix query
+        let like_expr = state
+            .create_logical_expr("color LIKE 'foo%'", &df_schema)
+            .unwrap();
+        let like_result = apply_scalar_indices(like_expr, &index_info).unwrap();
+
+        // Compare the queries - both should be LikePrefix("foo")
+        if let (
+            Some(ScalarIndexExpr::Query(starts_with_search)),
+            Some(ScalarIndexExpr::Query(like_search)),
+        ) = (&result.scalar_query, &like_result.scalar_query)
+        {
+            let sw_query = starts_with_search
+                .query
+                .as_any()
+                .downcast_ref::<SargableQuery>()
+                .unwrap();
+            let like_query = like_search
+                .query
+                .as_any()
+                .downcast_ref::<SargableQuery>()
+                .unwrap();
+            assert_eq!(
+                sw_query, like_query,
+                "starts_with and LIKE 'prefix%' should produce identical queries"
+            );
+        }
+    }
 }
diff --git a/rust/lance-index/src/scalar/inverted.rs b/rust/lance-index/src/scalar/inverted.rs
index 8cb5131e417..5057b587000 100644
--- a/rust/lance-index/src/scalar/inverted.rs
+++ b/rust/lance-index/src/scalar/inverted.rs
@@ -6,7 +6,6 @@ mod encoding;
 mod index;
 mod iter;
 pub mod json;
-mod merger;
 pub mod parser;
 pub mod query;
 mod scorer;
@@ -21,8 +20,8 @@ pub use builder::InvertedIndexBuilder;
 use datafusion::execution::SendableRecordBatchStream;
 pub use index::*;
 use lance_core::{Result, cache::LanceCache};
+pub use lance_tokenizer::Language;
 pub use scorer::MemBM25Scorer;
-use tantivy::tokenizer::Language;
 pub use tokenizer::*;
 
 use lance_core::Error;
@@ -66,10 +65,11 @@ impl InvertedIndexPlugin {
         let mut inverted_index =
             InvertedIndexBuilder::new_with_fragment_mask(params, fragment_mask)
                 .with_progress(progress);
-        inverted_index.update(data, index_store).await?;
+        inverted_index.update(data, index_store, None).await?;
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&details).unwrap(),
-            index_version: INVERTED_INDEX_VERSION,
+            index_version: current_fts_format_version().index_version(),
+            files: Some(index_store.list_files_with_sizes().await?),
         })
     }
 
@@ -138,7 +138,7 @@ impl ScalarIndexPlugin for InvertedIndexPlugin {
     }
 
     fn version(&self) -> u32 {
-        INVERTED_INDEX_VERSION
+        max_supported_fts_format_version().index_version()
     }
 
     fn new_query_parser(
@@ -215,3 +215,17 @@ impl ScalarIndexPlugin for InvertedIndexPlugin {
         Ok(serde_json::json!(&index_params))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_plugin_version_tracks_max_supported_format() {
+        let plugin = InvertedIndexPlugin;
+        assert_eq!(
+            plugin.version(),
+            max_supported_fts_format_version().index_version()
+        );
+    }
+}
diff --git a/rust/lance-index/src/scalar/inverted/builder.rs b/rust/lance-index/src/scalar/inverted/builder.rs
index b52a656628e..1da3a423bb8 100644
--- a/rust/lance-index/src/scalar/inverted/builder.rs
+++ b/rust/lance-index/src/scalar/inverted/builder.rs
@@ -1,34 +1,33 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use super::{
-    InvertedIndexParams,
-    index::*,
-    merger::{Merger, PartitionSource, SizeBasedMerger},
-};
-use crate::scalar::IndexStore;
+use super::{InvertedIndexParams, index::*};
+use crate::scalar::inverted::document_tokenizer::DocType;
 use crate::scalar::inverted::json::JsonTextStream;
-use crate::scalar::inverted::lance_tokenizer::DocType;
-use crate::scalar::inverted::tokenizer::lance_tokenizer::LanceTokenizer;
+use crate::scalar::inverted::tokenizer::document_tokenizer::LanceTokenizer;
+#[cfg(test)]
 use crate::scalar::lance_format::LanceIndexStore;
+use crate::scalar::{IndexStore, OldIndexDataFilter};
 use crate::vector::graph::OrderedFloat;
 use crate::{progress::IndexBuildProgress, progress::noop_progress};
 use arrow::array::AsArray;
 use arrow::datatypes;
-use arrow_array::{Array, RecordBatch, UInt64Array};
+use arrow_array::{Array, BinaryArray, RecordBatch, UInt64Array};
 use arrow_schema::{DataType, Field, Schema, SchemaRef};
 use bitpacking::{BitPacker, BitPacker4x};
 use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream};
 use deepsize::DeepSizeOf;
+use fst::Streamer;
 use futures::{Stream, StreamExt, TryStreamExt};
 use lance_arrow::json::JSON_EXT_NAME;
 use lance_arrow::{ARROW_EXT_NAME_KEY, iter_str_array};
 use lance_core::cache::LanceCache;
-use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu};
+use lance_core::error::LanceOptionExt;
+use lance_core::utils::tokio::{IO_CORE_RESERVATION, get_num_compute_intensive_cpus, spawn_cpu};
 use lance_core::{Error, ROW_ID, ROW_ID_FIELD, Result};
-use lance_core::{error::LanceOptionExt, utils::tempfile::TempDir};
 use lance_io::object_store::ObjectStore;
 use object_store::path::Path;
+use roaring::RoaringBitmap;
 use smallvec::SmallVec;
 use std::collections::HashMap;
 use std::pin::Pin;
@@ -44,31 +43,77 @@ use tracing::instrument;
 // WARNING: changing this value will break the compatibility with existing indexes
 pub const BLOCK_SIZE: usize = BitPacker4x::BLOCK_LEN;
 
-// the number of shards to split the indexing work,
-// the indexing process would spawn `LANCE_FTS_NUM_SHARDS` workers to build FTS,
-// higher for faster indexing performance, but more memory usage,
-// it's `the number of compute intensive CPUs` by default
+// The default number of workers to use for FTS builds.
+// By default this is roughly `num_cpus / 2`, but it can be overridden
+// with `LANCE_FTS_NUM_SHARDS`.
 pub static LANCE_FTS_NUM_SHARDS: LazyLock<usize> = LazyLock::new(|| {
     std::env::var("LANCE_FTS_NUM_SHARDS")
-        .unwrap_or_else(|_| get_num_compute_intensive_cpus().to_string())
+        .unwrap_or_else(|_| default_num_workers().to_string())
         .parse()
         .expect("failed to parse LANCE_FTS_NUM_SHARDS")
 });
-// the partition size limit in MiB (uncompressed format)
-// higher for better indexing & query performance, but more memory usage,
+// The default per-worker memory limit in MiB for FTS builds.
 pub static LANCE_FTS_PARTITION_SIZE: LazyLock<u64> = LazyLock::new(|| {
     std::env::var("LANCE_FTS_PARTITION_SIZE")
-        .unwrap_or_else(|_| "256".to_string())
+        .unwrap_or_else(|_| "2048".to_string())
         .parse()
         .expect("failed to parse LANCE_FTS_PARTITION_SIZE")
 });
-// the target size of partition after merging in MiB (uncompressed format)
-pub static LANCE_FTS_TARGET_SIZE: LazyLock<u64> = LazyLock::new(|| {
-    std::env::var("LANCE_FTS_TARGET_SIZE")
-        .unwrap_or_else(|_| "4096".to_string())
+static LANCE_FTS_WRITE_QUEUE_SIZE: LazyLock<usize> = LazyLock::new(|| {
+    std::env::var("LANCE_FTS_WRITE_QUEUE_SIZE")
+        .unwrap_or_else(|_| "1".to_string())
+        .parse()
+        .expect("failed to parse LANCE_FTS_WRITE_QUEUE_SIZE")
+});
+static LANCE_FTS_POSTING_BATCH_ROWS: LazyLock<usize> = LazyLock::new(|| {
+    std::env::var("LANCE_FTS_POSTING_BATCH_ROWS")
+        .unwrap_or_else(|_| "256".to_string())
         .parse()
-        .expect("failed to parse LANCE_FTS_TARGET_SIZE")
+        .expect("failed to parse LANCE_FTS_POSTING_BATCH_ROWS")
 });
+const MAX_RETAINED_TOKEN_IDS: usize = 8 * 1024;
+
+fn default_num_workers() -> usize {
+    let total_cpus = get_num_compute_intensive_cpus() + *IO_CORE_RESERVATION;
+    std::cmp::max(1, total_cpus / 2)
+}
+
+fn resolve_num_workers(params: &InvertedIndexParams) -> usize {
+    let max_workers = get_num_compute_intensive_cpus().max(1);
+    params
+        .num_workers
+        .unwrap_or(*LANCE_FTS_NUM_SHARDS)
+        .clamp(1, max_workers)
+}
+
+fn resolve_worker_memory_limit_bytes(params: &InvertedIndexParams, num_workers: usize) -> u64 {
+    let default_worker_memory_limit_bytes = *LANCE_FTS_PARTITION_SIZE << 20;
+    params
+        .memory_limit_mb
+        .map(|memory_limit_mb| (memory_limit_mb << 20) / num_workers as u64)
+        .unwrap_or(default_worker_memory_limit_bytes)
+}
+
+fn merge_all_tail_partitions(tails: Vec<TailPartition>) -> Result<Option<InnerBuilder>> {
+    if tails.is_empty() {
+        return Ok(None);
+    }
+    merge_tail_partition_group(tails).map(Some)
+}
+
+fn merge_tail_partition_group(group: Vec<TailPartition>) -> Result<InnerBuilder> {
+    let mut group = group.into_iter();
+    let mut merged = group
+        .next()
+        .ok_or_else(|| {
+            Error::invalid_input("cannot merge an empty tail partition group".to_owned())
+        })?
+        .builder;
+    for tail in group {
+        merged.merge_from(tail.builder)?;
+    }
+    Ok(merged)
+}
 
 #[derive(Debug)]
 pub struct InvertedIndexBuilder {
@@ -77,10 +122,11 @@ pub struct InvertedIndexBuilder {
     new_partitions: Vec<u64>,
     fragment_mask: Option<u64>,
     token_set_format: TokenSetFormat,
-    _tmpdir: TempDir,
-    local_store: Arc<dyn IndexStore>,
-    src_store: Arc<dyn IndexStore>,
+    format_version: InvertedListFormatVersion,
+    posting_tail_codec: PostingTailCodec,
+    src_store: Option<Arc<dyn IndexStore>>,
     progress: Arc<dyn IndexBuildProgress>,
+    deleted_fragments: RoaringBitmap,
 }
 
 impl InvertedIndexBuilder {
@@ -95,6 +141,7 @@ impl InvertedIndexBuilder {
             Vec::new(),
             TokenSetFormat::default(),
             fragment_mask,
+            RoaringBitmap::new(),
         )
     }
 
@@ -110,27 +157,35 @@ impl InvertedIndexBuilder {
         partitions: Vec<u64>,
         token_set_format: TokenSetFormat,
         fragment_mask: Option<u64>,
+        deleted_fragments: RoaringBitmap,
     ) -> Self {
-        let tmpdir = TempDir::default();
-        let local_store = Arc::new(LanceIndexStore::new(
-            ObjectStore::local().into(),
-            tmpdir.obj_path(),
-            Arc::new(LanceCache::no_cache()),
-        ));
-        let src_store = store.unwrap_or_else(|| local_store.clone());
         Self {
             params,
             partitions,
             new_partitions: Vec::new(),
-            _tmpdir: tmpdir,
-            local_store,
-            src_store,
+            src_store: store,
             token_set_format,
             fragment_mask,
+            format_version: current_fts_format_version(),
+            posting_tail_codec: current_fts_format_version().posting_tail_codec(),
             progress: noop_progress(),
+            deleted_fragments,
         }
     }
 
+    pub fn with_posting_tail_codec(mut self, posting_tail_codec: PostingTailCodec) -> Self {
+        self.format_version =
+            InvertedListFormatVersion::from_posting_tail_codec(posting_tail_codec);
+        self.posting_tail_codec = posting_tail_codec;
+        self
+    }
+
+    pub fn with_format_version(mut self, format_version: InvertedListFormatVersion) -> Self {
+        self.format_version = format_version;
+        self.posting_tail_codec = format_version.posting_tail_codec();
+        self
+    }
+
     pub fn with_progress(mut self, progress: Arc<dyn IndexBuildProgress>) -> Self {
         self.progress = progress;
         self
@@ -140,6 +195,7 @@ impl InvertedIndexBuilder {
         &mut self,
         new_data: SendableRecordBatchStream,
         dest_store: &dyn IndexStore,
+        old_data_filter: Option<crate::scalar::OldIndexDataFilter>,
     ) -> Result<()> {
         let schema = new_data.schema();
         let doc_col = schema.field(0).name();
@@ -157,41 +213,51 @@ impl InvertedIndexBuilder {
         self.progress
             .stage_start("tokenize_docs", None, "rows")
             .await?;
-        self.update_index(new_data).await?;
+        self.update_index(new_data, dest_store).await?;
+
+        if let Some(OldIndexDataFilter::Fragments { to_remove, .. }) = old_data_filter {
+            self.deleted_fragments.extend(to_remove);
+        }
+
         self.progress.stage_complete("tokenize_docs").await?;
         self.write(dest_store).await?;
         Ok(())
     }
 
     #[instrument(level = "debug", skip_all)]
-    async fn update_index(&mut self, stream: SendableRecordBatchStream) -> Result<()> {
-        let num_workers = *LANCE_FTS_NUM_SHARDS;
+    async fn update_index(
+        &mut self,
+        stream: SendableRecordBatchStream,
+        dest_store: &dyn IndexStore,
+    ) -> Result<()> {
+        let num_workers = resolve_num_workers(&self.params);
         let tokenizer = self.params.build()?;
         let with_position = self.params.with_position;
+        let worker_memory_limit_bytes =
+            resolve_worker_memory_limit_bytes(&self.params, num_workers);
+        let worker_config = IndexWorkerConfig {
+            with_position,
+            format_version: self.format_version,
+            fragment_mask: self.fragment_mask,
+            token_set_format: self.token_set_format,
+            worker_memory_limit_bytes,
+        };
         let next_id = self.partitions.iter().map(|id| id + 1).max().unwrap_or(0);
         let id_alloc = Arc::new(AtomicU64::new(next_id));
         let tokenized_count = Arc::new(AtomicU64::new(0));
         let (sender, receiver) = async_channel::bounded(num_workers);
+        let dest_store = dest_store.clone_arc();
         let mut index_tasks = Vec::with_capacity(num_workers);
         for _ in 0..num_workers {
-            let store = self.local_store.clone();
             let tokenizer = tokenizer.clone();
             let receiver: async_channel::Receiver<RecordBatch> = receiver.clone();
+            let dest_store = dest_store.clone();
             let id_alloc = id_alloc.clone();
             let progress = self.progress.clone();
-            let fragment_mask = self.fragment_mask;
-            let token_set_format = self.token_set_format;
             let tokenized_count = tokenized_count.clone();
             index_tasks.push(tokio::task::spawn(async move {
-                let mut worker = IndexWorker::new(
-                    store,
-                    tokenizer,
-                    with_position,
-                    id_alloc,
-                    fragment_mask,
-                    token_set_format,
-                )
-                .await?;
+                let mut worker =
+                    IndexWorker::new(tokenizer, dest_store, id_alloc, worker_config).await?;
                 while let Ok(batch) = receiver.recv().await {
                     let num_rows = batch.num_rows();
                     worker.process_batch(batch).await?;
@@ -205,49 +271,66 @@ impl InvertedIndexBuilder {
                 worker.finish().await
             }));
         }
-        // Keep the channel lifetime tied to the worker tasks so senders observe
-        // worker exits instead of blocking on an orphaned receiver handle.
-        drop(receiver);
 
-        let mut stream = Box::pin(stream);
-        log::info!("indexing FTS with {} workers", num_workers);
+        let index_build = async {
+            // Keep the channel lifetime tied to the worker tasks so senders observe
+            // worker exits instead of blocking on an orphaned receiver handle.
+            drop(receiver);
+
+            let mut stream = Box::pin(stream);
+            log::info!("indexing FTS with {} workers", num_workers);
 
-        let mut last_num_rows = 0;
-        let mut total_num_rows = 0;
-        let start = std::time::Instant::now();
-        while let Some(batch) = stream.try_next().await? {
-            let num_rows = batch.num_rows();
+            let mut last_num_rows = 0;
+            let mut total_num_rows = 0;
+            let start = std::time::Instant::now();
+            while let Some(batch) = stream.try_next().await? {
+                let num_rows = batch.num_rows();
+
+                if sender.send(batch).await.is_err() {
+                    // this only happens if all workers have exited,
+                    // so we don't return the send error here,
+                    // avoiding hiding the real error from workers.
+                    break;
+                }
 
-            if sender.send(batch).await.is_err() {
-                // this only happens if all workers have existed,
-                // so we don't return the send error here,
-                // avoiding hiding the real error from workers.
-                break;
+                total_num_rows += num_rows;
+                if total_num_rows >= last_num_rows + 1_000_000 {
+                    log::debug!(
+                        "indexed {} documents, elapsed: {:?}, speed: {}rows/s",
+                        total_num_rows,
+                        start.elapsed(),
+                        total_num_rows as f32 / start.elapsed().as_secs_f32()
+                    );
+                    last_num_rows = total_num_rows;
+                }
             }
+            // drop the sender to stop receivers
+            drop(stream);
+            drop(sender);
+            log::info!("dispatching elapsed: {:?}", start.elapsed());
 
-            total_num_rows += num_rows;
-            if total_num_rows >= last_num_rows + 1_000_000 {
-                log::debug!(
-                    "indexed {} documents, elapsed: {:?}, speed: {}rows/s",
-                    total_num_rows,
-                    start.elapsed(),
-                    total_num_rows as f32 / start.elapsed().as_secs_f32()
-                );
-                last_num_rows = total_num_rows;
+            // wait for the workers to finish
+            let start = std::time::Instant::now();
+            let mut tail_partitions = Vec::new();
+            for index_task in index_tasks {
+                let output = index_task.await??;
+                self.new_partitions.extend(output.partitions);
+                if let Some(tail_partition) = output.tail_partition {
+                    tail_partitions.push(tail_partition);
+                }
             }
-        }
-        // drop the sender to stop receivers
-        drop(stream);
-        drop(sender);
-        log::info!("dispatching elapsed: {:?}", start.elapsed());
+            let merged_tail_partitions =
+                spawn_cpu(move || merge_all_tail_partitions(tail_partitions)).await?;
+            if let Some(builder) = merged_tail_partitions {
+                self.new_partitions.push(builder.id());
+                let mut builder = builder;
+                builder.write(dest_store.as_ref()).await?;
+            }
+            log::info!("wait workers indexing elapsed: {:?}", start.elapsed());
+            Result::Ok(())
+        };
 
-        // wait for the workers to finish
-        let start = std::time::Instant::now();
-        for index_task in index_tasks {
-            self.new_partitions.extend(index_task.await??);
-        }
-        log::info!("wait workers indexing elapsed: {:?}", start.elapsed());
-        Ok(())
+        index_build.await
     }
 
     pub async fn remap(
@@ -281,17 +364,54 @@ impl InvertedIndexBuilder {
     }
 
     async fn write_metadata(&self, dest_store: &dyn IndexStore, partitions: &[u64]) -> Result<()> {
-        let metadata = HashMap::from_iter(vec![
+        let mut serialized_deleted_fragments =
+            Vec::with_capacity(self.deleted_fragments.serialized_size());
+        self.deleted_fragments
+            .serialize_into(&mut serialized_deleted_fragments)?;
+
+        let mut metadata = HashMap::from_iter(vec![
             ("partitions".to_owned(), serde_json::to_string(&partitions)?),
             ("params".to_owned(), serde_json::to_string(&self.params)?),
             (
                 TOKEN_SET_FORMAT_KEY.to_owned(),
                 self.token_set_format.to_string(),
             ),
+            (
+                POSTING_TAIL_CODEC_KEY.to_owned(),
+                self.posting_tail_codec.as_str().to_owned(),
+            ),
         ]);
+
+        if self.params.with_position && self.format_version.uses_shared_position_stream() {
+            metadata.insert(
+                POSITIONS_LAYOUT_KEY.to_owned(),
+                POSITIONS_LAYOUT_SHARED_STREAM_V2.to_owned(),
+            );
+            metadata.insert(
+                POSITIONS_CODEC_KEY.to_owned(),
+                self.format_version
+                    .position_codec()
+                    .expect("shared positions require a codec")
+                    .as_str()
+                    .to_owned(),
+            );
+        }
+
+        let metadata_file_schema = Arc::new(Schema::new(vec![Field::new(
+            DELETED_FRAGMENTS_COL,
+            DataType::Binary,
+            false,
+        )]));
+        let deleted_fragments_col = Arc::new(BinaryArray::from(vec![
+            serialized_deleted_fragments.as_slice(),
+        ])) as Arc<dyn Array>;
+        let record_batch =
+            RecordBatch::try_new(metadata_file_schema.clone(), vec![deleted_fragments_col])?;
+
         let mut writer = dest_store
-            .new_index_file(METADATA_FILE, Arc::new(Schema::empty()))
+            .new_index_file(METADATA_FILE, metadata_file_schema)
             .await?;
+        writer.write_record_batch(record_batch).await?;
         writer.finish_with_metadata(metadata).await?;
         Ok(())
     }
@@ -306,14 +426,32 @@ impl InvertedIndexBuilder {
         partition: u64, // Modify parameter type
     ) -> Result<()> {
         let partitions = vec![partition];
-        let metadata = HashMap::from_iter(vec![
+        let mut metadata = HashMap::from_iter(vec![
             ("partitions".to_owned(), serde_json::to_string(&partitions)?),
             ("params".to_owned(), serde_json::to_string(&self.params)?),
             (
                 TOKEN_SET_FORMAT_KEY.to_owned(),
                 self.token_set_format.to_string(),
             ),
+            (
+                POSTING_TAIL_CODEC_KEY.to_owned(),
+                self.posting_tail_codec.as_str().to_owned(),
+            ),
         ]);
+        if self.params.with_position && self.format_version.uses_shared_position_stream() {
+            metadata.insert(
+                POSITIONS_LAYOUT_KEY.to_owned(),
+                POSITIONS_LAYOUT_SHARED_STREAM_V2.to_owned(),
+            );
+            metadata.insert(
+                POSITIONS_CODEC_KEY.to_owned(),
+                self.format_version
+                    .position_codec()
+                    .expect("shared positions require a codec")
+                    .as_str()
+                    .to_owned(),
+            );
+        }
         // Use partition ID to generate a unique temporary filename
         let file_name = part_metadata_file_path(partition);
         let mut writer = dest_store
@@ -354,84 +492,47 @@ impl InvertedIndexBuilder {
     }
 
     async fn write(&self, dest_store: &dyn IndexStore) -> Result<()> {
-        if self.params.skip_merge {
-            let mut partitions =
-                Vec::with_capacity(self.partitions.len() + self.new_partitions.len());
-            partitions.extend_from_slice(&self.partitions);
-            partitions.extend_from_slice(&self.new_partitions);
-            partitions.sort_unstable();
-
-            self.progress
-                .stage_start(
-                    "copy_partitions",
-                    Some(partitions.len() as u64),
-                    "partitions",
-                )
-                .await?;
-            let mut copied = 0;
-            for part in self.partitions.iter() {
-                self.src_store
-                    .copy_index_file(&token_file_path(*part), dest_store)
-                    .await?;
-                self.src_store
-                    .copy_index_file(&posting_file_path(*part), dest_store)
-                    .await?;
-                self.src_store
-                    .copy_index_file(&doc_file_path(*part), dest_store)
-                    .await?;
-                copied += 1;
-                self.progress
-                    .stage_progress("copy_partitions", copied)
-                    .await?;
-            }
-            for part in self.new_partitions.iter() {
-                self.local_store
-                    .copy_index_file(&token_file_path(*part), dest_store)
-                    .await?;
-                self.local_store
-                    .copy_index_file(&posting_file_path(*part), dest_store)
-                    .await?;
-                self.local_store
-                    .copy_index_file(&doc_file_path(*part), dest_store)
-                    .await?;
-                copied += 1;
-                self.progress
-                    .stage_progress("copy_partitions", copied)
-                    .await?;
-            }
-            self.progress.stage_complete("copy_partitions").await?;
-
-            self.write_metadata_with_progress(dest_store, &partitions)
-                .await?;
-            return Ok(());
-        }
+        let mut partitions = Vec::with_capacity(self.partitions.len() + self.new_partitions.len());
+        partitions.extend_from_slice(&self.partitions);
+        partitions.extend_from_slice(&self.new_partitions);
+        partitions.sort_unstable();
 
-        let partitions = self
-            .partitions
-            .iter()
-            .map(|part| PartitionSource::new(self.src_store.clone(), *part))
-            .chain(
-                self.new_partitions
-                    .iter()
-                    .map(|part| PartitionSource::new(self.local_store.clone(), *part)),
-            )
-            .collect::<Vec<_>>();
         self.progress
             .stage_start(
-                "merge_partitions",
+                "copy_partitions",
                 Some(partitions.len() as u64),
                 "partitions",
             )
             .await?;
-        let mut merger = SizeBasedMerger::new(
-            dest_store,
-            partitions,
-            *LANCE_FTS_TARGET_SIZE << 20,
-            self.token_set_format,
-            self.progress.clone(),
-        );
-        let partitions = merger.merge().await?;
-        self.progress.stage_complete("merge_partitions").await?;
+        let mut copied = 0;
+        for part in self.partitions.iter() {
+            self.src_store
+                .as_ref()
+                .expect("existing partitions require a source store")
+                .copy_index_file(&token_file_path(*part), dest_store)
+                .await?;
+            self.src_store
+                .as_ref()
+                .expect("existing partitions require a source store")
+                .copy_index_file(&posting_file_path(*part), dest_store)
+                .await?;
+            self.src_store
+                .as_ref()
+                .expect("existing partitions require a source store")
+                .copy_index_file(&doc_file_path(*part), dest_store)
+                .await?;
+            copied += 1;
+            self.progress
+                .stage_progress("copy_partitions", copied)
+                .await?;
+        }
+        for _part in self.new_partitions.iter() {
+            copied += 1;
+            self.progress
+                .stage_progress("copy_partitions", copied)
+                .await?;
+        }
+        self.progress.stage_complete("copy_partitions").await?;
 
         self.write_metadata_with_progress(dest_store, &partitions)
             .await?;
@@ -452,6 +553,8 @@ pub struct InnerBuilder {
     id: u64,
     with_position: bool,
     token_set_format: TokenSetFormat,
+    format_version: InvertedListFormatVersion,
+    posting_tail_codec: PostingTailCodec,
     pub(crate) tokens: TokenSet,
     pub(crate) posting_lists: Vec<PostingListBuilder>,
     pub(crate) docs: DocSet,
@@ -459,16 +562,49 @@ pub struct InnerBuilder {
 
 impl InnerBuilder {
     pub fn new(id: u64, with_position: bool, token_set_format: TokenSetFormat) -> Self {
+        Self::new_with_format_version(
+            id,
+            with_position,
+            token_set_format,
+            current_fts_format_version(),
+        )
+    }
+
+    pub fn new_with_format_version(
+        id: u64,
+        with_position: bool,
+        token_set_format: TokenSetFormat,
+        format_version: InvertedListFormatVersion,
+    ) -> Self {
         Self {
             id,
             with_position,
             token_set_format,
+            format_version,
+            posting_tail_codec: format_version.posting_tail_codec(),
             tokens: TokenSet::default(),
             posting_lists: Vec::new(),
             docs: DocSet::default(),
         }
     }
 
+    pub fn new_with_posting_tail_codec(
+        id: u64,
+        with_position: bool,
+        token_set_format: TokenSetFormat,
+        posting_tail_codec: PostingTailCodec,
+    ) -> Self {
+        let format_version = if posting_tail_codec == PostingTailCodec::Fixed32 {
+            InvertedListFormatVersion::V1
+        } else {
+            InvertedListFormatVersion::V2
+        };
+        let mut builder =
+            Self::new_with_format_version(id, with_position, token_set_format, format_version);
+        builder.posting_tail_codec = posting_tail_codec;
+        builder
+    }
+
     pub fn id(&self) -> u64 {
         self.id
     }
@@ -514,6 +650,90 @@ impl InnerBuilder {
         Ok(())
     }
 
+    pub fn merge_from(&mut self, other: Self) -> Result<()> {
+        let Self {
+            id: _,
+            with_position,
+            token_set_format,
+            format_version,
+            posting_tail_codec,
+            tokens,
+            posting_lists,
+            docs,
+        } = other;
+
+        if self.with_position != with_position {
+            return Err(Error::index(format!(
+                "cannot merge partitions with mismatched positions settings: {} vs {}",
+                self.with_position, with_position
+            )));
+        }
+        if self.token_set_format != token_set_format {
+            return Err(Error::index(format!(
+                "cannot merge partitions with mismatched token set formats: {:?} vs {:?}",
+                self.token_set_format, token_set_format
+            )));
+        }
+        if self.format_version != format_version {
+            return Err(Error::index(format!(
+                "cannot merge partitions with mismatched FTS format versions: {:?} vs {:?}",
+                self.format_version, format_version
+            )));
+        }
+        if self.posting_tail_codec != posting_tail_codec {
+            return Err(Error::index(format!(
+                "cannot merge partitions with mismatched posting tail codecs: {:?} vs {:?}",
+                self.posting_tail_codec, posting_tail_codec
+            )));
+        }
+
+        let mut token_id_map = vec![u32::MAX; posting_lists.len()];
+        match tokens.tokens {
+            TokenMap::HashMap(map) => {
+                for (token, token_id) in map {
+                    let new_token_id = self.tokens.get_or_add(token.as_str());
+                    token_id_map[token_id as usize] = new_token_id;
+                }
+            }
+            TokenMap::Fst(map) => {
+                let mut stream = map.stream();
+                while let Some((token, token_id)) = stream.next() {
+                    let new_token_id = self
+                        .tokens
+                        .get_or_add(String::from_utf8_lossy(token).as_ref());
+                    token_id_map[token_id as usize] = new_token_id;
+                }
+            }
+        }
+
+        let doc_id_offset = self.docs.len() as u32;
+        for (row_id, num_tokens) in docs.iter() {
+            self.docs.append(*row_id, *num_tokens);
+        }
+        self.posting_lists.resize_with(self.tokens.len(), || {
+            PostingListBuilder::new_with_posting_tail_codec(with_position, self.posting_tail_codec)
+        });
+
+        for (token_id, posting_list) in posting_lists.into_iter().enumerate() {
+            if posting_list.is_empty() {
+                continue;
+            }
+            let new_token_id = token_id_map[token_id];
+            debug_assert_ne!(new_token_id, u32::MAX);
+            let merged_posting = &mut self.posting_lists[new_token_id as usize];
+            posting_list.for_each_entry(|doc_id, freq, positions| {
+                let positions = match positions {
+                    Some(positions) => PositionRecorder::Position(positions.into()),
+                    None => PositionRecorder::Count(freq),
+                };
+                merged_posting.add(doc_id_offset + doc_id, positions);
+                Ok::<(), Error>(())
+            })?;
+        }
+
+        Ok(())
+    }
+
     pub async fn write(&mut self, store: &dyn IndexStore) -> Result<()> {
         let docs = Arc::new(std::mem::take(&mut self.docs));
         self.write_posting_lists(store, docs.clone()).await?;
@@ -532,7 +752,7 @@ impl InnerBuilder {
         let mut writer = store
             .new_index_file(
                 &posting_file_path(self.id),
-                inverted_list_schema(self.with_position),
+                inverted_list_schema_for_version(self.with_position, self.format_version),
             )
             .await?;
         let posting_lists = std::mem::take(&mut self.posting_lists);
@@ -543,44 +763,57 @@ impl InnerBuilder {
             id,
             self.with_position
         );
-        let schema = inverted_list_schema(self.with_position);
+        let with_position = self.with_position;
+        let format_version = self.format_version;
+        let schema = inverted_list_schema_for_version(self.with_position, self.format_version);
         let docs_for_batches = docs.clone();
         let schema_for_batches = schema.clone();
-        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
+        let batch_rows = *LANCE_FTS_POSTING_BATCH_ROWS;
+        let (tx, rx) = async_channel::bounded(*LANCE_FTS_WRITE_QUEUE_SIZE);
         let producer = spawn_cpu(move || {
+            let mut batch_builder = PostingListBatchBuilder::new(
+                schema_for_batches.clone(),
+                with_position,
+                format_version,
+                batch_rows,
+            );
             for posting_list in posting_lists {
-                let batch = posting_list
-                    .to_batch_with_docs(&docs_for_batches, schema_for_batches.clone())?;
-                if let Err(err) = tx.send(batch) {
+                posting_list.append_to_batch_with_docs(
+                    &docs_for_batches,
+                    &mut batch_builder,
+                    format_version,
+                )?;
+                if batch_builder.len() < batch_rows {
+                    continue;
+                }
+
+                let batch = batch_builder.finish()?;
+                if let Err(err) = tx.send_blocking(batch) {
+                    return Err(Error::execution(format!(
+                        "failed to send posting list batch to writer: {err}"
+                    )));
+                }
+            }
+
+            if !batch_builder.is_empty() {
+                let batch = batch_builder.finish()?;
+                if let Err(err) = tx.send_blocking(batch) {
                     return Err(Error::execution(format!(
                         "failed to send posting list batch to writer: {err}"
                     )));
                 }
             }
+
             Result::Ok(())
         });
 
-        let mut write_duration = std::time::Duration::ZERO;
-        let mut num_posting_lists = 0;
-        while let Some(batch) = rx.recv().await {
-            num_posting_lists += 1;
-            let start = std::time::Instant::now();
+        while let Ok(batch) = rx.recv().await {
             if let Err(err) = writer.write_record_batch(batch).await {
                 drop(rx);
                 // Wait for producer to stop; preserve the write error as the primary failure.
                 let _ = producer.await;
                 return Err(err);
             }
-            write_duration += start.elapsed();
-
-            if num_posting_lists % 500_000 == 0 {
-                log::info!(
-                    "wrote {} posting lists of partition {}, writing elapsed: {:?}",
-                    num_posting_lists,
-                    id,
-                    write_duration,
-                );
-            }
         }
         drop(rx);
         producer.await?;
@@ -616,58 +849,110 @@ impl InnerBuilder {
 }
 
 struct IndexWorker {
-    store: Arc<dyn IndexStore>,
     tokenizer: Box<dyn LanceTokenizer>,
+    dest_store: Arc<dyn IndexStore>,
     id_alloc: Arc<AtomicU64>,
     builder: InnerBuilder,
     partitions: Vec<u64>,
     schema: SchemaRef,
-    estimated_size: u64,
+    memory_size: u64,
+    worker_memory_limit_bytes: u64,
     total_doc_length: usize,
     fragment_mask: Option<u64>,
     token_set_format: TokenSetFormat,
-    token_occurrences: HashMap<u32, PositionRecorder>,
     token_ids: Vec<u32>,
     last_token_count: usize,
-    last_unique_token_count: usize,
+}
+
+struct TailPartition {
+    builder: InnerBuilder,
+}
+
+struct WorkerOutput {
+    partitions: Vec<u64>,
+    tail_partition: Option<TailPartition>,
+}
+
+#[derive(Debug, Clone, Copy)]
+struct IndexWorkerConfig {
+    with_position: bool,
+    format_version: InvertedListFormatVersion,
+    fragment_mask: Option<u64>,
+    token_set_format: TokenSetFormat,
+    worker_memory_limit_bytes: u64,
 }
 
 impl IndexWorker {
+    fn posting_lists_overhead_size(&self) -> u64 {
+        (self.builder.posting_lists.capacity() * std::mem::size_of::<PostingListBuilder>()) as u64
+    }
+
+    fn adjust_tracked_value(tracked: &mut u64, old: u64, new: u64) {
+        if new >= old {
+            *tracked += new - old;
+        } else {
+            *tracked -= old - new;
+        }
+    }
+
+    fn adjust_tracked_memory_size(&mut self, old_memory_size: u64, new_memory_size: u64) {
+        Self::adjust_tracked_value(&mut self.memory_size, old_memory_size, new_memory_size);
+    }
+
+    fn apply_delta(total: &mut u64, delta: i64) {
+        if delta >= 0 {
+            *total += delta as u64;
+        } else {
+            *total -= (-delta) as u64;
+        }
+    }
+
+    fn temporary_memory_size(&self) -> u64 {
+        (self.token_ids.capacity() * std::mem::size_of::<u32>()) as u64
+    }
+
+    fn trim_temporary_buffers(&mut self) {
+        if self.token_ids.capacity() > MAX_RETAINED_TOKEN_IDS {
+            self.token_ids = Vec::with_capacity(self.last_token_count.min(MAX_RETAINED_TOKEN_IDS));
+        }
+    }
+
     async fn new(
-        store: Arc<dyn IndexStore>,
         tokenizer: Box<dyn LanceTokenizer>,
-        with_position: bool,
+        dest_store: Arc<dyn IndexStore>,
         id_alloc: Arc<AtomicU64>,
-        fragment_mask: Option<u64>,
-        token_set_format: TokenSetFormat,
+        config: IndexWorkerConfig,
     ) -> Result<Self> {
-        let schema = inverted_list_schema(with_position);
+        let schema = inverted_list_schema_for_version(config.with_position, config.format_version);
 
         Ok(Self {
-            store,
             tokenizer,
-            builder: InnerBuilder::new(
+            dest_store,
+            builder: InnerBuilder::new_with_format_version(
                 id_alloc.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
-                    | fragment_mask.unwrap_or(0),
-                with_position,
-                token_set_format,
+                    | config.fragment_mask.unwrap_or(0),
+                config.with_position,
+                config.token_set_format,
+                config.format_version,
             ),
             partitions: Vec::new(),
             id_alloc,
             schema,
-            estimated_size: 0,
+            memory_size: 0,
+            worker_memory_limit_bytes: config.worker_memory_limit_bytes,
             total_doc_length: 0,
-            fragment_mask,
-            token_set_format,
-            token_occurrences: HashMap::new(),
+            fragment_mask: config.fragment_mask,
+            token_set_format: config.token_set_format,
             token_ids: Vec::new(),
             last_token_count: 0,
-            last_unique_token_count: 0,
         })
     }
 
     fn has_position(&self) -> bool {
-        self.schema.column_with_name(POSITION_COL).is_some()
+        self.schema
+            .column_with_name(COMPRESSED_POSITION_COL)
+            .is_some()
+            || self.schema.column_with_name(POSITION_COL).is_some()
     }
 
     async fn process_batch(&mut self, batch: RecordBatch) -> Result<()> {
@@ -680,23 +965,55 @@ impl IndexWorker {
 
         let with_position = self.has_position();
         for (doc, row_id) in docs {
+            let builder_was_empty = self.builder.docs.is_empty();
+            let old_temporary_memory_size = self.temporary_memory_size();
+            let old_token_memory_size = self.builder.tokens.memory_size() as u64;
+            let doc_id = self.builder.docs.len() as u32;
             let mut token_num: u32 = 0;
+            let mut posting_memory_delta = 0i64;
             if with_position {
-                if self.token_occurrences.capacity() < self.last_unique_token_count {
-                    self.token_occurrences
-                        .reserve(self.last_unique_token_count - self.token_occurrences.capacity());
+                if self.token_ids.capacity() < self.last_token_count {
+                    self.token_ids
+                        .reserve(self.last_token_count - self.token_ids.capacity());
                 }
-                self.token_occurrences.clear();
+                self.token_ids.clear();
+                let builder = &mut self.builder;
+                let token_ids = &mut self.token_ids;
+                let memory_size = &mut self.memory_size;
+                let posting_tail_codec = builder.posting_tail_codec;
 
                 let mut token_stream = self.tokenizer.token_stream_for_doc(doc);
                 while token_stream.advance() {
                     let token = token_stream.token_mut();
                     let token_text = std::mem::take(&mut token.text);
-                    let token_id = self.builder.tokens.add(token_text);
-                    self.token_occurrences
-                        .entry(token_id)
-                        .or_insert_with(|| PositionRecorder::new(true))
-                        .push(token.position as u32);
+                    let token_id = builder.tokens.add(token_text);
+                    if token_id as usize == builder.posting_lists.len() {
+                        let old_posting_lists_overhead_size = (builder.posting_lists.capacity()
+                            * std::mem::size_of::<PostingListBuilder>())
+                            as u64;
+                        builder.posting_lists.push(
+                            PostingListBuilder::new_with_posting_tail_codec(
+                                true,
+                                posting_tail_codec,
+                            ),
+                        );
+                        let new_posting_lists_overhead_size = (builder.posting_lists.capacity()
+                            * std::mem::size_of::<PostingListBuilder>())
+                            as u64;
+                        Self::adjust_tracked_value(
+                            memory_size,
+                            old_posting_lists_overhead_size,
+                            new_posting_lists_overhead_size,
+                        );
+                    }
+                    let posting_list = &mut builder.posting_lists[token_id as usize];
+                    let old_posting_memory_size = posting_list.size();
+                    if posting_list.add_occurrence(doc_id, token.position as u32)? {
+                        token_ids.push(token_id);
+                    }
+                    let new_posting_memory_size = posting_list.size();
+                    posting_memory_delta +=
+                        new_posting_memory_size as i64 - old_posting_memory_size as i64;
                     token_num += 1;
                 }
             } else {
@@ -715,25 +1032,51 @@ impl IndexWorker {
                     token_num += 1;
                 }
             }
-            self.builder
-                .posting_lists
-                .resize_with(self.builder.tokens.len(), || {
-                    PostingListBuilder::new(with_position)
-                });
-            let doc_id = self.builder.docs.append(row_id, token_num);
+            self.adjust_tracked_memory_size(
+                old_token_memory_size,
+                self.builder.tokens.memory_size() as u64,
+            );
+
+            if !with_position {
+                let old_posting_lists_overhead_size = self.posting_lists_overhead_size();
+                self.builder
+                    .posting_lists
+                    .resize_with(self.builder.tokens.len(), || {
+                        PostingListBuilder::new_with_posting_tail_codec(
+                            false,
+                            self.builder.posting_tail_codec,
+                        )
+                    });
+                let new_posting_lists_overhead_size = self.posting_lists_overhead_size();
+                Self::adjust_tracked_value(
+                    &mut self.memory_size,
+                    old_posting_lists_overhead_size,
+                    new_posting_lists_overhead_size,
+                );
+            }
+
+            let old_doc_memory_size = self.builder.docs.memory_size() as u64;
+            let appended_doc_id = self.builder.docs.append(row_id, token_num);
+            debug_assert_eq!(appended_doc_id, doc_id);
+            self.adjust_tracked_memory_size(
+                old_doc_memory_size,
+                self.builder.docs.memory_size() as u64,
+            );
             self.total_doc_length += doc.len();
 
             if with_position {
-                let unique_tokens = self.token_occurrences.len();
-                for (token_id, term_positions) in self.token_occurrences.drain() {
-                    let posting_list = &mut self.builder.posting_lists[token_id as usize];
-
-                    let old_size = posting_list.size();
-                    posting_list.add(doc_id, term_positions);
-                    let new_size = posting_list.size();
-                    self.estimated_size += new_size - old_size;
+                for &token_id in &self.token_ids {
+                    let (old_posting_memory_size, new_posting_memory_size) = {
+                        let posting_list = &mut self.builder.posting_lists[token_id as usize];
+                        let old_posting_memory_size = posting_list.size();
+                        posting_list.finish_open_doc(doc_id)?;
+                        let new_posting_memory_size = posting_list.size();
+                        (old_posting_memory_size, new_posting_memory_size)
+                    };
+                    posting_memory_delta +=
+                        new_posting_memory_size as i64 - old_posting_memory_size as i64;
                 }
-                self.last_unique_token_count = unique_tokens;
+                Self::apply_delta(&mut self.memory_size, posting_memory_delta);
             } else if token_num > 0 {
                 self.token_ids.sort_unstable();
                 let mut iter = self.token_ids.iter();
@@ -745,25 +1088,46 @@ impl IndexWorker {
                         continue;
                     }
 
-                    let posting_list = &mut self.builder.posting_lists[current as usize];
-                    let old_size = posting_list.size();
-                    posting_list.add(doc_id, PositionRecorder::Count(count));
-                    let new_size = posting_list.size();
-                    self.estimated_size += new_size - old_size;
+                    let (old_posting_memory_size, new_posting_memory_size) = {
+                        let posting_list = &mut self.builder.posting_lists[current as usize];
+                        let old_posting_memory_size = posting_list.size();
+                        posting_list.add(doc_id, PositionRecorder::Count(count));
+                        let new_posting_memory_size = posting_list.size();
+                        (old_posting_memory_size, new_posting_memory_size)
+                    };
+                    posting_memory_delta +=
+                        new_posting_memory_size as i64 - old_posting_memory_size as i64;
 
                     current = token_id;
                     count = 1;
                 }
-                let posting_list = &mut self.builder.posting_lists[current as usize];
-                let old_size = posting_list.size();
-                posting_list.add(doc_id, PositionRecorder::Count(count));
-                let new_size = posting_list.size();
-                self.estimated_size += new_size - old_size;
+                let (old_posting_memory_size, new_posting_memory_size) = {
+                    let posting_list = &mut self.builder.posting_lists[current as usize];
+                    let old_posting_memory_size = posting_list.size();
+                    posting_list.add(doc_id, PositionRecorder::Count(count));
+                    let new_posting_memory_size = posting_list.size();
+                    (old_posting_memory_size, new_posting_memory_size)
+                };
+                posting_memory_delta +=
+                    new_posting_memory_size as i64 - old_posting_memory_size as i64;
+                Self::apply_delta(&mut self.memory_size, posting_memory_delta);
+            }
+            self.last_token_count = self.token_ids.len();
+            self.trim_temporary_buffers();
+            self.adjust_tracked_memory_size(
+                old_temporary_memory_size,
+                self.temporary_memory_size(),
+            );
+
+            if self.builder.docs.len() == 1 && self.memory_size > self.worker_memory_limit_bytes {
+                return Err(Error::invalid_input(format!(
+                    "single document row_id={} exceeds worker memory limit: {} > {} bytes",
+                    row_id, self.memory_size, self.worker_memory_limit_bytes
+                )));
             }
-            self.last_token_count = token_num as usize;
 
             if self.builder.docs.len() as u32 == u32::MAX
-                || self.estimated_size >= *LANCE_FTS_PARTITION_SIZE << 20
+                || (!builder_was_empty && self.memory_size >= self.worker_memory_limit_bytes)
             {
                 self.flush().await?;
             }
@@ -779,56 +1143,60 @@ impl IndexWorker {
         }
 
         log::info!(
-            "flushing posting lists, estimated size: {} MiB",
-            self.estimated_size / (1024 * 1024)
+            "flushing posting lists, memory size: {} MiB",
+            self.memory_size / (1024 * 1024)
         );
-        self.estimated_size = 0;
+        self.memory_size = self.temporary_memory_size();
         let with_position = self.has_position();
-        let mut builder = std::mem::replace(
+        let format_version = self.builder.format_version;
+        let builder = std::mem::replace(
             &mut self.builder,
-            InnerBuilder::new(
+            InnerBuilder::new_with_format_version(
                 self.id_alloc
                     .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
                     | self.fragment_mask.unwrap_or(0),
                 with_position,
                 self.token_set_format,
+                format_version,
             ),
         );
-        builder.write(self.store.as_ref()).await?;
-        self.partitions.push(builder.id());
+        let written_partition_id = builder.id();
+        let mut builder = builder;
+        builder
+            .write(self.dest_store.as_ref())
+            .await
+            .map_err(|err| {
+                Error::execution(format!(
+                    "failed to write finalized partition {}: {err}",
+                    written_partition_id
+                ))
+            })?;
+        self.partitions.push(written_partition_id);
         Ok(())
     }
 
-    async fn finish(mut self) -> Result<Vec<u64>> {
-        if !self.builder.tokens.is_empty() {
-            self.flush().await?;
-        }
-        Ok(self.partitions)
+    async fn finish(self) -> Result<WorkerOutput> {
+        let tail_partition = if self.builder.tokens.is_empty() {
+            None
+        } else {
+            Some(TailPartition {
+                builder: self.builder,
+            })
+        };
+        Ok(WorkerOutput {
+            partitions: self.partitions,
+            tail_partition,
+        })
     }
 }
 
 #[derive(Debug, Clone)]
 pub enum PositionRecorder {
-    Position(SmallVec<[u32; 4]>),
+    Position(SmallVec<[u32; 2]>),
     Count(u32),
 }
 
 impl PositionRecorder {
-    fn new(with_position: bool) -> Self {
-        if with_position {
-            Self::Position(SmallVec::new())
-        } else {
-            Self::Count(0)
-        }
-    }
-
-    fn push(&mut self, position: u32) {
-        match self {
-            Self::Position(positions) => positions.push(position),
-            Self::Count(count) => *count += 1,
-        }
-    }
-
     pub fn len(&self) -> u32 {
         match self {
             Self::Position(positions) => positions.len() as u32,
@@ -895,9 +1263,25 @@ pub fn legacy_inverted_list_schema(with_position: bool) -> SchemaRef {
 }
 
 pub fn inverted_list_schema(with_position: bool) -> SchemaRef {
+    inverted_list_schema_for_version(with_position, current_fts_format_version())
+}
+
+pub fn inverted_list_schema_for_version(
+    with_position: bool,
+    format_version: InvertedListFormatVersion,
+) -> SchemaRef {
+    match format_version {
+        InvertedListFormatVersion::V1 => inverted_list_schema_v1(with_position),
+        InvertedListFormatVersion::V2 => inverted_list_schema_with_tail_codec_and_position_codec(
+            with_position,
+            PostingTailCodec::VarintDelta,
+            Some(PositionStreamCodec::PackedDelta),
+        ),
+    }
+}
+
+fn inverted_list_schema_v1(with_position: bool) -> SchemaRef {
     let mut fields = vec![
-        // we compress the posting lists (including row ids and frequencies),
-        // and store the compressed posting lists, so it's a large binary array
         arrow_schema::Field::new(
             POSTING_COL,
             datatypes::DataType::List(Arc::new(Field::new(
@@ -928,19 +1312,83 @@ pub fn inverted_list_schema(with_position: bool) -> SchemaRef {
     Arc::new(arrow_schema::Schema::new(fields))
 }
 
-/// Flatten the string list stream into a string stream
-pub struct FlattenStream {
-    /// Inner record batch stream with 2 columns:
-    /// 1. doc_col: List(Utf8) or List(LargeUtf8)
-    /// 2. row_id_col: UInt64
-    inner: SendableRecordBatchStream,
-    field_type: DataType,
-    data_type: DataType,
+pub fn inverted_list_schema_with_tail_codec(
+    with_position: bool,
+    posting_tail_codec: PostingTailCodec,
+) -> SchemaRef {
+    inverted_list_schema_with_tail_codec_and_position_codec(
+        with_position,
+        posting_tail_codec,
+        Some(PositionStreamCodec::PackedDelta),
+    )
 }
 
-impl FlattenStream {
-    pub fn new(input: SendableRecordBatchStream) -> Self {
-        let schema = input.schema();
+fn inverted_list_schema_with_tail_codec_and_position_codec(
+    with_position: bool,
+    posting_tail_codec: PostingTailCodec,
+    position_codec: Option<PositionStreamCodec>,
+) -> SchemaRef {
+    let mut fields = vec![
+        // we compress the posting lists (including row ids and frequencies),
+        // and store the compressed posting lists, so it's a large binary array
+        arrow_schema::Field::new(
+            POSTING_COL,
+            datatypes::DataType::List(Arc::new(Field::new(
+                "item",
+                datatypes::DataType::LargeBinary,
+                true,
+            ))),
+            false,
+        ),
+        arrow_schema::Field::new(MAX_SCORE_COL, datatypes::DataType::Float32, false),
+        arrow_schema::Field::new(LENGTH_COL, datatypes::DataType::UInt32, false),
+    ];
+    if with_position {
+        fields.push(arrow_schema::Field::new(
+            COMPRESSED_POSITION_COL,
+            arrow_schema::DataType::LargeBinary,
+            false,
+        ));
+        fields.push(arrow_schema::Field::new(
+            POSITION_BLOCK_OFFSET_COL,
+            arrow_schema::DataType::List(Arc::new(arrow_schema::Field::new(
+                "item",
+                arrow_schema::DataType::UInt32,
+                true,
+            ))),
+            false,
+        ));
+    }
+    let mut metadata = HashMap::from([(
+        POSTING_TAIL_CODEC_KEY.to_owned(),
+        posting_tail_codec.as_str().to_owned(),
+    )]);
+    if let Some(position_codec) = position_codec.filter(|_| with_position) {
+        metadata.insert(
+            POSITIONS_LAYOUT_KEY.to_owned(),
+            POSITIONS_LAYOUT_SHARED_STREAM_V2.to_owned(),
+        );
+        metadata.insert(
+            POSITIONS_CODEC_KEY.to_owned(),
+            position_codec.as_str().to_owned(),
+        );
+    }
+    Arc::new(arrow_schema::Schema::new_with_metadata(fields, metadata))
+}
+
+/// Flatten the string list stream into a string stream
+pub struct FlattenStream {
+    /// Inner record batch stream with 2 columns:
+    /// 1. doc_col: List(Utf8) or List(LargeUtf8)
+    /// 2. row_id_col: UInt64
+    inner: SendableRecordBatchStream,
+    field_type: DataType,
+    data_type: DataType,
+}
+
+impl FlattenStream {
+    pub fn new(input: SendableRecordBatchStream) -> Self {
+        let schema = input.schema();
         let field = schema.field(0);
         let data_type = match field.data_type() {
             DataType::List(f) if matches!(f.data_type(), DataType::Utf8) => DataType::Utf8,
@@ -1067,16 +1515,23 @@ pub(crate) fn part_metadata_file_path(partition_id: u64) -> String {
     format!("part_{}_{}", partition_id, METADATA_FILE)
 }
 
+const PARTITION_FILE_SUFFIXES: [&str; 3] = [TOKENS_FILE, INVERT_LIST_FILE, DOCS_FILE];
+// Each remapped file is renamed twice: first to a temp path (phase 1), then to
+// its final path (phase 2). Keep in sync with the two rename loops below in
+// `merge_metadata_files`.
+const PARTITION_FILE_RENAME_PHASES: u64 = 2;
+
 pub async fn merge_index_files(
     object_store: &ObjectStore,
     index_dir: &Path,
     store: Arc<dyn IndexStore>,
+    progress: Arc<dyn IndexBuildProgress>,
 ) -> Result<()> {
     // List all partition metadata files in the index directory
     let part_metadata_files = list_metadata_files(object_store, index_dir).await?;
 
     // Call merge_metadata_files function for inverted index
-    merge_metadata_files(store, &part_metadata_files).await
+    merge_metadata_files(store, &part_metadata_files, progress).await
 }
 
 /// List and filter metadata files from the index directory
@@ -1116,13 +1571,24 @@ async fn list_metadata_files(object_store: &ObjectStore, index_dir: &Path) -> Re
 async fn merge_metadata_files(
     store: Arc<dyn IndexStore>,
     part_metadata_files: &[String],
+    progress: Arc<dyn IndexBuildProgress>,
 ) -> Result<()> {
     // Collect all partition IDs and params
     let mut all_partitions = Vec::new();
     let mut params = None;
     let mut token_set_format = None;
+    let mut format_version = None;
+    let mut posting_tail_codec = None;
+    let mut deleted_fragments = RoaringBitmap::new();
+    progress
+        .stage_start(
+            "read_partition_metadata",
+            Some(part_metadata_files.len() as u64),
+            "files",
+        )
+        .await?;
 
-    for file_name in part_metadata_files {
+    for (idx, file_name) in part_metadata_files.iter().enumerate() {
         let reader = store.open_index_file(file_name).await?;
         let metadata = &reader.schema().metadata;
 
@@ -1151,7 +1617,31 @@ async fn merge_metadata_files(
         {
             token_set_format = Some(TokenSetFormat::from_str(name)?);
         }
+        if format_version.is_none() {
+            format_version = Some(parse_format_version_from_metadata(metadata)?);
+        }
+        if posting_tail_codec.is_none() {
+            posting_tail_codec = Some(parse_posting_tail_codec(metadata)?);
+        }
+
+        if reader.num_rows() > 0 {
+            let metadata_batch = reader.read_range(0..1, None).await?;
+            let deleted_fragments_col = metadata_batch
+                .column_by_name(DELETED_FRAGMENTS_COL)
+                .expect_ok()?;
+            let deleted_fragments_arr = deleted_fragments_col
+                .as_any()
+                .downcast_ref::<BinaryArray>()
+                .expect_ok()?;
+            let part_deleted_fragments =
+                RoaringBitmap::deserialize_from(deleted_fragments_arr.value(0))?;
+            deleted_fragments.extend(part_deleted_fragments);
+        }
+        progress
+            .stage_progress("read_partition_metadata", idx as u64 + 1)
+            .await?;
     }
+    progress.stage_complete("read_partition_metadata").await?;
 
     // Create ID mapping: sorted original IDs -> 0,1,2...
     let mut sorted_ids = all_partitions.clone();
@@ -1170,12 +1660,24 @@ async fn merge_metadata_files(
         .unwrap()
         .as_secs();
 
+    let changed_partition_count = id_mapping
+        .iter()
+        .filter(|(old_id, new_id)| old_id != new_id)
+        .count() as u64;
+    let total_renames = changed_partition_count
+        * PARTITION_FILE_SUFFIXES.len() as u64
+        * PARTITION_FILE_RENAME_PHASES;
+    progress
+        .stage_start("remap_partition_files", Some(total_renames), "files")
+        .await?;
+
     // Phase 1: Move files to temporary locations
     let mut temp_files: Vec<(String, String, String)> = Vec::new(); // (temp_path, old_path, final_path)
+    let mut renamed_files = 0u64;
 
     for (&old_id, &new_id) in &id_mapping {
         if old_id != new_id {
-            for suffix in [TOKENS_FILE, INVERT_LIST_FILE, DOCS_FILE] {
+            for suffix in PARTITION_FILE_SUFFIXES {
                 let old_path = format!("part_{}_{}", old_id, suffix);
                 let new_path = format!("part_{}_{}", new_id, suffix);
                 let temp_path = format!("temp_{}_{}", timestamp, old_path);
@@ -1192,6 +1694,10 @@ async fn merge_metadata_files(
                     )));
                 }
                 temp_files.push((temp_path, old_path, new_path));
+                renamed_files += 1;
+                progress
+                    .stage_progress("remap_partition_files", renamed_files)
+                    .await?;
             }
         }
     }
@@ -1217,7 +1723,12 @@ async fn merge_metadata_files(
             )));
         }
         completed_renames.push((final_path.clone(), temp_path.clone()));
+        renamed_files += 1;
+        progress
+            .stage_progress("remap_partition_files", renamed_files)
+            .await?;
     }
+    progress.stage_complete("remap_partition_files").await?;
 
     // Write merged metadata with remapped IDs
     let remapped_partitions: Vec<u64> = (0..id_mapping.len() as u64).collect();
@@ -1229,10 +1740,18 @@ async fn merge_metadata_files(
         remapped_partitions.clone(),
         token_set_format,
         None,
-    );
+        deleted_fragments,
+    )
+    .with_format_version(format_version.unwrap_or(InvertedListFormatVersion::V1))
+    .with_posting_tail_codec(posting_tail_codec.unwrap_or(PostingTailCodec::Fixed32));
+    progress
+        .stage_start("write_merged_metadata", Some(1), "files")
+        .await?;
     builder
         .write_metadata(&*store, &remapped_partitions)
         .await?;
+    progress.stage_progress("write_merged_metadata", 1).await?;
+    progress.stage_complete("write_merged_metadata").await?;
 
     // Cleanup partition metadata files
     for file_name in part_metadata_files {
@@ -1287,7 +1806,7 @@ mod tests {
     use super::*;
     use crate::metrics::NoOpMetricsCollector;
     use crate::progress::IndexBuildProgress;
-    use crate::scalar::{IndexReader, IndexWriter};
+    use crate::scalar::{IndexFile, IndexReader, IndexWriter, ScalarIndex};
     use arrow_array::{RecordBatch, StringArray, UInt64Array};
     use arrow_schema::{DataType, Field, Schema};
     use async_trait::async_trait;
@@ -1299,7 +1818,6 @@ mod tests {
     use std::any::Any;
     use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
     use std::time::Duration;
-    use tokio::sync::Mutex;
 
     fn make_doc_batch(doc: &str, row_id: u64) -> RecordBatch {
         let schema = Arc::new(Schema::new(vec![
@@ -1311,7 +1829,7 @@ mod tests {
         RecordBatch::try_new(schema, vec![docs, row_ids]).unwrap()
     }
 
-    #[derive(Debug, Default)]
+    #[derive(Debug, Default, Clone)]
     struct CountingStore {
         write_count: Arc<AtomicUsize>,
     }
@@ -1360,6 +1878,10 @@ mod tests {
             self
         }
 
+        fn clone_arc(&self) -> Arc<dyn IndexStore> {
+            Arc::new(self.clone())
+        }
+
         fn io_parallelism(&self) -> usize {
             1
         }
@@ -1397,10 +1919,14 @@ mod tests {
                 "CountingStore does not support deleting",
             ))
         }
+
+        async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>> {
+            Ok(vec![])
+        }
     }
 
     #[tokio::test]
-    async fn test_write_posting_lists_writes_each_batch() -> Result<()> {
+    async fn test_write_posting_lists_batches_multiple_rows() -> Result<()> {
         let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
         for doc_id in 0..3u64 {
             builder.docs.append(doc_id, 1);
@@ -1416,12 +1942,12 @@ mod tests {
         let docs = Arc::new(std::mem::take(&mut builder.docs));
         builder.write_posting_lists(&store, docs).await?;
 
-        assert_eq!(store.write_count(), 3);
+        assert_eq!(store.write_count(), 1);
         Ok(())
     }
 
     #[tokio::test]
-    async fn test_skip_merge_writes_partitions_as_is() -> Result<()> {
+    async fn test_build_only_path_writes_partitions_as_is() -> Result<()> {
         let src_dir = TempDir::default();
         let dest_dir = TempDir::default();
         let src_store = Arc::new(LanceIndexStore::new(
@@ -1441,42 +1967,61 @@ mod tests {
         let id_alloc = Arc::new(AtomicU64::new(0));
 
         let mut worker1 = IndexWorker::new(
-            src_store.clone(),
             tokenizer.clone(),
-            params.with_position,
+            src_store.clone(),
             id_alloc.clone(),
-            None,
-            token_set_format,
+            IndexWorkerConfig {
+                with_position: params.with_position,
+                format_version: InvertedListFormatVersion::V1,
+                fragment_mask: None,
+                token_set_format,
+                worker_memory_limit_bytes: u64::MAX,
+            },
         )
         .await?;
         worker1
             .process_batch(make_doc_batch("hello world", 0))
             .await?;
-        let mut partitions = worker1.finish().await?;
+        let output1 = worker1.finish().await?;
+        let mut partitions = output1.partitions;
+        if let Some(mut tail_partition) = output1.tail_partition {
+            partitions.push(tail_partition.builder.id());
+            tail_partition.builder.write(src_store.as_ref()).await?;
+        }
 
         let mut worker2 = IndexWorker::new(
-            src_store.clone(),
             tokenizer.clone(),
-            params.with_position,
+            src_store.clone(),
             id_alloc.clone(),
-            None,
-            token_set_format,
+            IndexWorkerConfig {
+                with_position: params.with_position,
+                format_version: InvertedListFormatVersion::V1,
+                fragment_mask: None,
+                token_set_format,
+                worker_memory_limit_bytes: u64::MAX,
+            },
         )
         .await?;
         worker2
             .process_batch(make_doc_batch("goodbye world", 1))
             .await?;
-        partitions.extend(worker2.finish().await?);
+        let output2 = worker2.finish().await?;
+        partitions.extend(output2.partitions);
+        if let Some(mut tail_partition) = output2.tail_partition {
+            partitions.push(tail_partition.builder.id());
+            tail_partition.builder.write(src_store.as_ref()).await?;
+        }
         partitions.sort_unstable();
         assert_eq!(partitions.len(), 2);
         assert_ne!(partitions[0], partitions[1]);
 
         let builder = InvertedIndexBuilder::from_existing_index(
-            InvertedIndexParams::default().skip_merge(true),
+            InvertedIndexParams::default(),
             Some(src_store.clone()),
             partitions.clone(),
             token_set_format,
             None,
+            RoaringBitmap::new(),
         );
         builder.write(dest_store.as_ref()).await?;
 
@@ -1497,6 +2042,101 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn test_update_preserves_existing_posting_tail_codec() -> Result<()> {
+        let src_dir = TempDir::default();
+        let dest_dir = TempDir::default();
+        let src_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            src_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let dest_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            dest_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let posting_tail_codec = PostingTailCodec::Fixed32;
+        let mut partition = InnerBuilder::new_with_posting_tail_codec(
+            0,
+            false,
+            TokenSetFormat::default(),
+            posting_tail_codec,
+        );
+        partition.tokens.add("hello".to_owned());
+        let mut posting_list =
+            PostingListBuilder::new_with_posting_tail_codec(false, posting_tail_codec);
+        posting_list.add(0, PositionRecorder::Count(1));
+        partition.posting_lists.push(posting_list);
+        partition.docs.append(100, 1);
+        partition.write(src_store.as_ref()).await?;
+
+        let metadata_writer = InvertedIndexBuilder::from_existing_index(
+            InvertedIndexParams::default(),
+            Some(src_store.clone()),
+            vec![0],
+            TokenSetFormat::default(),
+            None,
+            RoaringBitmap::new(),
+        )
+        .with_posting_tail_codec(posting_tail_codec);
+        metadata_writer
+            .write_metadata(src_store.as_ref(), &[0])
+            .await?;
+
+        let index = InvertedIndex::load(src_store, None, &LanceCache::no_cache()).await?;
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("doc", DataType::Utf8, true),
+            Field::new(ROW_ID, DataType::UInt64, false),
+        ]));
+        let docs = Arc::new(StringArray::from(vec![Some("hello again")]));
+        let row_ids = Arc::new(UInt64Array::from(vec![101u64]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![docs, row_ids])?;
+        let stream = RecordBatchStreamAdapter::new(schema, stream::iter(vec![Ok(batch)]));
+        index
+            .update(Box::pin(stream), dest_store.as_ref(), None)
+            .await?;
+
+        let updated =
+            InvertedIndex::load(dest_store.clone(), None, &LanceCache::no_cache()).await?;
+        assert_eq!(updated.partitions.len(), 2);
+        for partition in &updated.partitions {
+            assert_eq!(
+                partition.inverted_list.posting_tail_codec(),
+                posting_tail_codec
+            );
+        }
+
+        let metadata = dest_store.open_index_file(METADATA_FILE).await?;
+        assert_eq!(
+            metadata.schema().metadata.get(POSTING_TAIL_CODEC_KEY),
+            Some(&posting_tail_codec.as_str().to_owned())
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_with_posting_tail_codec_syncs_format_version() {
+        let builder = InvertedIndexBuilder::from_existing_index(
+            InvertedIndexParams::default(),
+            None,
+            Vec::new(),
+            TokenSetFormat::default(),
+            None,
+            RoaringBitmap::new(),
+        )
+        .with_format_version(InvertedListFormatVersion::V2)
+        .with_posting_tail_codec(PostingTailCodec::Fixed32);
+        assert_eq!(builder.format_version, InvertedListFormatVersion::V1);
+        assert_eq!(builder.posting_tail_codec, PostingTailCodec::Fixed32);
+
+        let builder = builder.with_posting_tail_codec(PostingTailCodec::VarintDelta);
+        assert_eq!(builder.format_version, InvertedListFormatVersion::V2);
+        assert_eq!(builder.posting_tail_codec, PostingTailCodec::VarintDelta);
+    }
+
     #[tokio::test]
     async fn test_inverted_index_without_positions_tracks_frequency() -> Result<()> {
         let index_dir = TempDir::default();
@@ -1516,17 +2156,15 @@ mod tests {
         let stream = RecordBatchStreamAdapter::new(schema, stream::iter(vec![Ok(batch)]));
         let stream = Box::pin(stream);
 
-        let params = InvertedIndexParams::new(
-            "whitespace".to_string(),
-            tantivy::tokenizer::Language::English,
-        )
-        .with_position(false)
-        .remove_stop_words(false)
-        .stem(false)
-        .max_token_length(None);
+        let params =
+            InvertedIndexParams::new("whitespace".to_string(), lance_tokenizer::Language::English)
+                .with_position(false)
+                .remove_stop_words(false)
+                .stem(false)
+                .max_token_length(None);
 
         let mut builder = InvertedIndexBuilder::new(params);
-        builder.update(stream, store.as_ref()).await?;
+        builder.update(stream, store.as_ref(), None).await?;
 
         let index = InvertedIndex::load(store, None, &LanceCache::no_cache()).await?;
         assert_eq!(index.partitions.len(), 1);
@@ -1547,38 +2185,7 @@ mod tests {
         Ok(())
     }
 
-    #[derive(Debug, Default)]
-    struct RecordingProgress {
-        events: Mutex<Vec<(String, String, u64)>>,
-    }
-
-    #[async_trait]
-    impl IndexBuildProgress for RecordingProgress {
-        async fn stage_start(&self, stage: &str, total: Option<u64>, _unit: &str) -> Result<()> {
-            self.events.lock().await.push((
-                "start".to_string(),
-                stage.to_string(),
-                total.unwrap_or(0),
-            ));
-            Ok(())
-        }
-
-        async fn stage_progress(&self, stage: &str, completed: u64) -> Result<()> {
-            self.events
-                .lock()
-                .await
-                .push(("progress".to_string(), stage.to_string(), completed));
-            Ok(())
-        }
-
-        async fn stage_complete(&self, stage: &str) -> Result<()> {
-            self.events
-                .lock()
-                .await
-                .push(("complete".to_string(), stage.to_string(), 0));
-            Ok(())
-        }
-    }
+    lance_testing::define_stage_event_progress!(RecordingProgress, IndexBuildProgress, Result<()>);
 
     #[derive(Debug, Default)]
     struct FailingProgress;
@@ -1617,12 +2224,11 @@ mod tests {
         let stream = Box::pin(stream);
 
         let progress = Arc::new(RecordingProgress::default());
-        let mut builder =
-            InvertedIndexBuilder::new(InvertedIndexParams::default().skip_merge(true))
-                .with_progress(progress.clone());
-        builder.update(stream, store.as_ref()).await?;
+        let mut builder = InvertedIndexBuilder::new(InvertedIndexParams::default())
+            .with_progress(progress.clone());
+        builder.update(stream, store.as_ref(), None).await?;
 
-        let events = progress.events.lock().await.clone();
+        let events = progress.recorded_events();
         let tags = events
             .iter()
             .map(|(kind, stage, _)| format!("{kind}:{stage}"))
@@ -1692,15 +2298,369 @@ mod tests {
         );
         assert!(
             !tags.iter().any(|e| e == "start:merge_partitions"),
-            "merge_partitions should not run in skip_merge mode"
+            "merge_partitions should not run in the build-only path"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_builder_default_path_skips_merge_stage() -> Result<()> {
+        let index_dir = TempDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let batch = make_doc_batch("hello world", 0);
+        let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)]));
+        let stream = Box::pin(stream);
+
+        let progress = Arc::new(RecordingProgress::default());
+        let mut builder = InvertedIndexBuilder::new(InvertedIndexParams::default())
+            .with_progress(progress.clone());
+        builder.update(stream, store.as_ref(), None).await?;
+
+        let tags = progress
+            .recorded_events()
+            .iter()
+            .map(|(kind, stage, _)| format!("{kind}:{stage}"))
+            .collect::<Vec<_>>();
+
+        assert!(
+            tags.iter().any(|e| e == "start:copy_partitions"),
+            "default path should copy finalized partitions"
+        );
+        assert!(
+            !tags.iter().any(|e| e == "start:merge_partitions"),
+            "default path should not run merge_partitions"
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_merge_index_files_reports_progress_stages() -> Result<()> {
+        let index_dir = TempDir::default();
+        let index_path = index_dir.obj_path();
+        let object_store = ObjectStore::local();
+        let store = Arc::new(LanceIndexStore::new(
+            object_store.clone().into(),
+            index_path.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        for (fragment_id, row_id, doc) in [
+            (1_u64 << 32, 0_u64, "hello world"),
+            (2_u64 << 32, 1_u64, "goodbye world"),
+        ] {
+            let batch = make_doc_batch(doc, row_id);
+            let stream =
+                RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)]));
+            let stream = Box::pin(stream);
+            let mut builder = InvertedIndexBuilder::new_with_fragment_mask(
+                InvertedIndexParams::default(),
+                Some(fragment_id),
+            )
+            .with_progress(noop_progress());
+            builder.update(stream, store.as_ref(), None).await?;
+        }
+
+        let progress = Arc::new(RecordingProgress::default());
+        merge_index_files(&object_store, &index_path, store.clone(), progress.clone()).await?;
+
+        let events = progress.recorded_events();
+        let tags = events
+            .iter()
+            .map(|(kind, stage, _)| format!("{kind}:{stage}"))
+            .collect::<Vec<_>>();
+        let remap_progress = events
+            .iter()
+            .filter_map(|(kind, stage, completed)| {
+                if kind == "progress" && stage == "remap_partition_files" {
+                    Some(*completed)
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>();
+
+        let read_start = tags
+            .iter()
+            .position(|e| e == "start:read_partition_metadata")
+            .expect("missing read_partition_metadata start");
+        let read_complete = tags
+            .iter()
+            .position(|e| e == "complete:read_partition_metadata")
+            .expect("missing read_partition_metadata complete");
+        let remap_start = tags
+            .iter()
+            .position(|e| e == "start:remap_partition_files")
+            .expect("missing remap_partition_files start");
+        let remap_complete = tags
+            .iter()
+            .position(|e| e == "complete:remap_partition_files")
+            .expect("missing remap_partition_files complete");
+        let metadata_start = tags
+            .iter()
+            .position(|e| e == "start:write_merged_metadata")
+            .expect("missing write_merged_metadata start");
+        let metadata_complete = tags
+            .iter()
+            .position(|e| e == "complete:write_merged_metadata")
+            .expect("missing write_merged_metadata complete");
+
+        assert!(read_start < read_complete);
+        assert!(read_complete < remap_start);
+        assert!(remap_start < remap_complete);
+        assert!(remap_complete < metadata_start);
+        assert!(metadata_start < metadata_complete);
+
+        assert!(
+            tags.iter().any(|e| e == "progress:read_partition_metadata"),
+            "expected progress callback for read_partition_metadata"
+        );
+        assert_eq!(
+            remap_progress.last().copied().unwrap_or_default(),
+            12,
+            "expected remap_partition_files progress to cover both rename phases"
+        );
+        assert!(
+            tags.iter().any(|e| e == "progress:write_merged_metadata"),
+            "expected progress callback for write_merged_metadata"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_worker_memory_limit_rejects_single_large_doc() {
+        let index_dir = TempDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let batch = make_doc_batch("hello world", 42);
+        let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)]));
+        let stream = Box::pin(stream);
+
+        let mut builder =
+            InvertedIndexBuilder::new(InvertedIndexParams::default().memory_limit_mb(0));
+        let err = builder
+            .update(stream, store.as_ref(), None)
+            .await
+            .expect_err("single doc should exceed zero worker memory limit");
+        assert!(
+            err.to_string().contains("row_id=42"),
+            "unexpected error: {err}"
         );
+    }
+
+    #[tokio::test]
+    async fn test_worker_trims_position_temp_buffers() -> Result<()> {
+        let tokenizer = InvertedIndexParams::default().with_position(true).build()?;
+        let store = Arc::new(CountingStore::new());
+        let id_alloc = Arc::new(AtomicU64::new(0));
+        let mut worker = IndexWorker::new(
+            tokenizer,
+            store,
+            id_alloc,
+            IndexWorkerConfig {
+                with_position: true,
+                format_version: InvertedListFormatVersion::V1,
+                fragment_mask: None,
+                token_set_format: TokenSetFormat::default(),
+                worker_memory_limit_bytes: u64::MAX,
+            },
+        )
+        .await?;
+
+        let doc = (0..(MAX_RETAINED_TOKEN_IDS * 2))
+            .map(|i| format!("tok{i}"))
+            .collect::<Vec<_>>()
+            .join(" ");
+        worker.process_batch(make_doc_batch(&doc, 0)).await?;
+
+        assert!(worker.token_ids.is_empty());
+        assert!(worker.token_ids.capacity() <= MAX_RETAINED_TOKEN_IDS);
+        assert!(worker.memory_size >= worker.temporary_memory_size());
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_worker_flush_keeps_position_temp_memory_bounded() -> Result<()> {
+        let tokenizer = InvertedIndexParams::default().with_position(true).build()?;
+        let store = Arc::new(CountingStore::new());
+        let id_alloc = Arc::new(AtomicU64::new(0));
+        let mut worker = IndexWorker::new(
+            tokenizer,
+            store,
+            id_alloc,
+            IndexWorkerConfig {
+                with_position: true,
+                format_version: InvertedListFormatVersion::V1,
+                fragment_mask: None,
+                token_set_format: TokenSetFormat::default(),
+                worker_memory_limit_bytes: u64::MAX,
+            },
+        )
+        .await?;
 
+        let doc = std::iter::repeat_n("common", 32_768)
+            .collect::<Vec<_>>()
+            .join(" ");
+        let mut observed_post_flush_memory = Vec::new();
+        for row_id in 0..8 {
+            worker.process_batch(make_doc_batch(&doc, row_id)).await?;
+            worker.flush().await?;
+            observed_post_flush_memory.push(worker.memory_size);
+        }
+
+        let max_memory = *observed_post_flush_memory.iter().max().unwrap();
+        let min_memory = *observed_post_flush_memory.iter().min().unwrap();
+        assert!(
+            max_memory <= min_memory.saturating_add(256 * 1024),
+            "post-flush worker memory drifted upward: {observed_post_flush_memory:?}"
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_worker_flush_writes_partition_directly() -> Result<()> {
+        let tokenizer = InvertedIndexParams::default().with_position(true).build()?;
+        let store = Arc::new(CountingStore::new());
+        let id_alloc = Arc::new(AtomicU64::new(0));
+        let mut worker = IndexWorker::new(
+            tokenizer,
+            store.clone(),
+            id_alloc,
+            IndexWorkerConfig {
+                with_position: true,
+                format_version: InvertedListFormatVersion::V1,
+                fragment_mask: None,
+                token_set_format: TokenSetFormat::default(),
+                worker_memory_limit_bytes: u64::MAX,
+            },
+        )
+        .await?;
+        worker
+            .process_batch(make_doc_batch("alpha beta gamma", 0))
+            .await?;
+        worker.flush().await?;
+        assert!(store.write_count() > 0);
+        Ok(())
+    }
+
+    #[test]
+    fn test_resolve_worker_memory_limit_uses_default_when_unset() {
+        let params = InvertedIndexParams::default();
+        assert_eq!(
+            resolve_worker_memory_limit_bytes(&params, 8),
+            *LANCE_FTS_PARTITION_SIZE << 20
+        );
+    }
+
+    #[test]
+    fn test_resolve_num_workers_uses_default_when_unset() {
+        let expected = default_num_workers().clamp(1, get_num_compute_intensive_cpus().max(1));
+        assert_eq!(
+            resolve_num_workers(&InvertedIndexParams::default()),
+            expected
+        );
+    }
+
+    #[test]
+    fn test_resolve_num_workers_clamps_requested_value() {
+        let max_workers = get_num_compute_intensive_cpus().max(1);
+        assert_eq!(
+            resolve_num_workers(&InvertedIndexParams::default().num_workers(0)),
+            1
+        );
+        assert_eq!(
+            resolve_num_workers(&InvertedIndexParams::default().num_workers(max_workers + 10)),
+            max_workers
+        );
+    }
+
+    #[test]
+    fn test_resolve_worker_memory_limit_splits_total_memory_limit() {
+        let params = InvertedIndexParams::default().memory_limit_mb(4096);
+        assert_eq!(resolve_worker_memory_limit_bytes(&params, 16), 256 << 20);
+    }
+
+    #[test]
+    fn test_merge_all_tail_partitions_combines_everything() -> Result<()> {
+        let merged = merge_all_tail_partitions(vec![
+            TailPartition {
+                builder: InnerBuilder::new(0, false, TokenSetFormat::default()),
+            },
+            TailPartition {
+                builder: InnerBuilder::new(1, false, TokenSetFormat::default()),
+            },
+            TailPartition {
+                builder: InnerBuilder::new(2, false, TokenSetFormat::default()),
+            },
+        ])?;
+
+        assert_eq!(merged.expect("merged builder should exist").id(), 0);
+        Ok(())
+    }
+
+    #[test]
+    fn test_merge_all_tail_partitions_returns_none_for_empty_input() -> Result<()> {
+        assert!(merge_all_tail_partitions(Vec::new())?.is_none());
+        Ok(())
+    }
+
+    #[test]
+    fn test_merge_tail_partition_group_combines_tail_builders() -> Result<()> {
+        let mut first = InnerBuilder::new(0, false, TokenSetFormat::default());
+        let hello = first.tokens.add("hello".to_owned());
+        first
+            .posting_lists
+            .resize_with(first.tokens.len(), || PostingListBuilder::new(false));
+        let first_doc = first.docs.append(10, 1);
+        first.posting_lists[hello as usize].add(first_doc, PositionRecorder::Count(1));
+
+        let mut second = InnerBuilder::new(1, false, TokenSetFormat::default());
+        let world = second.tokens.add("world".to_owned());
+        second
+            .posting_lists
+            .resize_with(second.tokens.len(), || PostingListBuilder::new(false));
+        let second_doc = second.docs.append(20, 2);
+        second.posting_lists[world as usize].add(second_doc, PositionRecorder::Count(2));
+
+        let merged = merge_tail_partition_group(vec![
+            TailPartition { builder: first },
+            TailPartition { builder: second },
+        ])?;
+
+        assert_eq!(merged.id(), 0);
+        assert_eq!(merged.docs.len(), 2);
+        assert_eq!(merged.tokens.len(), 2);
+        assert_eq!(merged.posting_lists.len(), 2);
+        assert_eq!(
+            merged.posting_lists[merged.tokens.get("hello").unwrap() as usize].len(),
+            1
+        );
+        assert_eq!(
+            merged.posting_lists[merged.tokens.get("world").unwrap() as usize].len(),
+            1
+        );
         Ok(())
     }
 
     #[tokio::test]
     async fn test_update_index_returns_worker_error_when_workers_exit_during_dispatch() {
         let num_batches = (*LANCE_FTS_NUM_SHARDS * 2 + 1) as u64;
+        let index_dir = TempDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
         let schema = make_doc_batch("hello world", 0).schema();
         let stream = RecordBatchStreamAdapter::new(
             schema,
@@ -1708,18 +2668,303 @@ mod tests {
         );
         let stream = Box::pin(stream);
 
-        let mut builder =
-            InvertedIndexBuilder::new(InvertedIndexParams::default().skip_merge(true))
-                .with_progress(Arc::new(FailingProgress));
+        let mut builder = InvertedIndexBuilder::new(InvertedIndexParams::default())
+            .with_progress(Arc::new(FailingProgress));
 
-        let result = tokio::time::timeout(Duration::from_secs(5), builder.update_index(stream))
-            .await
-            .expect("update_index should not hang")
-            .expect_err("worker failure should be returned");
+        let result = tokio::time::timeout(
+            Duration::from_secs(5),
+            builder.update_index(stream, store.as_ref()),
+        )
+        .await
+        .expect("update_index should not hang")
+        .expect_err("worker failure should be returned");
 
         assert!(
             result.to_string().contains("injected progress failure"),
             "unexpected error: {result}"
         );
     }
+
+    #[tokio::test]
+    async fn test_new_index_has_empty_deleted_fragments() {
+        let index_dir = TempDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let batch = make_doc_batch("hello world", 0);
+        let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)]));
+        let stream = Box::pin(stream);
+
+        let mut builder = InvertedIndexBuilder::new(InvertedIndexParams::default());
+        builder.update(stream, store.as_ref(), None).await.unwrap();
+
+        let index = InvertedIndex::load(store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        assert!(
+            index.deleted_fragments().is_empty(),
+            "new index should have empty deleted fragments, got {:?}",
+            index.deleted_fragments()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_remap_preserves_deleted_fragments() {
+        let src_dir = TempDir::default();
+        let dest_dir = TempDir::default();
+        let src_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            src_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let dest_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            dest_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // Build an initial index with some deleted fragments
+        let batch = make_doc_batch("hello world", 0);
+        let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)]));
+        let stream = Box::pin(stream);
+
+        let initial_deleted = RoaringBitmap::from_iter([5, 10, 42]);
+        let mut builder = InvertedIndexBuilder::from_existing_index(
+            InvertedIndexParams::default(),
+            None,
+            Vec::new(),
+            TokenSetFormat::default(),
+            None,
+            initial_deleted.clone(),
+        );
+        builder
+            .update(stream, src_store.as_ref(), None)
+            .await
+            .unwrap();
+
+        // Load it back and confirm the invalidated fragments are set
+        let index = InvertedIndex::load(src_store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        assert_eq!(index.deleted_fragments(), &initial_deleted);
+
+        // Remap the index via the ScalarIndex trait method
+        use crate::scalar::ScalarIndex;
+        let mapping = HashMap::from([(0u64, Some(50 << 32))]);
+        index.remap(&mapping, dest_store.as_ref()).await.unwrap();
+
+        // Reload from dest and verify deleted fragments are preserved
+        let remapped_index = InvertedIndex::load(dest_store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        assert_eq!(
+            remapped_index.deleted_fragments(),
+            &initial_deleted,
+            "remap should preserve deleted fragments"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_update_grows_deleted_fragments_from_old_data_filter() {
+        let index_dir = TempDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // Build an initial index with no deleted fragments
+        let batch = make_doc_batch("hello world", 0);
+        let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)]));
+        let stream = Box::pin(stream);
+
+        let mut builder = InvertedIndexBuilder::new(InvertedIndexParams::default());
+        builder.update(stream, store.as_ref(), None).await.unwrap();
+
+        // Load the index and update it with an old_data_filter that invalidates fragments
+        let index = InvertedIndex::load(store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        assert!(index.deleted_fragments().is_empty());
+
+        let update_dir = TempDir::default();
+        let update_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            update_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let batch2 = make_doc_batch("new document", 1 << 32 | 1);
+        let stream2 =
+            RecordBatchStreamAdapter::new(batch2.schema(), stream::iter(vec![Ok(batch2)]));
+        let stream2 = Box::pin(stream2);
+
+        let old_data_filter = Some(crate::scalar::OldIndexDataFilter::Fragments {
+            to_keep: RoaringBitmap::from_iter([0]),
+            to_remove: RoaringBitmap::from_iter([3, 7]),
+        });
+
+        // Use ScalarIndex::update trait method
+        use crate::scalar::ScalarIndex;
+        index
+            .update(stream2, update_store.as_ref(), old_data_filter)
+            .await
+            .unwrap();
+
+        let updated_index =
+            InvertedIndex::load(update_store.clone(), None, &LanceCache::no_cache())
+                .await
+                .unwrap();
+        assert_eq!(
+            updated_index.deleted_fragments(),
+            &RoaringBitmap::from_iter([3, 7]),
+            "update should add deleted fragments from old_data_filter"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_update_accumulates_deleted_fragments() {
+        let dir1 = TempDir::default();
+        let store1 = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            dir1.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // Build initial index
+        let batch = make_doc_batch("hello world", 0);
+        let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)]));
+        let stream = Box::pin(stream);
+
+        let mut builder = InvertedIndexBuilder::new(InvertedIndexParams::default());
+        builder.update(stream, store1.as_ref(), None).await.unwrap();
+
+        // First update: delete fragments 3 and 7
+        let index = InvertedIndex::load(store1.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        let dir2 = TempDir::default();
+        let store2 = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            dir2.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let batch2 = make_doc_batch("second doc", 1 << 32 | 1);
+        let stream2 =
+            RecordBatchStreamAdapter::new(batch2.schema(), stream::iter(vec![Ok(batch2)]));
+        let stream2 = Box::pin(stream2);
+
+        use crate::scalar::ScalarIndex;
+        index
+            .update(
+                stream2,
+                store2.as_ref(),
+                Some(crate::scalar::OldIndexDataFilter::Fragments {
+                    to_keep: RoaringBitmap::from_iter([0]),
+                    to_remove: RoaringBitmap::from_iter([3, 7]),
+                }),
+            )
+            .await
+            .unwrap();
+
+        // Second update: invalidate additional fragments 12 and 15
+        let index2 = InvertedIndex::load(store2.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        assert_eq!(
+            index2.deleted_fragments(),
+            &RoaringBitmap::from_iter([3, 7])
+        );
+
+        let dir3 = TempDir::default();
+        let store3 = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            dir3.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let batch3 = make_doc_batch("third doc", 2 << 32 | 2);
+        let stream3 =
+            RecordBatchStreamAdapter::new(batch3.schema(), stream::iter(vec![Ok(batch3)]));
+        let stream3 = Box::pin(stream3);
+
+        index2
+            .update(
+                stream3,
+                store3.as_ref(),
+                Some(crate::scalar::OldIndexDataFilter::Fragments {
+                    to_keep: RoaringBitmap::from_iter([0, 1]),
+                    to_remove: RoaringBitmap::from_iter([12, 15]),
+                }),
+            )
+            .await
+            .unwrap();
+
+        let index3 = InvertedIndex::load(store3.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        assert_eq!(
+            index3.deleted_fragments(),
+            &RoaringBitmap::from_iter([3, 7, 12, 15]),
+            "deleted fragments should accumulate across updates"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_update_with_rowid_filter_does_not_grow_deleted_fragments() {
+        let index_dir = TempDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let batch = make_doc_batch("hello world", 0);
+        let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)]));
+        let stream = Box::pin(stream);
+
+        let mut builder = InvertedIndexBuilder::new(InvertedIndexParams::default());
+        builder.update(stream, store.as_ref(), None).await.unwrap();
+
+        let index = InvertedIndex::load(store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        let update_dir = TempDir::default();
+        let update_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            update_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let batch2 = make_doc_batch("new doc", 1);
+        let stream2 =
+            RecordBatchStreamAdapter::new(batch2.schema(), stream::iter(vec![Ok(batch2)]));
+        let stream2 = Box::pin(stream2);
+
+        // Use RowIds filter instead of Fragments — should not affect deleted_fragments
+        let mut valid_ids = lance_core::utils::mask::RowAddrTreeMap::new();
+        valid_ids.insert(0);
+        let old_data_filter = Some(crate::scalar::OldIndexDataFilter::RowIds(valid_ids));
+
+        use crate::scalar::ScalarIndex;
+        index
+            .update(stream2, update_store.as_ref(), old_data_filter)
+            .await
+            .unwrap();
+
+        let updated_index =
+            InvertedIndex::load(update_store.clone(), None, &LanceCache::no_cache())
+                .await
+                .unwrap();
+        assert!(
+            updated_index.deleted_fragments().is_empty(),
+            "RowIds filter should not add to deleted fragments"
+        );
+    }
 }
diff --git a/rust/lance-index/src/scalar/inverted/encoding.rs b/rust/lance-index/src/scalar/inverted/encoding.rs
index 57bc80cda66..a25d7cfc4c9 100644
--- a/rust/lance-index/src/scalar/inverted/encoding.rs
+++ b/rust/lance-index/src/scalar/inverted/encoding.rs
@@ -4,11 +4,10 @@
 use std::io::Write;
 
 use super::builder::BLOCK_SIZE;
-use arrow::array::{AsArray, LargeBinaryBuilder};
-use arrow::array::{ListBuilder, UInt32Builder};
-use arrow_array::{Array, ListArray};
+use super::index::{PositionStreamCodec, PostingTailCodec};
+use arrow::array::LargeBinaryBuilder;
 use bitpacking::{BitPacker, BitPacker4x};
-use lance_core::Result;
+use lance_core::{Error, Result};
 
 // we compress the posting list to multiple blocks of fixed number of elements (BLOCK_SIZE),
 // returns a LargeBinaryArray, where each binary is a compressed block (128 row ids + 128 frequencies)
@@ -19,20 +18,34 @@ use lance_core::Result;
 // - n bytes for the packed doc ids
 // - 1 byte for the number of bits used to pack the frequencies
 // - n bytes for the packed frequencies
-// if the block is not full (the last block), we don't compress it
-// we directly write the remainder to the buffer with the format:
-// - 4 bytes for the max block score
-// - 4*n bytes for the doc ids
-// - 4*n bytes for the frequencies
-// where n is the number of elements in the block
+// if the block is not full (the last block), we encode the remainder separately
+// using the configured remainder codec.
 
 // compress the posting list to multiple blocks of fixed number of elements (BLOCK_SIZE),
 // returns a LargeBinaryArray, where each binary is a compressed block (128 row ids + 128 frequencies)
+#[cfg(test)]
 pub fn compress_posting_list<'a>(
+    length: usize,
+    doc_ids: impl Iterator<Item = &'a u32>,
+    frequencies: impl Iterator<Item = &'a u32>,
+    block_max_scores: impl Iterator<Item = f32>,
+) -> Result<arrow::array::LargeBinaryArray> {
+    compress_posting_list_with_tail_codec(
+        length,
+        doc_ids,
+        frequencies,
+        block_max_scores,
+        PostingTailCodec::VarintDelta,
+    )
+}
+
+#[cfg(test)]
+pub fn compress_posting_list_with_tail_codec<'a>(
     length: usize,
     doc_ids: impl Iterator<Item = &'a u32>,
     frequencies: impl Iterator<Item = &'a u32>,
     mut block_max_scores: impl Iterator<Item = f32>,
+    tail_codec: PostingTailCodec,
 ) -> Result<arrow::array::LargeBinaryArray> {
     if length < BLOCK_SIZE {
         // directly do remainder compression to avoid overhead of creating buffer
@@ -40,12 +53,10 @@ pub fn compress_posting_list<'a>(
         // write the max score of the block
         let max_score = block_max_scores.next().unwrap();
         let _ = builder.write(max_score.to_le_bytes().as_ref())?;
-        compress_remainder(
+        compress_posting_remainder(
             doc_ids.copied().collect::<Vec<_>>().as_slice(),
-            &mut builder,
-        )?;
-        compress_remainder(
             frequencies.copied().collect::<Vec<_>>().as_slice(),
+            tail_codec,
             &mut builder,
         )?;
         builder.append_value("");
@@ -83,102 +94,40 @@ pub fn compress_posting_list<'a>(
         // write the max score of the block
         let max_score = block_max_scores.next().unwrap();
         let _ = builder.write(max_score.to_le_bytes().as_ref())?;
-        compress_remainder(&doc_id_buffer, &mut builder)?;
-        compress_remainder(&freq_buffer, &mut builder)?;
+        compress_posting_remainder(&doc_id_buffer, &freq_buffer, tail_codec, &mut builder)?;
         builder.append_value("");
     }
     Ok(builder.finish())
 }
 
-pub fn compress_posting_list_with_scores<'a, F>(
-    length: usize,
-    doc_ids: impl Iterator<Item = &'a u32>,
-    frequencies: impl Iterator<Item = &'a u32>,
-    mut score_for: F,
-    idf_scale: f32,
-) -> Result<(arrow::array::LargeBinaryArray, f32)>
-where
-    F: FnMut(u32, u32) -> f32,
-{
-    // `length` comes from posting list size; zero would produce an invalid block
-    // (a max-score header with no doc/frequency data) and readers assume > 0 docs.
-    debug_assert!(length > 0);
-    if length < BLOCK_SIZE {
-        let mut builder = LargeBinaryBuilder::with_capacity(1, length * 4 * 2 + 1);
-        let mut max_score = f32::MIN;
-        let mut doc_id_buffer = Vec::with_capacity(length);
-        let mut freq_buffer = Vec::with_capacity(length);
-        for (doc_id, freq) in std::iter::zip(doc_ids, frequencies) {
-            let doc_id = *doc_id;
-            let freq = *freq;
-            doc_id_buffer.push(doc_id);
-            freq_buffer.push(freq);
-            let score = score_for(doc_id, freq);
-            if score > max_score {
-                max_score = score;
-            }
-        }
-        let max_score = max_score * idf_scale;
-        let _ = builder.write(max_score.to_le_bytes().as_ref())?;
-        compress_remainder(&doc_id_buffer, &mut builder)?;
-        compress_remainder(&freq_buffer, &mut builder)?;
-        builder.append_value("");
-        return Ok((builder.finish(), max_score));
-    }
-
-    let mut builder = LargeBinaryBuilder::with_capacity(length.div_ceil(BLOCK_SIZE), length * 3);
+pub fn encode_full_posting_block_into(
+    doc_ids: &[u32],
+    frequencies: &[u32],
+    block: &mut Vec<u8>,
+) -> Result<()> {
+    debug_assert_eq!(doc_ids.len(), BLOCK_SIZE);
+    debug_assert_eq!(frequencies.len(), BLOCK_SIZE);
+    block.extend_from_slice(&0f32.to_le_bytes());
     let mut buffer = [0u8; BLOCK_SIZE * 4 + 5];
-    let mut doc_id_buffer = Vec::with_capacity(BLOCK_SIZE);
-    let mut freq_buffer = Vec::with_capacity(BLOCK_SIZE);
-    let mut max_score = f32::MIN;
-    let mut block_max_score = f32::MIN;
-    for (doc_id, freq) in std::iter::zip(doc_ids, frequencies) {
-        let doc_id = *doc_id;
-        let freq = *freq;
-        doc_id_buffer.push(doc_id);
-        freq_buffer.push(freq);
-
-        let score = score_for(doc_id, freq);
-        if score > block_max_score {
-            block_max_score = score;
-        }
-
-        if doc_id_buffer.len() < BLOCK_SIZE {
-            continue;
-        }
-
-        let block_score = block_max_score * idf_scale;
-        if block_score > max_score {
-            max_score = block_score;
-        }
-        let _ = builder.write(block_score.to_le_bytes().as_ref())?;
-        compress_sorted_block(&doc_id_buffer, &mut buffer, &mut builder)?;
-        compress_block(&freq_buffer, &mut buffer, &mut builder)?;
-        builder.append_value("");
-        doc_id_buffer.clear();
-        freq_buffer.clear();
-        block_max_score = f32::MIN;
-    }
+    compress_sorted_block(doc_ids, &mut buffer, block)?;
+    compress_block(frequencies, &mut buffer, block)?;
+    Ok(())
+}
 
-    if !doc_id_buffer.is_empty() {
-        let block_score = block_max_score * idf_scale;
-        if block_score > max_score {
-            max_score = block_score;
-        }
-        let _ = builder.write(block_score.to_le_bytes().as_ref())?;
-        compress_remainder(&doc_id_buffer, &mut builder)?;
-        compress_remainder(&freq_buffer, &mut builder)?;
-        builder.append_value("");
-    }
-    Ok((builder.finish(), max_score))
+pub fn encode_remainder_posting_block_into(
+    doc_ids: &[u32],
+    frequencies: &[u32],
+    codec: PostingTailCodec,
+    block: &mut Vec<u8>,
+) -> Result<()> {
+    debug_assert_eq!(doc_ids.len(), frequencies.len());
+    block.extend_from_slice(&0f32.to_le_bytes());
+    compress_posting_remainder(doc_ids, frequencies, codec, block)?;
+    Ok(())
 }
 
 #[inline]
-fn compress_sorted_block(
-    data: &[u32],
-    buffer: &mut [u8],
-    builder: &mut LargeBinaryBuilder,
-) -> Result<()> {
+fn compress_sorted_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Write) -> Result<()> {
     let compressor = BitPacker4x::new();
     let num_bits = compressor.num_bits_sorted(data[0], data);
     let num_bytes = compressor.compress_sorted(data[0], data, buffer, num_bits);
@@ -189,7 +138,7 @@ fn compress_sorted_block(
 }
 
 #[inline]
-fn compress_block(data: &[u32], buffer: &mut [u8], builder: &mut LargeBinaryBuilder) -> Result<()> {
+fn compress_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Write) -> Result<()> {
     let compressor = BitPacker4x::new();
     let num_bits = compressor.num_bits(data);
     let num_bytes = compressor.compress(data, buffer, num_bits);
@@ -199,13 +148,65 @@ fn compress_block(data: &[u32], buffer: &mut [u8], builder: &mut LargeBinaryBuil
 }
 
 #[inline]
-fn compress_remainder(data: &[u32], builder: &mut LargeBinaryBuilder) -> Result<()> {
+fn compress_raw_remainder(data: &[u32], builder: &mut impl Write) -> Result<()> {
     for value in data.iter() {
         let _ = builder.write(value.to_le_bytes().as_ref())?;
     }
     Ok(())
 }
 
+#[inline]
+fn write_varint_u32(builder: &mut impl Write, mut value: u32) -> Result<()> {
+    let mut bytes = [0u8; 5];
+    let mut len = 0usize;
+    while value >= 0x80 {
+        bytes[len] = (value as u8) | 0x80;
+        value >>= 7;
+        len += 1;
+    }
+    bytes[len] = value as u8;
+    len += 1;
+    let _ = builder.write(&bytes[..len])?;
+    Ok(())
+}
+
+#[inline]
+fn compress_posting_remainder(
+    doc_ids: &[u32],
+    frequencies: &[u32],
+    codec: PostingTailCodec,
+    builder: &mut impl Write,
+) -> Result<()> {
+    debug_assert_eq!(doc_ids.len(), frequencies.len());
+    match codec {
+        PostingTailCodec::Fixed32 => {
+            compress_raw_remainder(doc_ids, builder)?;
+            compress_raw_remainder(frequencies, builder)?;
+        }
+        PostingTailCodec::VarintDelta => {
+            let mut previous = 0u32;
+            for (index, &doc_id) in doc_ids.iter().enumerate() {
+                let delta = if index == 0 {
+                    doc_id
+                } else {
+                    doc_id.checked_sub(previous).ok_or_else(|| {
+                        Error::index(format!(
+                            "doc ids must be sorted within a posting tail block, got {} after {}",
+                            doc_id, previous
+                        ))
+                    })?
+                };
+                write_varint_u32(builder, delta)?;
+                previous = doc_id;
+            }
+            for &frequency in frequencies {
+                write_varint_u32(builder, frequency)?;
+            }
+        }
+    }
+    Ok(())
+}
+
 pub fn compress_positions(positions: &[u32]) -> Result<arrow::array::LargeBinaryArray> {
     let mut builder = LargeBinaryBuilder::with_capacity(
         positions.len().div_ceil(BLOCK_SIZE),
@@ -227,19 +228,394 @@ pub fn compress_positions(positions: &[u32]) -> Result<arrow::array::LargeBinary
     let length = positions.len();
     let remainder = length % BLOCK_SIZE;
     if remainder > 0 {
-        compress_remainder(&positions[length - remainder..], &mut builder)?;
+        compress_raw_remainder(&positions[length - remainder..], &mut builder)?;
         builder.append_value("");
     }
 
     Ok(builder.finish())
 }
 
+#[inline]
+fn encode_varint_u32(dst: &mut Vec<u8>, mut value: u32) {
+    while value >= 0x80 {
+        dst.push((value as u8) | 0x80);
+        value >>= 7;
+    }
+    dst.push(value as u8);
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(super) struct PositionBlockBuilder {
+    codec: PositionStreamCodec,
+    encoded_bytes: Vec<u8>,
+    pending_deltas: Vec<u32>,
+}
+
+impl Default for PositionBlockBuilder {
+    fn default() -> Self {
+        Self::new(PositionStreamCodec::PackedDelta)
+    }
+}
+
+impl PositionBlockBuilder {
+    pub(super) fn new(codec: PositionStreamCodec) -> Self {
+        Self {
+            codec,
+            encoded_bytes: Vec::new(),
+            pending_deltas: Vec::new(),
+        }
+    }
+
+    pub(super) fn size(&self) -> usize {
+        self.encoded_bytes.capacity() + self.pending_deltas.capacity() * std::mem::size_of::<u32>()
+    }
+
+    pub(super) fn append_doc_positions(&mut self, positions: &[u32]) -> Result<()> {
+        let mut previous = 0u32;
+        for (index, &position) in positions.iter().enumerate() {
+            let delta = if index == 0 {
+                position
+            } else {
+                position.checked_sub(previous).ok_or_else(|| {
+                    Error::index(format!(
+                        "positions must be sorted within a document, got {} after {}",
+                        position, previous
+                    ))
+                })?
+            };
+            self.push_delta(delta)?;
+            previous = position;
+        }
+        Ok(())
+    }
+
+    pub(super) fn append_position(
+        &mut self,
+        position: u32,
+        previous_in_doc: Option<u32>,
+    ) -> Result<()> {
+        let delta = match previous_in_doc {
+            Some(previous) => position.checked_sub(previous).ok_or_else(|| {
+                Error::index(format!(
+                    "positions must be sorted within a document, got {} after {}",
+                    position, previous
+                ))
+            })?,
+            None => position,
+        };
+        self.push_delta(delta)
+    }
+
+    pub(super) fn finish(self) -> Vec<u8> {
+        let mut bytes = self.encoded_bytes;
+        match self.codec {
+            PositionStreamCodec::VarintDocDelta | PositionStreamCodec::PackedDelta => {
+                for delta in self.pending_deltas {
+                    encode_varint_u32(&mut bytes, delta);
+                }
+            }
+        }
+        bytes
+    }
+
+    pub(super) fn decode_into(&self, frequencies: &[u32], dst: &mut Vec<u32>) -> Result<()> {
+        let bytes = self.clone().finish();
+        decode_position_stream_block(bytes.as_slice(), frequencies, self.codec, dst)
+    }
+
+    fn push_delta(&mut self, delta: u32) -> Result<()> {
+        match self.codec {
+            PositionStreamCodec::VarintDocDelta => {
+                encode_varint_u32(&mut self.encoded_bytes, delta);
+            }
+            PositionStreamCodec::PackedDelta => {
+                self.pending_deltas.push(delta);
+                if self.pending_deltas.len() == BLOCK_SIZE {
+                    let mut packed_buffer = [0u8; BLOCK_SIZE * 4 + 1];
+                    compress_block(
+                        self.pending_deltas.as_slice(),
+                        &mut packed_buffer,
+                        &mut self.encoded_bytes,
+                    )?;
+                    self.pending_deltas.clear();
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+#[inline]
+fn decode_varint_u32(src: &[u8], offset: &mut usize) -> Result<u32> {
+    let mut value = 0u32;
+    let mut shift = 0u32;
+    while *offset < src.len() {
+        let byte = src[*offset];
+        *offset += 1;
+        value |= u32::from(byte & 0x7F) << shift;
+        if byte & 0x80 == 0 {
+            return Ok(value);
+        }
+        shift += 7;
+        if shift >= 35 {
+            return Err(Error::index(
+                "invalid u32 varint in position stream".to_owned(),
+            ));
+        }
+    }
+    Err(Error::index(
+        "unexpected EOF while decoding position stream".to_owned(),
+    ))
+}
+
+#[cfg(test)]
+fn encode_position_stream_varint_block_into(
+    positions: &[u32],
+    frequencies: &[u32],
+    dst: &mut Vec<u8>,
+) -> Result<()> {
+    let mut offset = 0usize;
+    for &frequency in frequencies {
+        let frequency = frequency as usize;
+        let end = offset
+            .checked_add(frequency)
+            .ok_or_else(|| Error::index("position block length overflow".to_owned()))?;
+        if end > positions.len() {
+            return Err(Error::index(format!(
+                "position block has {} positions but frequencies require at least {}",
+                positions.len(),
+                end
+            )));
+        }
+        let mut previous = 0u32;
+        for (index, &position) in positions[offset..end].iter().enumerate() {
+            let delta = if index == 0 {
+                position
+            } else {
+                position.checked_sub(previous).ok_or_else(|| {
+                    Error::index(format!(
+                        "positions must be sorted within a document, got {} after {}",
+                        position, previous
+                    ))
+                })?
+            };
+            encode_varint_u32(dst, delta);
+            previous = position;
+        }
+        offset = end;
+    }
+    if offset != positions.len() {
+        return Err(Error::index(format!(
+            "position block has {} trailing positions after consuming {} frequencies",
+            positions.len() - offset,
+            frequencies.len()
+        )));
+    }
+    Ok(())
+}
+
+fn decode_position_stream_varint_block(
+    src: &[u8],
+    frequencies: &[u32],
+    dst: &mut Vec<u32>,
+) -> Result<()> {
+    let mut offset = 0usize;
+    for &frequency in frequencies {
+        let mut previous = 0u32;
+        for index in 0..frequency as usize {
+            let delta = decode_varint_u32(src, &mut offset)?;
+            let position = if index == 0 {
+                delta
+            } else {
+                previous.checked_add(delta).ok_or_else(|| {
+                    Error::index("position stream overflow while decoding".to_owned())
+                })?
+            };
+            dst.push(position);
+            previous = position;
+        }
+    }
+    if offset != src.len() {
+        return Err(Error::index(format!(
+            "position stream has {} trailing bytes after decoding block",
+            src.len() - offset
+        )));
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+fn encode_position_stream_packed_block_into(
+    positions: &[u32],
+    frequencies: &[u32],
+    dst: &mut Vec<u8>,
+) -> Result<()> {
+    let mut delta_buffer = [0u32; BLOCK_SIZE];
+    let mut delta_count = 0usize;
+    let mut packed_buffer = [0u8; BLOCK_SIZE * 4 + 1];
+    let mut offset = 0usize;
+
+    for &frequency in frequencies {
+        let frequency = frequency as usize;
+        let end = offset
+            .checked_add(frequency)
+            .ok_or_else(|| Error::index("position block length overflow".to_owned()))?;
+        if end > positions.len() {
+            return Err(Error::index(format!(
+                "position block has {} positions but frequencies require at least {}",
+                positions.len(),
+                end
+            )));
+        }
+        let mut previous = 0u32;
+        for (index, &position) in positions[offset..end].iter().enumerate() {
+            let delta = if index == 0 {
+                position
+            } else {
+                position.checked_sub(previous).ok_or_else(|| {
+                    Error::index(format!(
+                        "positions must be sorted within a document, got {} after {}",
+                        position, previous
+                    ))
+                })?
+            };
+            delta_buffer[delta_count] = delta;
+            delta_count += 1;
+            if delta_count == BLOCK_SIZE {
+                compress_block(&delta_buffer, &mut packed_buffer, dst)?;
+                delta_count = 0;
+            }
+            previous = position;
+        }
+        offset = end;
+    }
+
+    if offset != positions.len() {
+        return Err(Error::index(format!(
+            "position block has {} trailing positions after consuming {} frequencies",
+            positions.len() - offset,
+            frequencies.len()
+        )));
+    }
+
+    for delta in &delta_buffer[..delta_count] {
+        encode_varint_u32(dst, *delta);
+    }
+    Ok(())
+}
+
+fn decode_position_stream_packed_block(
+    src: &[u8],
+    frequencies: &[u32],
+    dst: &mut Vec<u32>,
+) -> Result<()> {
+    let total_positions = frequencies.iter().try_fold(0usize, |total, &frequency| {
+        total.checked_add(frequency as usize).ok_or_else(|| {
+            Error::index("position stream length overflow while decoding".to_owned())
+        })
+    })?;
+
+    let full_delta_blocks = total_positions / BLOCK_SIZE;
+    let tail_len = total_positions % BLOCK_SIZE;
+
+    let compressor = BitPacker4x::new();
+    let mut packed_offset = 0usize;
+    let mut packed_values = [0u32; BLOCK_SIZE];
+    let mut deltas = Vec::with_capacity(total_positions);
+
+    for _ in 0..full_delta_blocks {
+        if packed_offset >= src.len() {
+            return Err(Error::index(
+                "unexpected EOF while decoding packed position stream".to_owned(),
+            ));
+        }
+        let num_bits = src[packed_offset];
+        packed_offset += 1;
+        let consumed = compressor.decompress(&src[packed_offset..], &mut packed_values, num_bits);
+        packed_offset += consumed;
+        deltas.extend_from_slice(&packed_values);
+    }
+
+    for _ in 0..tail_len {
+        deltas.push(decode_varint_u32(src, &mut packed_offset)?);
+    }
+
+    if packed_offset != src.len() {
+        return Err(Error::index(format!(
+            "position stream has {} trailing bytes after decoding block",
+            src.len() - packed_offset
+        )));
+    }
+
+    let mut delta_offset = 0usize;
+    for &frequency in frequencies {
+        let mut previous = 0u32;
+        for index in 0..frequency as usize {
+            let delta = deltas[delta_offset];
+            delta_offset += 1;
+            let position = if index == 0 {
+                delta
+            } else {
+                previous.checked_add(delta).ok_or_else(|| {
+                    Error::index("position stream overflow while decoding".to_owned())
+                })?
+            };
+            dst.push(position);
+            previous = position;
+        }
+    }
+    debug_assert_eq!(delta_offset, deltas.len());
+    Ok(())
+}
+
+#[cfg(test)]
+pub fn encode_position_stream_block_into(
+    positions: &[u32],
+    frequencies: &[u32],
+    codec: PositionStreamCodec,
+    dst: &mut Vec<u8>,
+) -> Result<()> {
+    match codec {
+        PositionStreamCodec::VarintDocDelta => {
+            encode_position_stream_varint_block_into(positions, frequencies, dst)
+        }
+        PositionStreamCodec::PackedDelta => {
+            encode_position_stream_packed_block_into(positions, frequencies, dst)
+        }
+    }
+}
+
+pub fn decode_position_stream_block(
+    src: &[u8],
+    frequencies: &[u32],
+    codec: PositionStreamCodec,
+    dst: &mut Vec<u32>,
+) -> Result<()> {
+    match codec {
+        PositionStreamCodec::VarintDocDelta => {
+            decode_position_stream_varint_block(src, frequencies, dst)
+        }
+        PositionStreamCodec::PackedDelta => {
+            decode_position_stream_packed_block(src, frequencies, dst)
+        }
+    }
+}
+
 /// decompress the posting list from a LargeBinaryArray
 /// returns a vector of (row_id, frequency) tuples
-#[allow(dead_code)]
+#[cfg(test)]
 pub fn decompress_posting_list(
     num_docs: u32,
     posting_list: &arrow::array::LargeBinaryArray,
+) -> Result<(Vec<u32>, Vec<u32>)> {
+    decompress_posting_list_with_tail_codec(num_docs, posting_list, PostingTailCodec::VarintDelta)
+}
+
+#[cfg(test)]
+pub fn decompress_posting_list_with_tail_codec(
+    num_docs: u32,
+    posting_list: &arrow::array::LargeBinaryArray,
+    tail_codec: PostingTailCodec,
 ) -> Result<(Vec<u32>, Vec<u32>)> {
     let mut doc_ids: Vec<u32> = Vec::with_capacity(num_docs as usize);
     let mut frequencies: Vec<u32> = Vec::with_capacity(num_docs as usize);
@@ -254,7 +630,13 @@ pub fn decompress_posting_list(
     let remainder = num_docs as usize % BLOCK_SIZE;
     if remainder > 0 {
         let compressed = posting_list.value(bitpacking_blocks);
-        decompress_posting_remainder(compressed, remainder, &mut doc_ids, &mut frequencies);
+        decompress_posting_remainder(
+            compressed,
+            remainder,
+            tail_codec,
+            &mut doc_ids,
+            &mut frequencies,
+        );
     }
 
     Ok((doc_ids, frequencies))
@@ -274,27 +656,12 @@ pub fn decompress_positions(compressed: &arrow::array::LargeBinaryArray) -> Vec<
     let remainder = num_positions as usize % BLOCK_SIZE;
     if remainder > 0 {
         let compressed_block = compressed.value(num_blocks + 1);
-        decompress_remainder(compressed_block, remainder, &mut positions);
+        decompress_raw_remainder(compressed_block, remainder, &mut positions);
     }
 
     positions
 }
 
-// decompress the positions list from a ListArray of binary
-// to a ListArray of u32
-#[allow(dead_code)]
-pub fn decompress_positions_list(compressed: &ListArray) -> Result<ListArray> {
-    let mut builder = ListBuilder::with_capacity(UInt32Builder::new(), compressed.len());
-    for i in 0..compressed.len() {
-        let compressed = compressed.value(i);
-        let compressed = compressed.as_binary::<i64>();
-        let positions = decompress_positions(compressed);
-        builder.values().append_slice(&positions);
-        builder.append(true);
-    }
-    Ok(builder.finish())
-}
-
 pub fn read_num_positions(compressed: &arrow::array::LargeBinaryArray) -> u32 {
     u32::from_le_bytes(compressed.value(0).try_into().unwrap())
 }
@@ -314,12 +681,50 @@ pub fn decompress_posting_block(
 pub fn decompress_posting_remainder(
     block: &[u8],
     n: usize,
+    codec: PostingTailCodec,
     doc_ids: &mut Vec<u32>,
     frequencies: &mut Vec<u32>,
 ) {
     let block = &block[4..];
-    decompress_remainder(block, n, doc_ids);
-    decompress_remainder(&block[n * 4..], n, frequencies);
+    match codec {
+        PostingTailCodec::Fixed32 => {
+            decompress_raw_remainder(block, n, doc_ids);
+            decompress_raw_remainder(&block[n * 4..], n, frequencies);
+        }
+        PostingTailCodec::VarintDelta => {
+            let mut offset = 0usize;
+            let mut previous = 0u32;
+            for index in 0..n {
+                let delta = decode_varint_u32(block, &mut offset)
+                    .expect("posting tail doc ids should contain valid varints");
+                let doc_id = if index == 0 {
+                    delta
+                } else {
+                    previous
+                        .checked_add(delta)
+                        .expect("posting tail doc id delta should not overflow")
+                };
+                doc_ids.push(doc_id);
+                previous = doc_id;
+            }
+            for _ in 0..n {
+                let frequency = decode_varint_u32(block, &mut offset)
+                    .expect("posting tail frequencies should contain valid varints");
+                frequencies.push(frequency);
+            }
+            assert_eq!(
+                offset,
+                block.len(),
+                "posting tail block has {} trailing bytes after decoding",
+                block.len() - offset
+            );
+        }
+    }
+}
+
+pub fn decode_full_posting_block(block: &[u8], doc_ids: &mut Vec<u32>, frequencies: &mut Vec<u32>) {
+    let mut buffer = [0u32; BLOCK_SIZE];
+    decompress_posting_block(block, &mut buffer, doc_ids, frequencies);
 }
 
 pub fn decompress_sorted_block(
@@ -342,13 +747,24 @@ fn decompress_block(block: &[u8], buffer: &mut [u32; BLOCK_SIZE], res: &mut Vec<
     res.extend_from_slice(&buffer[..]);
 }
 
-pub fn decompress_remainder(compressed: &[u8], n: usize, dest: &mut Vec<u32>) {
+pub fn decompress_raw_remainder(compressed: &[u8], n: usize, dest: &mut Vec<u32>) {
     for bytes in compressed.chunks_exact(4).take(n) {
         let data = u32::from_le_bytes(bytes.try_into().unwrap());
         dest.push(data);
     }
 }
 
+pub fn read_posting_tail_first_doc(block: &[u8], codec: PostingTailCodec) -> u32 {
+    match codec {
+        PostingTailCodec::Fixed32 => u32::from_le_bytes(block[4..8].try_into().unwrap()),
+        PostingTailCodec::VarintDelta => {
+            let mut offset = 4usize;
+            decode_varint_u32(block, &mut offset)
+                .expect("posting tail block should contain a valid first doc id")
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -393,6 +809,27 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_compress_posting_list_fixed32_tail_still_roundtrips() -> Result<()> {
+        let doc_ids = vec![3_u32, 10_u32, 24_u32];
+        let frequencies = vec![1_u32, 7_u32, 2_u32];
+        let posting_list = compress_posting_list_with_tail_codec(
+            doc_ids.len(),
+            doc_ids.iter(),
+            frequencies.iter(),
+            std::iter::once(1.0_f32),
+            PostingTailCodec::Fixed32,
+        )?;
+        let (decoded_doc_ids, decoded_frequencies) = decompress_posting_list_with_tail_codec(
+            doc_ids.len() as u32,
+            &posting_list,
+            PostingTailCodec::Fixed32,
+        )?;
+        assert_eq!(decoded_doc_ids, doc_ids);
+        assert_eq!(decoded_frequencies, frequencies);
+        Ok(())
+    }
+
     #[test]
     fn test_compress_positions() -> Result<()> {
         let num_positions: usize = BLOCK_SIZE * 2 - 7;
@@ -417,4 +854,47 @@ mod tests {
         assert_eq!(positions.len(), num_positions);
         Ok(())
     }
+
+    #[test]
+    fn test_encode_position_stream_block_roundtrip() -> Result<()> {
+        let frequencies = vec![1, 3, 2, 4];
+        let positions = vec![7, 1, 3, 8, 2, 100, 0, 4, 9, 25];
+        for codec in [
+            PositionStreamCodec::VarintDocDelta,
+            PositionStreamCodec::PackedDelta,
+        ] {
+            let mut encoded = Vec::new();
+            encode_position_stream_block_into(&positions, &frequencies, codec, &mut encoded)?;
+            let mut decoded = Vec::new();
+            decode_position_stream_block(&encoded, &frequencies, codec, &mut decoded)?;
+            assert_eq!(decoded, positions);
+            assert!(encoded.len() < positions.len() * std::mem::size_of::<u32>());
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_incremental_position_block_builder_matches_batch_encoder() -> Result<()> {
+        let frequencies = vec![1, 3, 2, 4, 1, 5];
+        let positions = vec![7, 1, 3, 8, 2, 100, 0, 4, 9, 25, 11, 2, 6, 7, 10, 15];
+
+        let mut builder = PositionBlockBuilder::new(PositionStreamCodec::PackedDelta);
+        let mut offset = 0usize;
+        for &frequency in &frequencies {
+            let end = offset + frequency as usize;
+            builder.append_doc_positions(&positions[offset..end])?;
+            offset = end;
+        }
+
+        let incremental = builder.finish();
+        let mut batch = Vec::new();
+        encode_position_stream_block_into(
+            &positions,
+            &frequencies,
+            PositionStreamCodec::PackedDelta,
+            &mut batch,
+        )?;
+        assert_eq!(incremental, batch);
+        Ok(())
+    }
 }
diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs
index dbd5c2f6eff..4f60ebd05f5 100644
--- a/rust/lance-index/src/scalar/inverted/index.rs
+++ b/rust/lance-index/src/scalar/inverted/index.rs
@@ -8,7 +8,7 @@ use std::{
     cmp::{Reverse, min},
     collections::BinaryHeap,
 };
-use std::{collections::HashMap, ops::Range};
+use std::{collections::HashMap, ops::Range, time::Instant};
 
 use crate::metrics::NoOpMetricsCollector;
 use crate::prefilter::NoFilter;
@@ -19,7 +19,7 @@ use arrow::{
     array::{
         AsArray, LargeBinaryBuilder, ListBuilder, StringBuilder, UInt32Builder, UInt64Builder,
     },
-    buffer::OffsetBuffer,
+    buffer::{Buffer, OffsetBuffer},
 };
 use arrow::{buffer::ScalarBuffer, datatypes::UInt32Type};
 use arrow_array::{
@@ -38,21 +38,20 @@ use lance_arrow::{RecordBatchExt, iter_str_array};
 use lance_core::cache::{CacheKey, LanceCache, WeakLanceCache};
 use lance_core::error::{DataFusionResult, LanceOptionExt};
 use lance_core::utils::mask::{RowAddrMask, RowAddrTreeMap};
+use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu};
 use lance_core::utils::tracing::{IO_TYPE_LOAD_SCALAR_PART, TRACE_IO_EVENTS};
 use lance_core::{Error, ROW_ID, ROW_ID_FIELD, Result};
-use lance_core::{
-    container::list::ExpLinkedList,
-    utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu},
-};
 use roaring::RoaringBitmap;
 use std::sync::LazyLock;
 use tokio::task::spawn_blocking;
 use tracing::{info, instrument};
 
+use super::encoding::PositionBlockBuilder;
+use super::iter::PostingListIterator;
 use super::{InvertedIndexBuilder, InvertedIndexParams, wand::*};
 use super::{
     builder::{
-        BLOCK_SIZE, ScoredDoc, doc_file_path, inverted_list_schema, posting_file_path,
+        BLOCK_SIZE, ScoredDoc, doc_file_path, inverted_list_schema_for_version, posting_file_path,
         token_file_path,
     },
     iter::PlainPostingListIterator,
@@ -61,26 +60,27 @@ use super::{
 };
 use super::{
     builder::{InnerBuilder, PositionRecorder},
-    encoding::{compress_posting_list, compress_posting_list_with_scores},
     iter::CompressedPostingListIterator,
 };
-use super::{encoding::compress_positions, iter::PostingListIterator};
-use crate::Index;
 use crate::frag_reuse::FragReuseIndex;
 use crate::pbold;
-use crate::scalar::inverted::lance_tokenizer::TextTokenizer;
+use crate::scalar::inverted::document_tokenizer::TextTokenizer;
 use crate::scalar::inverted::scorer::MemBM25Scorer;
-use crate::scalar::inverted::tokenizer::lance_tokenizer::LanceTokenizer;
+use crate::scalar::inverted::tokenizer::document_tokenizer::LanceTokenizer;
 use crate::scalar::{
     AnyQuery, BuiltinIndexType, CreatedIndex, IndexReader, IndexStore, MetricsCollector,
     ScalarIndex, ScalarIndexParams, SearchResult, TokenQuery, UpdateCriteria,
 };
+use crate::{FtsPrewarmOptions, Index};
 use crate::{prefilter::PreFilter, scalar::inverted::iter::take_fst_keys};
+use lance_tokenizer::{SimpleTokenizer, TextAnalyzer};
 use std::str::FromStr;
 
 // Version 0: Arrow TokenSetFormat (legacy)
-// Version 1: Fst TokenSetFormat (new default, incompatible clients < 0.38)
-pub const INVERTED_INDEX_VERSION: u32 = 1;
+// Version 1: Fst TokenSetFormat with per-doc compressed positions
+// Version 2: Fst TokenSetFormat with shared posting-list position streams.
+pub const INVERTED_INDEX_VERSION_V1: u32 = 1;
+pub const INVERTED_INDEX_VERSION_V2: u32 = 2;
 pub const TOKENS_FILE: &str = "tokens.lance";
 pub const INVERT_LIST_FILE: &str = "invert.lance";
 pub const DOCS_FILE: &str = "docs.lance";
@@ -94,6 +94,7 @@ pub const TOKEN_TOTAL_LENGTH_COL: &str = "_token_total_length";
 pub const FREQUENCY_COL: &str = "_frequency";
 pub const POSITION_COL: &str = "_position";
 pub const COMPRESSED_POSITION_COL: &str = "_compressed_position";
+pub const POSITION_BLOCK_OFFSET_COL: &str = "_position_block_offset";
 pub const POSTING_COL: &str = "_posting";
 pub const MAX_SCORE_COL: &str = "_max_score";
 pub const LENGTH_COL: &str = "_length";
@@ -101,6 +102,15 @@ pub const BLOCK_MAX_SCORE_COL: &str = "_block_max_score";
 pub const NUM_TOKEN_COL: &str = "_num_tokens";
 pub const SCORE_COL: &str = "_score";
 pub const TOKEN_SET_FORMAT_KEY: &str = "token_set_format";
+pub const POSTING_TAIL_CODEC_KEY: &str = "posting_tail_codec";
+pub const POSITIONS_LAYOUT_KEY: &str = "positions_layout";
+pub const POSITIONS_CODEC_KEY: &str = "positions_codec";
+pub const POSTING_TAIL_CODEC_FIXED32_V1: &str = "fixed32_v1";
+pub const POSTING_TAIL_CODEC_VARINT_DELTA_V1: &str = "varint_delta_v1";
+pub const POSITIONS_LAYOUT_SHARED_STREAM_V2: &str = "shared_stream_v2";
+pub const POSITIONS_CODEC_VARINT_DOC_DELTA_V2: &str = "varint_doc_delta_v2";
+pub const POSITIONS_CODEC_PACKED_DELTA_V1: &str = "packed_delta_v1";
+pub const DELETED_FRAGMENTS_COL: &str = "deleted_fragments";
 
 // Just a heuristic when we need to pre-allocate memory for tokens
 pub const ESTIMATED_MAX_TOKENS_PER_ROW: usize = 4 * 1024;
@@ -112,6 +122,77 @@ pub static FTS_SCHEMA: LazyLock<SchemaRef> =
 static ROW_ID_SCHEMA: LazyLock<SchemaRef> =
     LazyLock::new(|| Arc::new(Schema::new(vec![ROW_ID_FIELD.clone()])));
 
+fn resolve_fts_format_version(
+    value: Option<&str>,
+) -> std::result::Result<InvertedListFormatVersion, Error> {
+    value.unwrap_or("1").parse()
+}
+
+pub fn current_fts_format_version() -> InvertedListFormatVersion {
+    resolve_fts_format_version(std::env::var("LANCE_FTS_FORMAT_VERSION").ok().as_deref())
+        .expect("failed to parse LANCE_FTS_FORMAT_VERSION")
+}
+
+pub fn max_supported_fts_format_version() -> InvertedListFormatVersion {
+    InvertedListFormatVersion::V2
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
+pub enum InvertedListFormatVersion {
+    #[default]
+    V1,
+    V2,
+}
+
+impl InvertedListFormatVersion {
+    pub fn from_posting_tail_codec(codec: PostingTailCodec) -> Self {
+        match codec {
+            PostingTailCodec::Fixed32 => Self::V1,
+            PostingTailCodec::VarintDelta => Self::V2,
+        }
+    }
+
+    pub fn index_version(self) -> u32 {
+        match self {
+            Self::V1 => INVERTED_INDEX_VERSION_V1,
+            Self::V2 => INVERTED_INDEX_VERSION_V2,
+        }
+    }
+
+    pub fn posting_tail_codec(self) -> PostingTailCodec {
+        match self {
+            Self::V1 => PostingTailCodec::Fixed32,
+            Self::V2 => PostingTailCodec::VarintDelta,
+        }
+    }
+
+    pub fn position_codec(self) -> Option<PositionStreamCodec> {
+        match self {
+            Self::V1 => None,
+            Self::V2 => Some(PositionStreamCodec::PackedDelta),
+        }
+    }
+
+    pub fn uses_shared_position_stream(self) -> bool {
+        matches!(self, Self::V2)
+    }
+}
+
+impl FromStr for InvertedListFormatVersion {
+    type Err = Error;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        match s.trim() {
+            "1" | "v1" | "V1" => Ok(Self::V1),
+            "2" | "v2" | "V2" => Ok(Self::V2),
+            other => Err(Error::index(format!(
+                "unsupported FTS format version {}, expected 1 or 2",
+                other
+            ))),
+        }
+    }
+}
+
 #[derive(Debug)]
 struct PartitionCandidates {
     tokens_by_position: Vec<String>,
@@ -165,6 +246,97 @@ impl DeepSizeOf for TokenSetFormat {
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
+pub enum PositionStreamCodec {
+    VarintDocDelta,
+    #[default]
+    PackedDelta,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
+pub enum PostingTailCodec {
+    Fixed32,
+    #[default]
+    VarintDelta,
+}
+
+impl PostingTailCodec {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::Fixed32 => POSTING_TAIL_CODEC_FIXED32_V1,
+            Self::VarintDelta => POSTING_TAIL_CODEC_VARINT_DELTA_V1,
+        }
+    }
+
+    fn from_metadata_value(value: &str) -> Result<Self> {
+        match value.trim() {
+            POSTING_TAIL_CODEC_FIXED32_V1 => Ok(Self::Fixed32),
+            POSTING_TAIL_CODEC_VARINT_DELTA_V1 => Ok(Self::VarintDelta),
+            other => Err(Error::index(format!(
+                "unsupported posting tail codec {}",
+                other
+            ))),
+        }
+    }
+}
+
+pub(super) fn parse_posting_tail_codec(
+    metadata: &HashMap<String, String>,
+) -> Result<PostingTailCodec> {
+    Ok(metadata
+        .get(POSTING_TAIL_CODEC_KEY)
+        .map(|codec| PostingTailCodec::from_metadata_value(codec))
+        .transpose()?
+        .unwrap_or(PostingTailCodec::Fixed32))
+}
+
+impl PositionStreamCodec {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::VarintDocDelta => POSITIONS_CODEC_VARINT_DOC_DELTA_V2,
+            Self::PackedDelta => POSITIONS_CODEC_PACKED_DELTA_V1,
+        }
+    }
+
+    fn from_metadata_value(value: &str) -> Result<Self> {
+        match value.trim() {
+            POSITIONS_CODEC_VARINT_DOC_DELTA_V2 => Ok(Self::VarintDocDelta),
+            POSITIONS_CODEC_PACKED_DELTA_V1 => Ok(Self::PackedDelta),
+            other => Err(Error::index(format!(
+                "unsupported positions codec {}",
+                other
+            ))),
+        }
+    }
+}
+
+fn parse_shared_position_codec(metadata: &HashMap<String, String>) -> Result<PositionStreamCodec> {
+    if let Some(codec) = metadata.get(POSITIONS_CODEC_KEY) {
+        return PositionStreamCodec::from_metadata_value(codec);
+    }
+
+    match metadata
+        .get(POSITIONS_LAYOUT_KEY)
+        .map(|layout| layout.as_str())
+    {
+        Some(POSITIONS_LAYOUT_SHARED_STREAM_V2) => Ok(PositionStreamCodec::VarintDocDelta),
+        _ => Ok(PositionStreamCodec::VarintDocDelta),
+    }
+}
+
+pub(super) fn parse_format_version_from_metadata(
+    metadata: &HashMap<String, String>,
+) -> Result<InvertedListFormatVersion> {
+    if metadata.contains_key(POSITIONS_CODEC_KEY) || metadata.contains_key(POSITIONS_LAYOUT_KEY) {
+        return Ok(InvertedListFormatVersion::V2);
+    }
+    if parse_posting_tail_codec(metadata)? == PostingTailCodec::VarintDelta {
+        Ok(InvertedListFormatVersion::V2)
+    } else {
+        Ok(InvertedListFormatVersion::V1)
+    }
+}
+
 #[derive(Clone)]
 pub struct InvertedIndex {
     params: InvertedIndexParams,
@@ -172,6 +344,9 @@ pub struct InvertedIndex {
     tokenizer: Box<dyn LanceTokenizer>,
     token_set_format: TokenSetFormat,
     pub(crate) partitions: Vec<Arc<InvertedPartition>>,
+    // Fragments which are contained in the index, but no longer in the dataset.
+    // These should be pruned at search time since we don't prune them at update time.
+    deleted_fragments: RoaringBitmap,
 }
 
 impl Debug for InvertedIndex {
@@ -180,6 +355,7 @@ impl Debug for InvertedIndex {
             .field("params", &self.params)
             .field("token_set_format", &self.token_set_format)
             .field("partitions", &self.partitions)
+            .field("deleted_fragments", &self.deleted_fragments)
             .finish()
     }
 }
@@ -191,6 +367,31 @@ impl DeepSizeOf for InvertedIndex {
 }
 
 impl InvertedIndex {
+    fn format_version(&self) -> InvertedListFormatVersion {
+        self.partitions
+            .first()
+            .map(|partition| {
+                InvertedListFormatVersion::from_posting_tail_codec(
+                    partition.inverted_list.posting_tail_codec(),
+                )
+            })
+            .unwrap_or_else(current_fts_format_version)
+    }
+
+    fn index_version(&self) -> u32 {
+        match self.token_set_format {
+            TokenSetFormat::Arrow => 0,
+            TokenSetFormat::Fst => self.format_version().index_version(),
+        }
+    }
+
+    fn posting_tail_codec(&self) -> PostingTailCodec {
+        self.partitions
+            .first()
+            .map(|partition| partition.inverted_list.posting_tail_codec())
+            .unwrap_or_default()
+    }
+
     fn to_builder(&self) -> InvertedIndexBuilder {
         self.to_builder_with_offset(None)
     }
@@ -204,7 +405,9 @@ impl InvertedIndex {
                 Vec::new(),
                 self.token_set_format,
                 fragment_mask,
+                self.deleted_fragments.clone(),
             )
+            .with_posting_tail_codec(self.posting_tail_codec())
         } else {
             let partitions = match fragment_mask {
                 Some(fragment_mask) => self
@@ -225,7 +428,9 @@ impl InvertedIndex {
                 partitions,
                 self.token_set_format,
                 fragment_mask,
+                self.deleted_fragments.clone(),
             )
+            .with_format_version(self.format_version())
         }
     }
 
@@ -242,6 +447,15 @@ impl InvertedIndex {
         self.partitions.len()
     }
 
+    /// Returns the set of fragments which are contained in the index, but no longer in the dataset.
+    ///
+    /// Most other indices remove data from deleted fragments when the index updates (copy-on-write).
+    /// However, this would require an expensive copy of the FTS index.  Instead, we track the deleted
+    /// fragments and prune them at search time (merge-on-read).
+    pub fn deleted_fragments(&self) -> &RoaringBitmap {
+        &self.deleted_fragments
+    }
+
     // search the documents that contain the query
     // return the row ids of the documents sorted by bm25 score
     // ref: https://en.wikipedia.org/wiki/Okapi_BM25
@@ -279,7 +493,12 @@ impl InvertedIndex {
                     if postings.is_empty() {
                         return Result::Ok(PartitionCandidates::empty());
                     }
-                    let mut tokens_by_position = vec![String::new(); postings.len()];
+                    let max_position = postings
+                        .iter()
+                        .map(|posting| posting.term_index() as usize)
+                        .max()
+                        .unwrap_or_default();
+                    let mut tokens_by_position = vec![String::new(); max_position + 1];
                     for posting in &postings {
                         let idx = posting.term_index() as usize;
                         tokens_by_position[idx] = posting.token().to_owned();
@@ -410,6 +629,7 @@ impl InvertedIndex {
                 docs,
                 token_set_format: TokenSetFormat::Arrow,
             })],
+            deleted_fragments: RoaringBitmap::new(),
         }))
     }
 
@@ -451,6 +671,19 @@ impl InvertedIndex {
                     .transpose()?
                     .unwrap_or(TokenSetFormat::Arrow);
 
+                // Load deleted_fragments if present (optional for backward compatibility)
+                let deleted_fragments = if reader.num_rows() > 0 {
+                    let metadata_batch = reader.read_range(0..1, None).await?;
+                    if let Some(col) = metadata_batch.column_by_name(DELETED_FRAGMENTS_COL) {
+                        let arr = col.as_binary_opt::<i32>().expect_ok()?;
+                        RoaringBitmap::deserialize_from(arr.value(0))?
+                    } else {
+                        RoaringBitmap::new()
+                    }
+                } else {
+                    RoaringBitmap::new()
+                };
+
                 let format = token_set_format;
                 let partitions = partitions.into_iter().map(|id| {
                     let store = store.clone();
@@ -483,6 +716,7 @@ impl InvertedIndex {
                     tokenizer,
                     token_set_format,
                     partitions,
+                    deleted_fragments,
                 }))
             }
             Err(_) => {
@@ -528,10 +762,8 @@ impl Index for InvertedIndex {
     }
 
     async fn prewarm(&self) -> Result<()> {
-        for part in &self.partitions {
-            part.inverted_list.prewarm().await?;
-        }
-        Ok(())
+        self.prewarm_with_options(&FtsPrewarmOptions::default())
+            .await
     }
 
     fn index_type(&self) -> crate::IndexType {
@@ -544,6 +776,25 @@ impl Index for InvertedIndex {
 }
 
 impl InvertedIndex {
+    pub async fn prewarm_with_options(&self, options: &FtsPrewarmOptions) -> Result<()> {
+        let with_position = options.with_position;
+        let io_parallelism = self.store.io_parallelism();
+        let prewarm_futures = self
+            .partitions
+            .iter()
+            .map(Arc::clone)
+            .map(|part| async move {
+                part.inverted_list
+                    .prewarm_posting_lists(with_position)
+                    .await?;
+                Result::Ok(())
+            });
+        stream::iter(prewarm_futures)
+            .buffer_unordered(io_parallelism)
+            .try_collect::<Vec<_>>()
+            .await?;
+        Ok(())
+    }
     /// Search docs match the input text.
     async fn do_search(&self, text: &str) -> Result<RecordBatch> {
         let params = FtsSearchParams::new();
@@ -608,15 +859,10 @@ impl ScalarIndex for InvertedIndex {
 
         let details = pbold::InvertedIndexDetails::try_from(&self.params)?;
 
-        // Use version 0 for Arrow format (legacy), version 1 for Fst format (new)
-        let index_version = match self.token_set_format {
-            TokenSetFormat::Arrow => 0,
-            TokenSetFormat::Fst => INVERTED_INDEX_VERSION,
-        };
-
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&details).unwrap(),
-            index_version,
+            index_version: self.index_version(),
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -624,21 +870,18 @@ impl ScalarIndex for InvertedIndex {
         &self,
         new_data: SendableRecordBatchStream,
         dest_store: &dyn IndexStore,
-        _old_data_filter: Option<crate::scalar::OldIndexDataFilter>,
+        old_data_filter: Option<crate::scalar::OldIndexDataFilter>,
     ) -> Result<CreatedIndex> {
-        self.to_builder().update(new_data, dest_store).await?;
+        self.to_builder()
+            .update(new_data, dest_store, old_data_filter)
+            .await?;
 
         let details = pbold::InvertedIndexDetails::try_from(&self.params)?;
 
-        // Use version 0 for Arrow format (legacy), version 1 for Fst format (new)
-        let index_version = match self.token_set_format {
-            TokenSetFormat::Arrow => 0,
-            TokenSetFormat::Fst => INVERTED_INDEX_VERSION,
-        };
-
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&details).unwrap(),
-            index_version,
+            index_version: self.index_version(),
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -781,11 +1024,14 @@ impl InvertedPartition {
             true => self.expand_fuzzy(tokens, params)?,
             false => tokens.clone(),
         };
+        let token_positions = (0..tokens.len())
+            .map(|index| tokens.position(index))
+            .collect::<Vec<_>>();
         let mut token_ids = Vec::with_capacity(tokens.len());
-        for token in tokens {
+        for (index, token) in tokens.into_iter().enumerate() {
             let token_id = self.map(&token);
             if let Some(token_id) = token_id {
-                token_ids.push((token_id, token));
+                token_ids.push((token_id, token, token_positions[index]));
             } else if is_phrase_query {
                 // if the token is not found, we can't do phrase query
                 return Ok(Vec::new());
@@ -795,24 +1041,25 @@ impl InvertedPartition {
             return Ok(Vec::new());
         }
         if !is_phrase_query {
-            // remove duplicates
-            token_ids.sort_unstable_by_key(|(token_id, _)| *token_id);
-            token_ids.dedup_by_key(|(token_id, _)| *token_id);
+            token_ids.sort_unstable_by_key(|(token_id, _, _)| *token_id);
+            token_ids.dedup_by_key(|(token_id, _, _)| *token_id);
         }
 
         let num_docs = self.docs.len();
         stream::iter(token_ids)
-            .enumerate()
-            .map(|(position, (token_id, token))| async move {
+            .map(|(token_id, token, position)| async move {
                 let posting = self
                     .inverted_list
                     .posting_list(token_id, is_phrase_query, metrics)
                     .await?;
 
-                Result::Ok(PostingIterator::new(
+                let query_weight = idf(posting.len(), num_docs);
+
+                Result::Ok(PostingIterator::with_query_weight(
                     token,
                     token_id,
-                    position as u32,
+                    position,
+                    query_weight,
                     posting,
                     num_docs,
                 ))
@@ -844,10 +1091,11 @@ impl InvertedPartition {
     }
 
     pub async fn into_builder(self) -> Result<InnerBuilder> {
-        let mut builder = InnerBuilder::new(
+        let mut builder = InnerBuilder::new_with_posting_tail_codec(
             self.id,
             self.inverted_list.has_positions(),
             self.token_set_format,
+            self.inverted_list.posting_tail_codec(),
         );
         builder.tokens = self.tokens;
         builder.docs = self.docs;
@@ -1195,6 +1443,19 @@ impl TokenSet {
     pub fn next_id(&self) -> u32 {
         self.next_id
     }
+
+    pub(crate) fn memory_size(&self) -> usize {
+        match &self.tokens {
+            TokenMap::HashMap(map) => {
+                self.total_length
+                    + map.capacity()
+                        * (std::mem::size_of::<String>()
+                            + std::mem::size_of::<u32>()
+                            + std::mem::size_of::<usize>())
+            }
+            TokenMap::Fst(map) => map.as_fst().size(),
+        }
+    }
 }
 
 pub struct PostingListReader {
@@ -1211,10 +1472,19 @@ pub struct PostingListReader {
     lengths: Option<Vec<u32>>,
 
     has_position: bool,
+    posting_tail_codec: PostingTailCodec,
+    positions_layout: PositionsLayout,
 
     index_cache: WeakLanceCache,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum PositionsLayout {
+    None,
+    LegacyPerDoc,
+    SharedStream(PositionStreamCodec),
+}
+
 impl std::fmt::Debug for PostingListReader {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("InvertedListReader")
@@ -1237,7 +1507,15 @@ impl PostingListReader {
         reader: Arc<dyn IndexReader>,
         index_cache: &LanceCache,
     ) -> Result<Self> {
-        let has_position = reader.schema().field(POSITION_COL).is_some();
+        let positions_layout = if reader.schema().field(COMPRESSED_POSITION_COL).is_some() {
+            PositionsLayout::SharedStream(parse_shared_position_codec(&reader.schema().metadata)?)
+        } else if reader.schema().field(POSITION_COL).is_some() {
+            PositionsLayout::LegacyPerDoc
+        } else {
+            PositionsLayout::None
+        };
+        let posting_tail_codec = parse_posting_tail_codec(&reader.schema().metadata)?;
+        let has_position = positions_layout != PositionsLayout::None;
         let (offsets, max_scores, lengths) = if reader.schema().field(POSTING_COL).is_none() {
             let (offsets, max_scores) = Self::load_metadata(reader.schema())?;
             (Some(offsets), max_scores, None)
@@ -1262,6 +1540,8 @@ impl PostingListReader {
             max_scores,
             lengths,
             has_position,
+            posting_tail_codec,
+            positions_layout,
             index_cache: WeakLanceCache::from(index_cache),
         })
     }
@@ -1301,6 +1581,10 @@ impl PostingListReader {
         self.has_position
     }
 
+    pub(crate) fn posting_tail_codec(&self) -> PostingTailCodec {
+        self.posting_tail_codec
+    }
+
     pub(crate) fn posting_len(&self, token_id: u32) -> usize {
         let token_id = token_id as usize;
 
@@ -1332,7 +1616,17 @@ impl PostingListReader {
         } else {
             let token_id = token_id as usize;
             let columns = if with_position {
-                vec![POSTING_COL, POSITION_COL]
+                match self.positions_layout {
+                    PositionsLayout::SharedStream(_) => {
+                        vec![
+                            POSTING_COL,
+                            COMPRESSED_POSITION_COL,
+                            POSITION_BLOCK_OFFSET_COL,
+                        ]
+                    }
+                    PositionsLayout::LegacyPerDoc => vec![POSTING_COL, POSITION_COL],
+                    PositionsLayout::None => vec![POSTING_COL],
+                }
             } else {
                 vec![POSTING_COL]
             };
@@ -1384,7 +1678,7 @@ impl PostingListReader {
             .as_ref()
             .clone();
 
-        if is_phrase_query {
+        if is_phrase_query && !posting.has_position() {
             // hit the cache and when the cache was populated, the positions column was not loaded
             let positions = self.read_positions(token_id).await?;
             posting.set_positions(positions);
@@ -1393,12 +1687,29 @@ impl PostingListReader {
         Ok(posting)
     }
 
+    fn posting_list_from_batch_parts(
+        batch: &RecordBatch,
+        max_score: Option<f32>,
+        length: Option<u32>,
+        posting_tail_codec: PostingTailCodec,
+        positions_layout: PositionsLayout,
+    ) -> Result<PostingList> {
+        let posting_list = PostingList::from_batch_with_tail_codec_and_positions_layout(
+            batch,
+            max_score,
+            length,
+            posting_tail_codec,
+            positions_layout,
+        )?;
+        Ok(posting_list)
+    }
+
     pub(crate) fn posting_list_from_batch(
         &self,
         batch: &RecordBatch,
         token_id: u32,
     ) -> Result<PostingList> {
-        let posting_list = PostingList::from_batch(
+        Self::posting_list_from_batch_parts(
             batch,
             self.max_scores
                 .as_ref()
@@ -1406,35 +1717,108 @@ impl PostingListReader {
             self.lengths
                 .as_ref()
                 .map(|lengths| lengths[token_id as usize]),
-        )?;
-        Ok(posting_list)
+            self.posting_tail_codec,
+            self.positions_layout,
+        )
     }
 
-    async fn prewarm(&self) -> Result<()> {
-        let batch = self.read_batch(false).await?;
-        for token_id in 0..self.len() {
-            let posting_range = self.posting_list_range(token_id as u32);
-            let batch = batch.slice(posting_range.start, posting_range.end - posting_range.start);
-            // Apply shrink_to_fit to create a deep copy with compacted buffers
-            // This ensures each cached entry has its own memory, not shared references
+    fn build_prewarm_posting_lists(
+        batch: RecordBatch,
+        offsets: Option<Vec<usize>>,
+        max_scores: Option<Vec<f32>>,
+        lengths: Option<Vec<u32>>,
+        posting_tail_codec: PostingTailCodec,
+        positions_layout: PositionsLayout,
+    ) -> Result<Vec<(u32, PostingList)>> {
+        let token_count = if let Some(offsets) = offsets.as_ref() {
+            offsets.len()
+        } else if let Some(lengths) = lengths.as_ref() {
+            lengths.len()
+        } else {
+            batch.num_rows()
+        };
+
+        let mut posting_lists = Vec::with_capacity(token_count);
+        for token_id in 0..token_count {
+            let batch = if let Some(offsets) = offsets.as_ref() {
+                let start = offsets[token_id];
+                let end = if token_id + 1 < offsets.len() {
+                    offsets[token_id + 1]
+                } else {
+                    batch.num_rows()
+                };
+                batch.slice(start, end - start)
+            } else {
+                batch.slice(token_id, 1)
+            };
             let batch = batch.shrink_to_fit()?;
-            let posting_list = self.posting_list_from_batch(&batch, token_id as u32)?;
-            let inserted = self
-                .index_cache
-                .insert_with_key(
-                    &PostingListKey {
-                        token_id: token_id as u32,
-                    },
-                    Arc::new(posting_list),
-                )
-                .await;
+            let posting_list = Self::posting_list_from_batch_parts(
+                &batch,
+                max_scores.as_ref().map(|scores| scores[token_id]),
+                lengths.as_ref().map(|lengths| lengths[token_id]),
+                posting_tail_codec,
+                positions_layout,
+            )?;
+            posting_lists.push((token_id as u32, posting_list));
+        }
 
-            if !inserted {
-                return Err(Error::internal(
-                    "Failed to prewarm index: cache is no longer available".to_string(),
-                ));
+        Ok(posting_lists)
+    }
+
+    async fn prewarm_posting_lists(&self, with_position: bool) -> Result<()> {
+        if with_position && !self.has_positions() {
+            return Err(Error::invalid_input(
+                "cannot prewarm positions for an inverted index that was built without positions; recreate the index with with_position=true".to_owned(),
+            ));
+        }
+
+        let read_batch_start = Instant::now();
+        let batch = self.read_batch(with_position).await?;
+        let read_batch_elapsed = read_batch_start.elapsed();
+
+        let legacy_layout = self.offsets.is_some();
+        let offsets = self.offsets.clone();
+        let max_scores = self.max_scores.clone();
+        let lengths = self.lengths.clone();
+        let posting_tail_codec = self.posting_tail_codec;
+        let positions_layout = self.positions_layout;
+        let populate_start = Instant::now();
+        let posting_lists = spawn_blocking(move || {
+            Self::build_prewarm_posting_lists(
+                batch,
+                offsets,
+                max_scores,
+                lengths,
+                posting_tail_codec,
+                positions_layout,
+            )
+        })
+        .await
+        .map_err(|err| {
+            Error::internal(format!(
+                "Failed to build prewarm posting lists in blocking task: {err}"
+            ))
+        })??;
+        for (token_id, mut posting_list) in posting_lists {
+            if with_position && let Some(positions) = posting_list.take_positions() {
+                self.index_cache
+                    .insert_with_key(&PositionKey { token_id }, Arc::new(Positions(positions)))
+                    .await;
             }
+            self.index_cache
+                .insert_with_key(&PostingListKey { token_id }, Arc::new(posting_list))
+                .await;
         }
+        let populate_elapsed = populate_start.elapsed();
+
+        info!(
+            legacy_layout,
+            with_position,
+            token_count = self.len(),
+            read_batch_ms = read_batch_elapsed.as_secs_f64() * 1000.0,
+            post_read_loop_ms = populate_elapsed.as_secs_f64() * 1000.0,
+            "posting list prewarm timing"
+        );
 
         Ok(())
     }
@@ -1461,20 +1845,57 @@ impl PostingListReader {
         }))
     }
 
-    async fn read_positions(&self, token_id: u32) -> Result<ListArray> {
+    async fn read_positions(&self, token_id: u32) -> Result<CompressedPositionStorage> {
         let positions = self.index_cache.get_or_insert_with_key(PositionKey { token_id }, || async move {
-            let batch = self
-                .reader
-                .read_range(self.posting_list_range(token_id), Some(&[POSITION_COL]))
-                .await.map_err(|e| {
-                    match e {
-                        Error::Schema { .. } => Error::invalid_input("position is not found but required for phrase queries, try recreating the index with position".to_owned()),
-                        e => e
-                    }
-                })?;
-            Result::Ok(Positions(batch[POSITION_COL]
-                .as_list::<i32>()
-                .clone()))
+            let positions = match self.positions_layout {
+                PositionsLayout::None => {
+                    return Err(Error::invalid_input(
+                        "position is not found but required for phrase queries, try recreating the index with position".to_owned(),
+                    ));
+                }
+                PositionsLayout::LegacyPerDoc => {
+                    let batch = self
+                        .reader
+                        .read_range(self.posting_list_range(token_id), Some(&[POSITION_COL]))
+                        .await
+                        .map_err(|e| match e {
+                            Error::Schema { .. } => Error::invalid_input("position is not found but required for phrase queries, try recreating the index with position".to_owned()),
+                            e => e,
+                        })?;
+                    CompressedPositionStorage::LegacyPerDoc(
+                        batch[POSITION_COL].as_list::<i32>().value(0).as_list::<i32>().clone(),
+                    )
+                }
+                PositionsLayout::SharedStream(codec) => {
+                    let batch = self
+                        .reader
+                        .read_range(
+                            self.posting_list_range(token_id),
+                            Some(&[COMPRESSED_POSITION_COL, POSITION_BLOCK_OFFSET_COL]),
+                        )
+                        .await
+                        .map_err(|e| match e {
+                            Error::Schema { .. } => Error::invalid_input("position is not found but required for phrase queries, try recreating the index with position".to_owned()),
+                            e => e,
+                        })?;
+                    let bytes = batch[COMPRESSED_POSITION_COL]
+                        .as_binary::<i64>()
+                        .value(0)
+                        .to_vec();
+                    let block_offsets = batch[POSITION_BLOCK_OFFSET_COL]
+                        .as_list::<i32>()
+                        .value(0)
+                        .as_primitive::<UInt32Type>()
+                        .values()
+                        .to_vec();
+                    CompressedPositionStorage::SharedStream(SharedPositionStream::new(
+                        codec,
+                        block_offsets,
+                        bytes,
+                    ))
+                }
+            };
+            Result::Ok(Positions(positions))
         }).await?;
         Ok(positions.0.clone())
     }
@@ -1499,7 +1920,14 @@ impl PostingListReader {
             None => vec![POSTING_COL],
         };
         if with_position {
-            base_columns.push(POSITION_COL);
+            match self.positions_layout {
+                PositionsLayout::None => {}
+                PositionsLayout::LegacyPerDoc => base_columns.push(POSITION_COL),
+                PositionsLayout::SharedStream(_) => {
+                    base_columns.push(COMPRESSED_POSITION_COL);
+                    base_columns.push(POSITION_BLOCK_OFFSET_COL);
+                }
+            }
         }
         base_columns
     }
@@ -1508,11 +1936,16 @@ impl PostingListReader {
 /// New type just to allow Positions implement DeepSizeOf so it can be put
 /// in the cache.
 #[derive(Clone)]
-pub struct Positions(ListArray);
+pub struct Positions(CompressedPositionStorage);
 
 impl DeepSizeOf for Positions {
     fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
-        self.0.get_buffer_memory_size()
+        match &self.0 {
+            CompressedPositionStorage::LegacyPerDoc(positions) => {
+                positions.get_buffer_memory_size()
+            }
+            CompressedPositionStorage::SharedStream(stream) => stream.size(),
+        }
     }
 }
 
@@ -1528,6 +1961,10 @@ impl CacheKey for PostingListKey {
     fn key(&self) -> std::borrow::Cow<'_, str> {
         format!("postings-{}", self.token_id).into()
     }
+
+    fn type_name() -> &'static str {
+        "PostingList"
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -1541,6 +1978,77 @@ impl CacheKey for PositionKey {
     fn key(&self) -> std::borrow::Cow<'_, str> {
         format!("positions-{}", self.token_id).into()
     }
+
+    fn type_name() -> &'static str {
+        "Position"
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum CompressedPositionStorage {
+    LegacyPerDoc(ListArray),
+    SharedStream(SharedPositionStream),
+}
+
+impl DeepSizeOf for CompressedPositionStorage {
+    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+        match self {
+            Self::LegacyPerDoc(positions) => positions.get_buffer_memory_size(),
+            Self::SharedStream(stream) => stream.size(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct SharedPositionStream {
+    codec: PositionStreamCodec,
+    block_offsets: Vec<u32>,
+    bytes: Vec<u8>,
+}
+
+impl SharedPositionStream {
+    pub fn new(codec: PositionStreamCodec, block_offsets: Vec<u32>, bytes: Vec<u8>) -> Self {
+        Self {
+            codec,
+            block_offsets,
+            bytes,
+        }
+    }
+
+    pub fn codec(&self) -> PositionStreamCodec {
+        self.codec
+    }
+
+    pub fn block_count(&self) -> usize {
+        self.block_offsets.len()
+    }
+
+    pub fn block_range(&self, index: usize) -> Range<usize> {
+        let start = self.block_offsets[index] as usize;
+        let end = self
+            .block_offsets
+            .get(index + 1)
+            .map(|offset| *offset as usize)
+            .unwrap_or(self.bytes.len());
+        start..end
+    }
+
+    pub fn block(&self, index: usize) -> &[u8] {
+        let range = self.block_range(index);
+        &self.bytes[range]
+    }
+
+    pub fn bytes(&self) -> &[u8] {
+        &self.bytes
+    }
+
+    pub fn block_offsets(&self) -> &[u32] {
+        &self.block_offsets
+    }
+
+    pub fn size(&self) -> usize {
+        self.block_offsets.capacity() * std::mem::size_of::<u32>() + self.bytes.capacity()
+    }
 }
 
 #[derive(Debug, Clone, DeepSizeOf)]
@@ -1554,12 +2062,56 @@ impl PostingList {
         batch: &RecordBatch,
         max_score: Option<f32>,
         length: Option<u32>,
+    ) -> Result<Self> {
+        let posting_tail_codec = parse_posting_tail_codec(batch.schema_ref().metadata())?;
+        Self::from_batch_with_tail_codec(batch, max_score, length, posting_tail_codec)
+    }
+
+    pub fn from_batch_with_tail_codec(
+        batch: &RecordBatch,
+        max_score: Option<f32>,
+        length: Option<u32>,
+        posting_tail_codec: PostingTailCodec,
+    ) -> Result<Self> {
+        let positions_layout = if batch.column_by_name(COMPRESSED_POSITION_COL).is_some() {
+            PositionsLayout::SharedStream(parse_shared_position_codec(
+                batch.schema_ref().metadata(),
+            )?)
+        } else if batch.column_by_name(POSITION_COL).is_some() {
+            PositionsLayout::LegacyPerDoc
+        } else {
+            PositionsLayout::None
+        };
+        Self::from_batch_with_tail_codec_and_positions_layout(
+            batch,
+            max_score,
+            length,
+            posting_tail_codec,
+            positions_layout,
+        )
+    }
+
+    fn from_batch_with_tail_codec_and_positions_layout(
+        batch: &RecordBatch,
+        max_score: Option<f32>,
+        length: Option<u32>,
+        posting_tail_codec: PostingTailCodec,
+        positions_layout: PositionsLayout,
     ) -> Result<Self> {
         match batch.column_by_name(POSTING_COL) {
             Some(_) => {
                 debug_assert!(max_score.is_some() && length.is_some());
-                let posting =
-                    CompressedPostingList::from_batch(batch, max_score.unwrap(), length.unwrap());
+                let shared_position_codec = match positions_layout {
+                    PositionsLayout::SharedStream(codec) => Some(codec),
+                    _ => None,
+                };
+                let posting = CompressedPostingList::from_batch(
+                    batch,
+                    max_score.unwrap(),
+                    length.unwrap(),
+                    posting_tail_codec,
+                    shared_position_codec,
+                );
                 Ok(Self::Compressed(posting))
             }
             None => {
@@ -1580,15 +2132,32 @@ impl PostingList {
         }
     }
 
-    pub fn set_positions(&mut self, positions: ListArray) {
+    pub fn set_positions(&mut self, positions: CompressedPositionStorage) {
         match self {
-            Self::Plain(posting) => posting.positions = Some(positions),
+            Self::Plain(posting) => match positions {
+                CompressedPositionStorage::LegacyPerDoc(positions) => {
+                    posting.positions = Some(positions)
+                }
+                CompressedPositionStorage::SharedStream(_) => {
+                    unreachable!("shared position stream is not supported for plain postings")
+                }
+            },
             Self::Compressed(posting) => {
-                posting.positions = Some(positions.value(0).as_list::<i32>().clone());
+                posting.positions = Some(positions);
             }
         }
     }
 
+    pub fn take_positions(&mut self) -> Option<CompressedPositionStorage> {
+        match self {
+            Self::Plain(posting) => posting
+                .positions
+                .take()
+                .map(CompressedPositionStorage::LegacyPerDoc),
+            Self::Compressed(posting) => posting.positions.take(),
+        }
+    }
+
     pub fn max_score(&self) -> Option<f32> {
         match self {
             Self::Plain(posting) => posting.max_score,
@@ -1608,7 +2177,14 @@ impl PostingList {
     }
 
     pub fn into_builder(self, docs: &DocSet) -> PostingListBuilder {
-        let mut builder = PostingListBuilder::new(self.has_position());
+        let posting_tail_codec = match &self {
+            Self::Plain(_) => PostingTailCodec::Fixed32,
+            Self::Compressed(posting) => posting.posting_tail_codec,
+        };
+        let mut builder = PostingListBuilder::new_with_posting_tail_codec(
+            self.has_position(),
+            posting_tail_codec,
+        );
         match self {
             // legacy format
             Self::Plain(posting) => {
@@ -1675,7 +2251,7 @@ impl DeepSizeOf for PlainPostingList {
             + self
                 .positions
                 .as_ref()
-                .map(|positions| positions.get_buffer_memory_size())
+                .map(Array::get_buffer_memory_size)
                 .unwrap_or(0)
     }
 }
@@ -1767,7 +2343,8 @@ pub struct CompressedPostingList {
     // each binary is a block of compressed data
     // that contains `BLOCK_SIZE` doc ids and then `BLOCK_SIZE` frequencies
     pub blocks: LargeBinaryArray,
-    pub positions: Option<ListArray>,
+    pub posting_tail_codec: PostingTailCodec,
+    pub positions: Option<CompressedPositionStorage>,
 }
 
 impl DeepSizeOf for CompressedPostingList {
@@ -1776,7 +2353,12 @@ impl DeepSizeOf for CompressedPostingList {
             + self
                 .positions
                 .as_ref()
-                .map(|positions| positions.get_buffer_memory_size())
+                .map(|positions| match positions {
+                    CompressedPositionStorage::LegacyPerDoc(positions) => {
+                        positions.get_buffer_memory_size()
+                    }
+                    CompressedPositionStorage::SharedStream(stream) => stream.size(),
+                })
                 .unwrap_or(0)
     }
 }
@@ -1786,31 +2368,59 @@ impl CompressedPostingList {
         blocks: LargeBinaryArray,
         max_score: f32,
         length: u32,
-        positions: Option<ListArray>,
+        posting_tail_codec: PostingTailCodec,
+        positions: Option<CompressedPositionStorage>,
     ) -> Self {
         Self {
             max_score,
             length,
             blocks,
+            posting_tail_codec,
             positions,
         }
     }
 
-    pub fn from_batch(batch: &RecordBatch, max_score: f32, length: u32) -> Self {
-        debug_assert_eq!(batch.num_rows(), 1);
+    pub fn from_batch(
+        batch: &RecordBatch,
+        max_score: f32,
+        length: u32,
+        posting_tail_codec: PostingTailCodec,
+        shared_position_codec: Option<PositionStreamCodec>,
+    ) -> Self {
+        debug_assert_eq!(batch.num_rows(), 1);
         let blocks = batch[POSTING_COL]
             .as_list::<i32>()
             .value(0)
             .as_binary::<i64>()
             .clone();
-        let positions = batch
-            .column_by_name(POSITION_COL)
-            .map(|col| col.as_list::<i32>().value(0).as_list::<i32>().clone());
+        let positions = if let Some(col) = batch.column_by_name(COMPRESSED_POSITION_COL) {
+            let bytes = col.as_binary::<i64>().value(0).to_vec();
+            let block_offsets = batch[POSITION_BLOCK_OFFSET_COL]
+                .as_list::<i32>()
+                .value(0)
+                .as_primitive::<UInt32Type>()
+                .values()
+                .to_vec();
+            let codec = shared_position_codec.unwrap_or_else(|| {
+                parse_shared_position_codec(batch.schema_ref().metadata())
+                    .expect("shared position stream codec metadata should be valid")
+            });
+            Some(CompressedPositionStorage::SharedStream(
+                SharedPositionStream::new(codec, block_offsets, bytes),
+            ))
+        } else {
+            batch.column_by_name(POSITION_COL).map(|col| {
+                CompressedPositionStorage::LegacyPerDoc(
+                    col.as_list::<i32>().value(0).as_list::<i32>().clone(),
+                )
+            })
+        };
 
         Self {
             max_score,
             length,
             blocks,
+            posting_tail_codec,
             positions,
         }
     }
@@ -1819,6 +2429,7 @@ impl CompressedPostingList {
         CompressedPostingListIterator::new(
             self.length as usize,
             self.blocks.clone(),
+            self.posting_tail_codec,
             self.positions.clone(),
         )
     }
@@ -1830,64 +2441,653 @@ impl CompressedPostingList {
 
     pub fn block_least_doc_id(&self, block_idx: usize) -> u32 {
         let block = self.blocks.value(block_idx);
-        block[4..8].try_into().map(u32::from_le_bytes).unwrap()
+        let remainder = self.length as usize % BLOCK_SIZE;
+        let is_remainder_block = remainder > 0 && block_idx + 1 == self.blocks.len();
+        if is_remainder_block {
+            super::encoding::read_posting_tail_first_doc(block, self.posting_tail_codec)
+        } else {
+            block[4..8].try_into().map(u32::from_le_bytes).unwrap()
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+struct EncodedBlocks {
+    offsets: Vec<u32>,
+    bytes: Vec<u8>,
+}
+
+impl EncodedBlocks {
+    fn len(&self) -> usize {
+        self.offsets.len()
+    }
+
+    fn size(&self) -> usize {
+        self.offsets.capacity() * std::mem::size_of::<u32>() + self.bytes.capacity()
+    }
+
+    fn push_full_block(&mut self, doc_ids: &[u32], frequencies: &[u32]) -> Result<usize> {
+        let start = self.bytes.len();
+        self.offsets.push(start as u32);
+        super::encoding::encode_full_posting_block_into(doc_ids, frequencies, &mut self.bytes)?;
+        Ok(self.bytes.len() - start)
+    }
+
+    fn block(&self, index: usize) -> &[u8] {
+        let (start, end) = self.block_range(index);
+        &self.bytes[start..end]
+    }
+
+    fn block_range(&self, index: usize) -> (usize, usize) {
+        let start = self.offsets[index] as usize;
+        let end = self
+            .offsets
+            .get(index + 1)
+            .map(|offset| *offset as usize)
+            .unwrap_or(self.bytes.len());
+        (start, end)
+    }
+
+    fn set_block_score(&mut self, index: usize, score: f32) {
+        let (start, _) = self.block_range(index);
+        self.bytes[start..start + 4].copy_from_slice(&score.to_le_bytes());
+    }
+
+    fn append_remainder_block_with_codec(
+        &mut self,
+        doc_ids: &[u32],
+        frequencies: &[u32],
+        codec: PostingTailCodec,
+    ) -> Result<()> {
+        self.offsets.push(self.bytes.len() as u32);
+        super::encoding::encode_remainder_posting_block_into(
+            doc_ids,
+            frequencies,
+            codec,
+            &mut self.bytes,
+        )
+    }
+
+    fn into_array(mut self) -> LargeBinaryArray {
+        let mut offsets = Vec::with_capacity(self.offsets.len() + 1);
+        offsets.extend(self.offsets.into_iter().map(i64::from));
+        offsets.push(self.bytes.len() as i64);
+        LargeBinaryArray::new(
+            OffsetBuffer::new(ScalarBuffer::from(offsets)),
+            Buffer::from_vec(std::mem::take(&mut self.bytes)),
+            None,
+        )
+    }
+
+    fn iter(&self) -> impl Iterator<Item = &[u8]> {
+        (0..self.len()).map(|index| self.block(index))
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+struct EncodedPositionBlocks {
+    offsets: Vec<u32>,
+    bytes: Vec<u8>,
+}
+
+impl EncodedPositionBlocks {
+    fn size(&self) -> usize {
+        self.offsets.capacity() * std::mem::size_of::<u32>() + self.bytes.capacity()
+    }
+
+    fn block(&self, index: usize) -> &[u8] {
+        let start = self.offsets[index] as usize;
+        let end = self
+            .offsets
+            .get(index + 1)
+            .map(|offset| *offset as usize)
+            .unwrap_or(self.bytes.len());
+        &self.bytes[start..end]
+    }
+
+    fn push_encoded_block(&mut self, block: &[u8]) -> usize {
+        let start = self.bytes.len();
+        self.offsets.push(start as u32);
+        self.bytes.extend_from_slice(block);
+        self.bytes.len() - start
+    }
+
+    fn into_stream(self) -> SharedPositionStream {
+        SharedPositionStream::new(PositionStreamCodec::PackedDelta, self.offsets, self.bytes)
     }
 }
 
 #[derive(Debug)]
 pub struct PostingListBuilder {
-    pub doc_ids: ExpLinkedList<u32>,
-    pub frequencies: ExpLinkedList<u32>,
-    pub positions: Option<PositionBuilder>,
+    with_positions: bool,
+    posting_tail_codec: PostingTailCodec,
+    encoded_blocks: Option<Box<EncodedBlocks>>,
+    encoded_position_blocks: Option<Box<EncodedPositionBlocks>>,
+    tail_entries: Vec<RawDocInfo>,
+    tail_positions: PositionBlockBuilder,
+    open_doc_id: Option<u32>,
+    open_doc_frequency: u32,
+    open_doc_last_position: Option<u32>,
+    memory_size_bytes: u32,
+    len: u32,
+}
+
+pub(super) struct PostingListBatchBuilder {
+    schema: SchemaRef,
+    postings: ListBuilder<LargeBinaryBuilder>,
+    max_scores: Float32Builder,
+    lengths: UInt32Builder,
+    positions: BatchPositionsBuilder,
+    len: usize,
+}
+
+enum BatchPositionsBuilder {
+    None,
+    Legacy(ListBuilder<ListBuilder<LargeBinaryBuilder>>),
+    Shared {
+        bytes: LargeBinaryBuilder,
+        block_offsets: ListBuilder<UInt32Builder>,
+    },
+}
+
+struct PostingListParts<'a> {
+    with_positions: bool,
+    posting_tail_codec: PostingTailCodec,
+    length: usize,
+    encoded_blocks: EncodedBlocks,
+    encoded_position_blocks: EncodedPositionBlocks,
+    tail_entries: &'a [RawDocInfo],
+    tail_position_block: Option<Vec<u8>>,
+}
+
+impl PostingListBatchBuilder {
+    pub fn new(
+        schema: SchemaRef,
+        with_positions: bool,
+        format_version: InvertedListFormatVersion,
+        capacity: usize,
+    ) -> Self {
+        let positions = if !with_positions {
+            BatchPositionsBuilder::None
+        } else if format_version.uses_shared_position_stream() {
+            BatchPositionsBuilder::Shared {
+                bytes: LargeBinaryBuilder::with_capacity(capacity, 0),
+                block_offsets: ListBuilder::with_capacity(UInt32Builder::new(), capacity),
+            }
+        } else {
+            BatchPositionsBuilder::Legacy(ListBuilder::with_capacity(
+                ListBuilder::new(LargeBinaryBuilder::new()),
+                capacity,
+            ))
+        };
+        Self {
+            schema,
+            postings: ListBuilder::with_capacity(LargeBinaryBuilder::new(), capacity),
+            max_scores: Float32Builder::with_capacity(capacity),
+            lengths: UInt32Builder::with_capacity(capacity),
+            positions,
+            len: 0,
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len == 0
+    }
+
+    fn append(
+        &mut self,
+        compressed: LargeBinaryArray,
+        max_score: f32,
+        length: u32,
+        positions: Option<&CompressedPositionStorage>,
+    ) -> Result<()> {
+        {
+            let values = self.postings.values();
+            for index in 0..compressed.len() {
+                values.append_value(compressed.value(index));
+            }
+        }
+        self.postings.append(true);
+        self.max_scores.append_value(max_score);
+        self.lengths.append_value(length);
+
+        match &mut self.positions {
+            BatchPositionsBuilder::None => {}
+            BatchPositionsBuilder::Shared {
+                bytes,
+                block_offsets,
+            } => {
+                let positions = positions.ok_or_else(|| {
+                    Error::index(format!(
+                        "positions builder missing position data for posting length {}",
+                        length
+                    ))
+                })?;
+                let CompressedPositionStorage::SharedStream(positions) = positions else {
+                    return Err(Error::index(
+                        "shared positions builder received legacy positions".to_owned(),
+                    ));
+                };
+                bytes.append_value(positions.bytes());
+                let offsets_builder = block_offsets.values();
+                for &offset in positions.block_offsets() {
+                    offsets_builder.append_value(offset);
+                }
+                block_offsets.append(true);
+            }
+            BatchPositionsBuilder::Legacy(position_lists) => {
+                let positions = positions.ok_or_else(|| {
+                    Error::index(format!(
+                        "positions builder missing position data for posting length {}",
+                        length
+                    ))
+                })?;
+                let CompressedPositionStorage::LegacyPerDoc(positions) = positions else {
+                    return Err(Error::index(
+                        "legacy positions builder received shared position stream".to_owned(),
+                    ));
+                };
+                let docs_builder = position_lists.values();
+                for doc_idx in 0..positions.len() {
+                    let doc_positions = positions.value(doc_idx);
+                    let compressed_positions = doc_positions.as_binary::<i64>();
+                    for block_idx in 0..compressed_positions.len() {
+                        docs_builder
+                            .values()
+                            .append_value(compressed_positions.value(block_idx));
+                    }
+                    docs_builder.append(true);
+                }
+                position_lists.append(true);
+            }
+        }
+
+        self.len += 1;
+        Ok(())
+    }
+
+    pub fn finish(&mut self) -> Result<RecordBatch> {
+        let mut columns = vec![
+            Arc::new(self.postings.finish()) as ArrayRef,
+            Arc::new(self.max_scores.finish()) as ArrayRef,
+            Arc::new(self.lengths.finish()) as ArrayRef,
+        ];
+        match &mut self.positions {
+            BatchPositionsBuilder::None => {}
+            BatchPositionsBuilder::Legacy(position_lists) => {
+                columns.push(Arc::new(position_lists.finish()) as ArrayRef);
+            }
+            BatchPositionsBuilder::Shared {
+                bytes,
+                block_offsets,
+            } => {
+                columns.push(Arc::new(bytes.finish()) as ArrayRef);
+                columns.push(Arc::new(block_offsets.finish()) as ArrayRef);
+            }
+        }
+        self.len = 0;
+        RecordBatch::try_new(self.schema.clone(), columns).map_err(Error::from)
+    }
 }
 
 impl PostingListBuilder {
     pub fn size(&self) -> u64 {
-        (std::mem::size_of::<u32>() * self.doc_ids.len()
-            + std::mem::size_of::<u32>() * self.frequencies.len()
-            + self
-                .positions
-                .as_ref()
-                .map(|positions| positions.size())
-                .unwrap_or(0)) as u64
+        self.memory_size_bytes as u64
     }
 
     pub fn has_positions(&self) -> bool {
-        self.positions.is_some()
+        self.with_positions
     }
 
     pub fn new(with_position: bool) -> Self {
+        Self::new_with_posting_tail_codec(
+            with_position,
+            current_fts_format_version().posting_tail_codec(),
+        )
+    }
+
+    pub fn new_with_posting_tail_codec(
+        with_position: bool,
+        posting_tail_codec: PostingTailCodec,
+    ) -> Self {
         Self {
-            doc_ids: ExpLinkedList::new().with_capacity_limit(128),
-            frequencies: ExpLinkedList::new().with_capacity_limit(128),
-            positions: with_position.then(PositionBuilder::new),
+            with_positions: with_position,
+            posting_tail_codec,
+            encoded_blocks: None,
+            encoded_position_blocks: None,
+            tail_entries: Vec::new(),
+            tail_positions: PositionBlockBuilder::default(),
+            open_doc_id: None,
+            open_doc_frequency: 0,
+            open_doc_last_position: None,
+            len: 0,
+            memory_size_bytes: 0,
         }
     }
 
     pub fn len(&self) -> usize {
-        self.doc_ids.len()
+        self.len as usize
     }
 
     pub fn is_empty(&self) -> bool {
-        self.len() == 0
+        self.len == 0
     }
 
-    pub fn iter(&self) -> impl Iterator<Item = (&u32, &u32, Option<&[u32]>)> {
-        self.doc_ids
-            .iter()
-            .zip(self.frequencies.iter())
-            .enumerate()
-            .map(|(idx, (doc_id, freq))| {
-                let positions = self.positions.as_ref().map(|positions| positions.get(idx));
-                (doc_id, freq, positions)
-            })
+    pub fn iter(&self) -> std::vec::IntoIter<(u32, u32, Option<Vec<u32>>)> {
+        self.collect_entries().into_iter()
+    }
+
+    pub fn for_each_entry<E>(
+        &self,
+        mut visit: impl FnMut(u32, u32, Option<Vec<u32>>) -> std::result::Result<(), E>,
+    ) -> std::result::Result<(), E> {
+        let mut doc_ids = Vec::with_capacity(BLOCK_SIZE);
+        let mut frequencies = Vec::with_capacity(BLOCK_SIZE);
+        let mut decoded_positions = Vec::new();
+        let mut position_block_index = 0usize;
+
+        if let Some(encoded_blocks) = self.encoded_blocks.as_deref() {
+            for block in encoded_blocks.iter() {
+                doc_ids.clear();
+                frequencies.clear();
+                super::encoding::decode_full_posting_block(block, &mut doc_ids, &mut frequencies);
+                decoded_positions.clear();
+                if self.with_positions {
+                    let position_blocks = self
+                        .encoded_position_blocks
+                        .as_deref()
+                        .expect("positions must exist for posting list");
+                    super::encoding::decode_position_stream_block(
+                        position_blocks.block(position_block_index),
+                        &frequencies,
+                        PositionStreamCodec::PackedDelta,
+                        &mut decoded_positions,
+                    )
+                    .expect("position stream decoding should succeed");
+                    position_block_index += 1;
+                }
+                let mut offset = 0usize;
+                for (doc_id, frequency) in doc_ids.iter().copied().zip(frequencies.iter().copied())
+                {
+                    let positions = self.with_positions.then(|| {
+                        let end = offset + frequency as usize;
+                        let doc_positions = decoded_positions[offset..end].to_vec();
+                        offset = end;
+                        doc_positions
+                    });
+                    visit(doc_id, frequency, positions)?;
+                }
+            }
+        }
+
+        let mut decoded_tail_positions = Vec::new();
+        if self.with_positions && !self.tail_entries.is_empty() {
+            let tail_frequencies = self
+                .tail_entries
+                .iter()
+                .map(|entry| entry.frequency)
+                .collect::<Vec<_>>();
+            self.tail_positions
+                .decode_into(tail_frequencies.as_slice(), &mut decoded_tail_positions)
+                .expect("tail position stream decoding should succeed");
+        }
+        let mut tail_offset = 0usize;
+        for entry in &self.tail_entries {
+            let positions = self.with_positions.then(|| {
+                let end = tail_offset + entry.frequency as usize;
+                let doc_positions = decoded_tail_positions[tail_offset..end].to_vec();
+                tail_offset = end;
+                doc_positions
+            });
+            visit(entry.doc_id, entry.frequency, positions)?;
+        }
+
+        Ok(())
     }
 
     pub fn add(&mut self, doc_id: u32, term_positions: PositionRecorder) {
-        self.doc_ids.push(doc_id);
-        self.frequencies.push(term_positions.len());
-        if let Some(positions) = self.positions.as_mut() {
-            positions.push(term_positions.into_vec());
+        debug_assert!(
+            self.open_doc_id.is_none(),
+            "cannot add closed doc while a positions doc is still open"
+        );
+        let tail_entries_capacity_before = self.tail_entries.capacity();
+        self.tail_entries
+            .push(RawDocInfo::new(doc_id, term_positions.len()));
+        let tail_entries_capacity_after = self.tail_entries.capacity();
+        if tail_entries_capacity_after > tail_entries_capacity_before {
+            self.add_memory_bytes(
+                (tail_entries_capacity_after - tail_entries_capacity_before)
+                    * std::mem::size_of::<RawDocInfo>(),
+            );
+        }
+        if let PositionRecorder::Position(positions_in_doc) = term_positions {
+            debug_assert!(self.with_positions);
+            let old_size = self.tail_positions.size();
+            self.tail_positions
+                .append_doc_positions(positions_in_doc.as_slice())
+                .expect("position stream encoding should succeed");
+            self.adjust_tail_positions_size(old_size);
+        }
+        self.len += 1;
+
+        if self.tail_entries.len() == BLOCK_SIZE {
+            self.flush_tail_block()
+                .expect("posting list block compression should succeed");
+        }
+    }
+
+    pub fn add_occurrence(&mut self, doc_id: u32, position: u32) -> Result<bool> {
+        if !self.with_positions {
+            return Err(Error::index(
+                "cannot append streamed positions to a posting list without positions".to_owned(),
+            ));
+        }
+
+        match self.open_doc_id {
+            Some(open_doc_id) if open_doc_id == doc_id => {
+                let old_size = self.tail_positions.size();
+                self.tail_positions
+                    .append_position(position, self.open_doc_last_position)?;
+                self.adjust_tail_positions_size(old_size);
+                self.open_doc_frequency += 1;
+                self.open_doc_last_position = Some(position);
+                Ok(false)
+            }
+            Some(open_doc_id) => Err(Error::index(format!(
+                "posting list received doc {} before finishing open doc {}",
+                doc_id, open_doc_id
+            ))),
+            None => {
+                let old_size = self.tail_positions.size();
+                self.tail_positions.append_position(position, None)?;
+                self.adjust_tail_positions_size(old_size);
+                self.open_doc_id = Some(doc_id);
+                self.open_doc_frequency = 1;
+                self.open_doc_last_position = Some(position);
+                self.len += 1;
+                Ok(true)
+            }
+        }
+    }
+
+    pub fn finish_open_doc(&mut self, doc_id: u32) -> Result<()> {
+        if !self.with_positions {
+            return Ok(());
+        }
+        match self.open_doc_id {
+            Some(open_doc_id) if open_doc_id == doc_id => {
+                let tail_entries_capacity_before = self.tail_entries.capacity();
+                self.tail_entries
+                    .push(RawDocInfo::new(doc_id, self.open_doc_frequency));
+                let tail_entries_capacity_after = self.tail_entries.capacity();
+                if tail_entries_capacity_after > tail_entries_capacity_before {
+                    self.add_memory_bytes(
+                        (tail_entries_capacity_after - tail_entries_capacity_before)
+                            * std::mem::size_of::<RawDocInfo>(),
+                    );
+                }
+                self.open_doc_id = None;
+                self.open_doc_frequency = 0;
+                self.open_doc_last_position = None;
+                if self.tail_entries.len() == BLOCK_SIZE {
+                    self.flush_tail_block()?;
+                }
+                Ok(())
+            }
+            Some(open_doc_id) => Err(Error::index(format!(
+                "attempted to finish doc {} while doc {} is still open",
+                doc_id, open_doc_id
+            ))),
+            None => Ok(()),
+        }
+    }
+
+    fn collect_entries(&self) -> Vec<(u32, u32, Option<Vec<u32>>)> {
+        let mut entries = Vec::with_capacity(self.len());
+        self.for_each_entry(|doc_id, frequency, positions| {
+            entries.push((doc_id, frequency, positions));
+            Ok::<(), ()>(())
+        })
+        .expect("collecting posting list entries should not fail");
+        entries
+    }
+
+    fn encoded_blocks_mut(&mut self) -> &mut EncodedBlocks {
+        if self.encoded_blocks.is_none() {
+            self.encoded_blocks = Some(Box::default());
+            self.add_memory_bytes(std::mem::size_of::<EncodedBlocks>());
+        }
+        self.encoded_blocks
+            .as_deref_mut()
+            .expect("encoded blocks must exist")
+    }
+
+    fn encoded_position_blocks_mut(&mut self) -> &mut EncodedPositionBlocks {
+        if self.encoded_position_blocks.is_none() {
+            self.encoded_position_blocks = Some(Box::default());
+            self.add_memory_bytes(std::mem::size_of::<EncodedPositionBlocks>());
+        }
+        self.encoded_position_blocks
+            .as_deref_mut()
+            .expect("encoded position blocks must exist")
+    }
+
+    fn flush_tail_block(&mut self) -> Result<()> {
+        if self.tail_entries.is_empty() {
+            return Ok(());
+        }
+        debug_assert!(
+            self.open_doc_id.is_none(),
+            "cannot flush a posting block while a document is still open"
+        );
+        debug_assert_eq!(self.tail_entries.len(), BLOCK_SIZE);
+        let mut doc_ids = [0u32; BLOCK_SIZE];
+        let mut frequencies = [0u32; BLOCK_SIZE];
+        for (index, entry) in self.tail_entries.iter().enumerate() {
+            doc_ids[index] = entry.doc_id;
+            frequencies[index] = entry.frequency;
+        }
+        let encoded_blocks_size_before = self
+            .encoded_blocks
+            .as_ref()
+            .map(|encoded_blocks| encoded_blocks.size())
+            .unwrap_or(0usize);
+        self.encoded_blocks_mut()
+            .push_full_block(&doc_ids, &frequencies)?;
+        let encoded_blocks_size_after = self
+            .encoded_blocks
+            .as_ref()
+            .map(|encoded_blocks| encoded_blocks.size())
+            .unwrap_or(0usize);
+        if encoded_blocks_size_after > encoded_blocks_size_before {
+            self.add_memory_bytes(encoded_blocks_size_after - encoded_blocks_size_before);
+        }
+        if self.with_positions {
+            let encoded_positions_size_before = self
+                .encoded_position_blocks
+                .as_ref()
+                .map(|encoded| encoded.size())
+                .unwrap_or(0usize);
+            let released_tail_positions_bytes = self.tail_positions.size();
+            let tail_position_block = std::mem::take(&mut self.tail_positions).finish();
+            self.encoded_position_blocks_mut()
+                .push_encoded_block(tail_position_block.as_slice());
+            let encoded_positions_size_after = self
+                .encoded_position_blocks
+                .as_ref()
+                .map(|encoded| encoded.size())
+                .unwrap_or(0usize);
+            if released_tail_positions_bytes > 0 {
+                self.subtract_memory_bytes(released_tail_positions_bytes);
+            }
+            if encoded_positions_size_after > encoded_positions_size_before {
+                self.add_memory_bytes(encoded_positions_size_after - encoded_positions_size_before);
+            }
+        }
+        self.tail_entries.clear();
+        Ok(())
+    }
+
+    fn adjust_tail_positions_size(&mut self, old_size: usize) {
+        let new_size = self.tail_positions.size();
+        if new_size > old_size {
+            self.add_memory_bytes(new_size - old_size);
+        } else if old_size > new_size {
+            self.subtract_memory_bytes(old_size - new_size);
+        }
+    }
+
+    fn add_memory_bytes(&mut self, bytes: usize) {
+        self.memory_size_bytes = self
+            .memory_size_bytes
+            .checked_add(
+                u32::try_from(bytes).expect("posting list memory size delta overflowed u32"),
+            )
+            .expect("posting list memory size overflowed u32");
+    }
+
+    fn subtract_memory_bytes(&mut self, bytes: usize) {
+        self.memory_size_bytes = self
+            .memory_size_bytes
+            .checked_sub(
+                u32::try_from(bytes).expect("posting list memory size delta overflowed u32"),
+            )
+            .expect("posting list memory size underflowed u32");
+    }
+
+    fn build_position_columns(
+        positions: Option<CompressedPositionStorage>,
+    ) -> Result<Vec<ArrayRef>> {
+        let Some(positions) = positions else {
+            return Ok(Vec::new());
+        };
+        match positions {
+            CompressedPositionStorage::LegacyPerDoc(positions) => {
+                Ok(vec![Arc::new(ListArray::try_new(
+                    Arc::new(Field::new("item", positions.data_type().clone(), true)),
+                    OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, positions.len() as i32])),
+                    Arc::new(positions) as ArrayRef,
+                    None,
+                )?) as ArrayRef])
+            }
+            CompressedPositionStorage::SharedStream(positions) => {
+                let mut columns = Vec::with_capacity(2);
+                columns.push(
+                    Arc::new(LargeBinaryArray::from(vec![Some(positions.bytes())])) as ArrayRef,
+                );
+
+                let mut offsets_builder = ListBuilder::new(UInt32Builder::new());
+                for &offset in positions.block_offsets() {
+                    offsets_builder.values().append_value(offset);
+                }
+                offsets_builder.append(true);
+                columns.push(Arc::new(offsets_builder.finish()) as ArrayRef);
+                Ok(columns)
+            }
         }
     }
 
@@ -1896,6 +3096,7 @@ impl PostingListBuilder {
         compressed: LargeBinaryArray,
         max_score: f32,
         schema: SchemaRef,
+        positions: Option<CompressedPositionStorage>,
     ) -> Result<RecordBatch> {
         let length = self.len();
         let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, compressed.len() as i32]));
@@ -1911,136 +3112,375 @@ impl PostingListBuilder {
                 length as u32,
             ))) as ArrayRef,
         ];
-
-        if let Some(positions) = self.positions.as_ref() {
-            let mut position_builder = ListBuilder::new(ListBuilder::with_capacity(
-                LargeBinaryBuilder::new(),
-                length,
-            ));
-            for index in 0..length {
-                let positions_in_doc = positions.get(index);
-                let compressed = compress_positions(positions_in_doc)?;
-                let inner_builder = position_builder.values();
-                inner_builder.append_value(compressed.into_iter());
-            }
-            position_builder.append(true);
-            let position_col = position_builder.finish();
-            columns.push(Arc::new(position_col));
-        }
+        columns.extend(Self::build_position_columns(positions)?);
 
         let batch = RecordBatch::try_new(schema, columns)?;
         Ok(batch)
     }
 
-    // assume the posting list is sorted by doc id
-    pub fn to_batch(self, block_max_scores: Vec<f32>) -> Result<RecordBatch> {
-        let max_score = block_max_scores.iter().copied().fold(f32::MIN, f32::max);
-        let schema = inverted_list_schema(self.has_positions());
-        let compressed = compress_posting_list(
-            self.doc_ids.len(),
-            self.doc_ids.iter(),
-            self.frequencies.iter(),
-            block_max_scores.into_iter(),
-        )?;
-        self.build_batch(compressed, max_score, schema)
+    fn build_legacy_positions(&self) -> Result<ListArray> {
+        let mut positions_builder = ListBuilder::new(LargeBinaryBuilder::new());
+        self.for_each_entry(|_doc_id, frequency, positions| {
+            let positions = positions.ok_or_else(|| {
+                Error::index(format!(
+                    "legacy position writer missing positions for frequency {}",
+                    frequency
+                ))
+            })?;
+            let compressed = super::encoding::compress_positions(positions.as_slice())?;
+            for block_idx in 0..compressed.len() {
+                positions_builder
+                    .values()
+                    .append_value(compressed.value(block_idx));
+            }
+            positions_builder.append(true);
+            Ok::<(), Error>(())
+        })?;
+        Ok(positions_builder.finish())
     }
 
-    pub fn to_batch_with_docs(self, docs: &DocSet, schema: SchemaRef) -> Result<RecordBatch> {
-        let length = self.len();
+    pub(super) fn append_to_batch_with_docs(
+        self,
+        docs: &DocSet,
+        batch_builder: &mut PostingListBatchBuilder,
+        format_version: InvertedListFormatVersion,
+    ) -> Result<()> {
+        let legacy_positions =
+            if self.with_positions && !format_version.uses_shared_position_stream() {
+                Some(self.build_legacy_positions()?)
+            } else {
+                None
+            };
+        let Self {
+            with_positions,
+            posting_tail_codec,
+            encoded_blocks,
+            encoded_position_blocks,
+            tail_entries,
+            tail_positions,
+            open_doc_id,
+            open_doc_frequency,
+            open_doc_last_position,
+            len,
+            ..
+        } = self;
+        debug_assert!(open_doc_id.is_none());
+        debug_assert_eq!(open_doc_frequency, 0);
+        debug_assert!(open_doc_last_position.is_none());
+        let parts = PostingListParts {
+            with_positions,
+            posting_tail_codec,
+            length: len as usize,
+            encoded_blocks: encoded_blocks
+                .map(|encoded_blocks| *encoded_blocks)
+                .unwrap_or_default(),
+            encoded_position_blocks: encoded_position_blocks
+                .map(|encoded_positions| *encoded_positions)
+                .unwrap_or_default(),
+            tail_entries: tail_entries.as_slice(),
+            tail_position_block: with_positions.then(|| tail_positions.finish()),
+        };
+        let (compressed, shared_positions, max_score) =
+            Self::build_compressed_with_scores_from_parts(parts, docs)?;
+        let positions = match legacy_positions {
+            Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)),
+            None => shared_positions.map(CompressedPositionStorage::SharedStream),
+        };
+        batch_builder.append(compressed, max_score, len, positions.as_ref())
+    }
+
+    fn extend_tail_components(
+        tail_entries: &[RawDocInfo],
+        doc_ids: &mut Vec<u32>,
+        frequencies: &mut Vec<u32>,
+    ) {
+        doc_ids.clear();
+        frequencies.clear();
+        doc_ids.extend(tail_entries.iter().map(|entry| entry.doc_id));
+        frequencies.extend(tail_entries.iter().map(|entry| entry.frequency));
+    }
+
+    fn build_compressed_with_scores_from_parts(
+        parts: PostingListParts<'_>,
+        docs: &DocSet,
+    ) -> Result<(LargeBinaryArray, Option<SharedPositionStream>, f32)> {
+        let PostingListParts {
+            with_positions,
+            posting_tail_codec,
+            length,
+            mut encoded_blocks,
+            mut encoded_position_blocks,
+            tail_entries,
+            tail_position_block,
+        } = parts;
         let avgdl = docs.average_length();
         let idf_scale = idf(length, docs.len()) * (K1 + 1.0);
-        let (compressed, max_score) = compress_posting_list_with_scores(
-            length,
-            self.doc_ids.iter(),
-            self.frequencies.iter(),
-            |doc_id, freq| {
-                let doc_norm = K1 * (1.0 - B + B * docs.num_tokens(doc_id) as f32 / avgdl);
-                let freq = freq as f32;
-                freq / (freq + doc_norm)
-            },
-            idf_scale,
-        )?;
-        self.build_batch(compressed, max_score, schema)
-    }
+        let mut max_score = f32::MIN;
+        let mut doc_ids = Vec::with_capacity(BLOCK_SIZE);
+        let mut frequencies = Vec::with_capacity(BLOCK_SIZE);
+
+        for index in 0..encoded_blocks.len() {
+            let block = encoded_blocks.block(index);
+            doc_ids.clear();
+            frequencies.clear();
+            super::encoding::decode_full_posting_block(block, &mut doc_ids, &mut frequencies);
+            let block_score = compute_block_score(
+                docs,
+                avgdl,
+                idf_scale,
+                doc_ids.iter().copied(),
+                frequencies.iter().copied(),
+            );
+            max_score = max_score.max(block_score);
+            encoded_blocks.set_block_score(index, block_score);
+        }
 
-    pub fn remap(&mut self, removed: &[u32]) {
-        let mut cursor = 0;
-        let mut new_doc_ids = ExpLinkedList::with_capacity(self.len());
-        let mut new_frequencies = ExpLinkedList::with_capacity(self.len());
-        let mut new_positions = self.positions.as_mut().map(|_| PositionBuilder::new());
-        for (&doc_id, &freq, positions) in self.iter() {
-            while cursor < removed.len() && removed[cursor] < doc_id {
-                cursor += 1;
-            }
-            if cursor < removed.len() && removed[cursor] == doc_id {
-                // this doc is removed
-                continue;
-            }
-            // there are cursor removed docs before this doc
-            // so we need to shift the doc id
-            new_doc_ids.push(doc_id - cursor as u32);
-            new_frequencies.push(freq);
-            if let Some(new_positions) = new_positions.as_mut() {
-                new_positions.push(positions.unwrap().to_vec());
+        if !tail_entries.is_empty() {
+            Self::extend_tail_components(tail_entries, &mut doc_ids, &mut frequencies);
+            let block_score = compute_block_score(
+                docs,
+                avgdl,
+                idf_scale,
+                doc_ids.iter().copied(),
+                frequencies.iter().copied(),
+            );
+            max_score = max_score.max(block_score);
+            encoded_blocks.append_remainder_block_with_codec(
+                doc_ids.as_slice(),
+                frequencies.as_slice(),
+                posting_tail_codec,
+            )?;
+            encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score);
+            if with_positions {
+                encoded_position_blocks.push_encoded_block(
+                    tail_position_block
+                        .as_deref()
+                        .expect("tail position block must exist for postings with positions"),
+                );
             }
         }
 
-        self.doc_ids = new_doc_ids;
-        self.frequencies = new_frequencies;
-        self.positions = new_positions;
+        Ok((
+            encoded_blocks.into_array(),
+            with_positions.then(|| encoded_position_blocks.into_stream()),
+            max_score,
+        ))
     }
-}
 
-#[derive(Debug, Clone, DeepSizeOf)]
-pub struct PositionBuilder {
-    positions: Vec<u32>,
-    offsets: Vec<i32>,
-}
-
-impl Default for PositionBuilder {
-    fn default() -> Self {
-        Self::new()
-    }
-}
+    fn build_compressed_with_block_scores_from_parts(
+        with_positions: bool,
+        posting_tail_codec: PostingTailCodec,
+        mut encoded_blocks: EncodedBlocks,
+        mut encoded_position_blocks: EncodedPositionBlocks,
+        tail_entries: &[RawDocInfo],
+        tail_position_block: Option<Vec<u8>>,
+        mut block_max_scores: impl Iterator<Item = f32>,
+    ) -> Result<(LargeBinaryArray, Option<SharedPositionStream>, f32)> {
+        let mut max_score = f32::MIN;
+        let mut doc_ids = Vec::with_capacity(BLOCK_SIZE);
+        let mut frequencies = Vec::with_capacity(BLOCK_SIZE);
+
+        for index in 0..encoded_blocks.len() {
+            let block_score = block_max_scores
+                .next()
+                .ok_or_else(|| Error::index("missing block max score".to_owned()))?;
+            max_score = max_score.max(block_score);
+            encoded_blocks.set_block_score(index, block_score);
+        }
 
-impl PositionBuilder {
-    pub fn new() -> Self {
-        Self {
-            positions: Vec::new(),
-            offsets: vec![0],
+        if !tail_entries.is_empty() {
+            let block_score = block_max_scores
+                .next()
+                .ok_or_else(|| Error::index("missing tail block max score".to_owned()))?;
+            max_score = max_score.max(block_score);
+            Self::extend_tail_components(tail_entries, &mut doc_ids, &mut frequencies);
+            encoded_blocks.append_remainder_block_with_codec(
+                doc_ids.as_slice(),
+                frequencies.as_slice(),
+                posting_tail_codec,
+            )?;
+            encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score);
+            if with_positions {
+                encoded_position_blocks.push_encoded_block(
+                    tail_position_block
+                        .as_deref()
+                        .expect("tail position block must exist for postings with positions"),
+                );
+            }
         }
-    }
 
-    pub fn size(&self) -> usize {
-        std::mem::size_of::<u32>() * self.positions.len()
-            + std::mem::size_of::<i32>() * self.offsets.len()
+        Ok((
+            encoded_blocks.into_array(),
+            with_positions.then(|| encoded_position_blocks.into_stream()),
+            max_score,
+        ))
     }
 
-    pub fn total_len(&self) -> usize {
-        self.positions.len()
+    pub fn to_batch(self, block_max_scores: Vec<f32>) -> Result<RecordBatch> {
+        let format_version = if self.posting_tail_codec == PostingTailCodec::Fixed32 {
+            InvertedListFormatVersion::V1
+        } else {
+            InvertedListFormatVersion::V2
+        };
+        let schema = inverted_list_schema_for_version(self.has_positions(), format_version);
+        let legacy_positions =
+            if self.with_positions && !format_version.uses_shared_position_stream() {
+                Some(self.build_legacy_positions()?)
+            } else {
+                None
+            };
+        let Self {
+            with_positions,
+            posting_tail_codec,
+            encoded_blocks,
+            encoded_position_blocks,
+            tail_entries,
+            tail_positions,
+            open_doc_id,
+            open_doc_frequency,
+            open_doc_last_position,
+            len,
+            ..
+        } = self;
+        debug_assert!(open_doc_id.is_none());
+        debug_assert_eq!(open_doc_frequency, 0);
+        debug_assert!(open_doc_last_position.is_none());
+        let (compressed, shared_positions, max_score) =
+            Self::build_compressed_with_block_scores_from_parts(
+                with_positions,
+                posting_tail_codec,
+                encoded_blocks
+                    .map(|encoded_blocks| *encoded_blocks)
+                    .unwrap_or_default(),
+                encoded_position_blocks
+                    .map(|encoded_positions| *encoded_positions)
+                    .unwrap_or_default(),
+                tail_entries.as_slice(),
+                with_positions.then(|| tail_positions.finish()),
+                block_max_scores.into_iter(),
+            )?;
+        let builder = Self {
+            with_positions,
+            posting_tail_codec,
+            encoded_blocks: None,
+            encoded_position_blocks: None,
+            tail_entries: Vec::new(),
+            tail_positions: PositionBlockBuilder::default(),
+            open_doc_id: None,
+            open_doc_frequency: 0,
+            open_doc_last_position: None,
+            memory_size_bytes: 0,
+            len,
+        };
+        let positions = match legacy_positions {
+            Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)),
+            None => shared_positions.map(CompressedPositionStorage::SharedStream),
+        };
+        builder.build_batch(compressed, max_score, schema, positions)
     }
 
-    pub fn push(&mut self, positions: Vec<u32>) {
-        self.positions.extend(positions);
-        self.offsets.push(self.positions.len() as i32);
+    pub fn to_batch_with_docs(self, docs: &DocSet, schema: SchemaRef) -> Result<RecordBatch> {
+        let format_version = if schema.column_with_name(POSITION_COL).is_some()
+            && schema.column_with_name(COMPRESSED_POSITION_COL).is_none()
+        {
+            InvertedListFormatVersion::V1
+        } else {
+            InvertedListFormatVersion::V2
+        };
+        let legacy_positions =
+            if self.with_positions && !format_version.uses_shared_position_stream() {
+                Some(self.build_legacy_positions()?)
+            } else {
+                None
+            };
+        let Self {
+            with_positions,
+            posting_tail_codec,
+            encoded_blocks,
+            encoded_position_blocks,
+            tail_entries,
+            tail_positions,
+            open_doc_id,
+            open_doc_frequency,
+            open_doc_last_position,
+            len,
+            ..
+        } = self;
+        debug_assert!(open_doc_id.is_none());
+        debug_assert_eq!(open_doc_frequency, 0);
+        debug_assert!(open_doc_last_position.is_none());
+        let parts = PostingListParts {
+            with_positions,
+            posting_tail_codec,
+            length: len as usize,
+            encoded_blocks: encoded_blocks
+                .map(|encoded_blocks| *encoded_blocks)
+                .unwrap_or_default(),
+            encoded_position_blocks: encoded_position_blocks
+                .map(|encoded_positions| *encoded_positions)
+                .unwrap_or_default(),
+            tail_entries: tail_entries.as_slice(),
+            tail_position_block: with_positions.then(|| tail_positions.finish()),
+        };
+        let (compressed, shared_positions, max_score) =
+            Self::build_compressed_with_scores_from_parts(parts, docs)?;
+        let builder = Self {
+            with_positions,
+            posting_tail_codec,
+            encoded_blocks: None,
+            encoded_position_blocks: None,
+            tail_entries: Vec::new(),
+            tail_positions: PositionBlockBuilder::default(),
+            open_doc_id: None,
+            open_doc_frequency: 0,
+            open_doc_last_position: None,
+            memory_size_bytes: 0,
+            len,
+        };
+        let positions = match legacy_positions {
+            Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)),
+            None => shared_positions.map(CompressedPositionStorage::SharedStream),
+        };
+        builder.build_batch(compressed, max_score, schema, positions)
     }
 
-    pub fn get(&self, i: usize) -> &[u32] {
-        let start = self.offsets[i] as usize;
-        let end = self.offsets[i + 1] as usize;
-        &self.positions[start..end]
+    pub fn remap(&mut self, removed: &[u32]) {
+        let mut cursor = 0;
+        let mut new_builder =
+            Self::new_with_posting_tail_codec(self.has_positions(), self.posting_tail_codec);
+        for (doc_id, freq, positions) in self.iter() {
+            while cursor < removed.len() && removed[cursor] < doc_id {
+                cursor += 1;
+            }
+            if cursor < removed.len() && removed[cursor] == doc_id {
+                continue;
+            }
+            let positions = match positions {
+                Some(positions) => PositionRecorder::Position(positions.into()),
+                None => PositionRecorder::Count(freq),
+            };
+            new_builder.add(doc_id - cursor as u32, positions);
+        }
+
+        *self = new_builder;
     }
 }
 
-impl From<Vec<Vec<u32>>> for PositionBuilder {
-    fn from(positions: Vec<Vec<u32>>) -> Self {
-        let mut builder = Self::new();
-        builder.offsets.reserve(positions.len());
-        for pos in positions {
-            builder.push(pos);
-        }
-        builder
-    }
+fn compute_block_score(
+    docs: &DocSet,
+    avgdl: f32,
+    idf_scale: f32,
+    doc_ids: impl Iterator<Item = u32>,
+    frequencies: impl Iterator<Item = u32>,
+) -> f32 {
+    let mut block_max_score = f32::MIN;
+    for (doc_id, freq) in doc_ids.zip(frequencies) {
+        let doc_norm = K1 * (1.0 - B + B * docs.num_tokens(doc_id) as f32 / avgdl);
+        let freq = freq as f32;
+        let score = freq / (freq + doc_norm);
+        block_max_score = block_max_score.max(score);
+    }
+    block_max_score * idf_scale
 }
 
 #[derive(Debug, Clone, DeepSizeOf, Copy)]
@@ -2381,6 +3821,12 @@ impl DocSet {
         self.total_tokens += num_tokens as u64;
         self.row_ids.len() as u32 - 1
     }
+
+    pub(crate) fn memory_size(&self) -> usize {
+        self.row_ids.capacity() * std::mem::size_of::<u64>()
+            + self.num_tokens.capacity() * std::mem::size_of::<u32>()
+            + self.inv.capacity() * std::mem::size_of::<(u64, u32)>()
+    }
 }
 
 pub fn flat_full_text_search(
@@ -2681,10 +4127,7 @@ pub async fn flat_bm25_search_stream(
     let mut tokenizer = match index {
         Some(index) => index.tokenizer(),
         None => Box::new(TextTokenizer::new(
-            tantivy::tokenizer::TextAnalyzer::builder(
-                tantivy::tokenizer::SimpleTokenizer::default(),
-            )
-            .build(),
+            TextAnalyzer::builder(SimpleTokenizer::default()).build(),
         )),
     };
     let query_tokens = Arc::new(collect_query_tokens(&query, &mut tokenizer));
@@ -2737,61 +4180,440 @@ pub fn is_phrase_query(query: &str) -> bool {
 
 #[cfg(test)]
 mod tests {
-    use crate::scalar::inverted::lance_tokenizer::DocType;
+    use crate::scalar::inverted::document_tokenizer::DocType;
+    use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+    use futures::stream;
     use lance_core::cache::LanceCache;
     use lance_core::utils::tempfile::TempObjDir;
     use lance_io::object_store::ObjectStore;
 
     use crate::metrics::NoOpMetricsCollector;
     use crate::prefilter::NoFilter;
+    use crate::scalar::ScalarIndex;
     use crate::scalar::inverted::builder::{InnerBuilder, PositionRecorder, inverted_list_schema};
-    use crate::scalar::inverted::encoding::decompress_posting_list;
+    use crate::scalar::inverted::encoding::{
+        compress_positions, compress_posting_list_with_tail_codec,
+        decompress_posting_list_with_tail_codec, encode_position_stream_block_into,
+    };
     use crate::scalar::inverted::query::{FtsSearchParams, Operator};
     use crate::scalar::lance_format::LanceIndexStore;
-    use arrow::array::AsArray;
+    use arrow::array::{AsArray, LargeBinaryBuilder, ListBuilder, UInt32Builder};
     use arrow::datatypes::{Float32Type, UInt32Type};
+    use arrow_array::{ArrayRef, Float32Array, RecordBatch, StringArray, UInt32Array, UInt64Array};
+    use arrow_schema::{DataType, Field, Schema};
+    use std::collections::HashMap;
+    use std::sync::Arc;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_posting_builder_remap() {
+        let posting_tail_codec = PostingTailCodec::Fixed32;
+        let mut builder =
+            PostingListBuilder::new_with_posting_tail_codec(false, posting_tail_codec);
+        let n = BLOCK_SIZE + 3;
+        for i in 0..n {
+            builder.add(i as u32, PositionRecorder::Count(1));
+        }
+        let removed = vec![5, 7];
+        builder.remap(&removed);
+
+        let mut expected =
+            PostingListBuilder::new_with_posting_tail_codec(false, posting_tail_codec);
+        for i in 0..n - removed.len() {
+            expected.add(i as u32, PositionRecorder::Count(1));
+        }
+        let expected_entries = expected.iter().collect::<Vec<_>>();
+        let actual_entries = builder.iter().collect::<Vec<_>>();
+        assert_eq!(actual_entries, expected_entries);
+
+        // BLOCK_SIZE + 3 elements should be reduced to BLOCK_SIZE + 1,
+        // there are still 2 blocks.
+        let batch = builder.to_batch(vec![1.0, 2.0]).unwrap();
+        let (doc_ids, freqs) = decompress_posting_list_with_tail_codec(
+            (n - removed.len()) as u32,
+            batch[POSTING_COL]
+                .as_list::<i32>()
+                .value(0)
+                .as_binary::<i64>(),
+            posting_tail_codec,
+        )
+        .unwrap();
+        assert!(
+            doc_ids
+                .iter()
+                .zip(expected_entries.iter().map(|(doc_id, _, _)| doc_id))
+                .all(|(a, b)| a == b)
+        );
+        assert!(
+            freqs
+                .iter()
+                .zip(expected_entries.iter().map(|(_, freq, _)| freq))
+                .all(|(a, b)| a == b)
+        );
+    }
+
+    #[test]
+    fn test_posting_builder_size_tracking_matches_structure() {
+        fn tracked_memory_size(builder: &PostingListBuilder) -> u64 {
+            let encoded_blocks_size = builder
+                .encoded_blocks
+                .iter()
+                .map(|encoded_blocks| std::mem::size_of::<EncodedBlocks>() + encoded_blocks.size())
+                .sum::<usize>();
+            let encoded_positions_size = builder
+                .encoded_position_blocks
+                .as_ref()
+                .map(|positions| std::mem::size_of::<EncodedPositionBlocks>() + positions.size())
+                .unwrap_or(0usize);
+            (encoded_blocks_size
+                + builder.tail_entries.capacity() * std::mem::size_of::<RawDocInfo>()
+                + builder.tail_positions.size()
+                + encoded_positions_size) as u64
+        }
+
+        let mut builder = PostingListBuilder::new(true);
+        for doc_id in 0..(BLOCK_SIZE + 5) as u32 {
+            builder.add(
+                doc_id,
+                PositionRecorder::Position(smallvec::smallvec![1, 3, 5]),
+            );
+        }
+
+        assert_eq!(builder.size(), tracked_memory_size(&builder));
+    }
+
+    #[test]
+    fn test_posting_builder_flush_releases_tail_position_capacity() {
+        let mut builder = PostingListBuilder::new(true);
+        let positions = smallvec::SmallVec::<[u32; 2]>::from_vec((0..1024).collect());
+        for doc_id in 0..BLOCK_SIZE as u32 {
+            builder.add(doc_id, PositionRecorder::Position(positions.clone()));
+        }
+
+        assert_eq!(builder.tail_positions.size(), 0);
+        assert_eq!(builder.size(), {
+            let encoded_blocks_size = builder
+                .encoded_blocks
+                .iter()
+                .map(|encoded_blocks| std::mem::size_of::<EncodedBlocks>() + encoded_blocks.size())
+                .sum::<usize>();
+            let encoded_positions_size = builder
+                .encoded_position_blocks
+                .as_ref()
+                .map(|positions| std::mem::size_of::<EncodedPositionBlocks>() + positions.size())
+                .unwrap_or(0usize);
+            (encoded_blocks_size
+                + builder.tail_entries.capacity() * std::mem::size_of::<RawDocInfo>()
+                + builder.tail_positions.size()
+                + encoded_positions_size) as u64
+        });
+    }
+
+    #[test]
+    fn test_posting_builder_streamed_positions_roundtrip() {
+        let mut builder = PostingListBuilder::new(true);
+        assert!(builder.add_occurrence(0, 1).unwrap());
+        assert!(!builder.add_occurrence(0, 4).unwrap());
+        assert!(!builder.add_occurrence(0, 9).unwrap());
+        builder.finish_open_doc(0).unwrap();
+
+        assert!(builder.add_occurrence(2, 3).unwrap());
+        builder.finish_open_doc(2).unwrap();
+
+        let entries = builder.iter().collect::<Vec<_>>();
+        assert_eq!(
+            entries,
+            vec![
+                (0_u32, 3_u32, Some(vec![1_u32, 4_u32, 9_u32])),
+                (2_u32, 1_u32, Some(vec![3_u32])),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_posting_builder_roundtrip_shared_positions() {
+        let entries = vec![
+            (0_u32, vec![1_u32, 5]),
+            (2, vec![0, 4, 9]),
+            (4, vec![7]),
+            (8, vec![3, 10]),
+            (13, vec![2, 11, 30]),
+        ];
+        let mut builder =
+            PostingListBuilder::new_with_posting_tail_codec(true, PostingTailCodec::VarintDelta);
+        for (doc_id, positions) in &entries {
+            builder.add(
+                *doc_id,
+                PositionRecorder::Position(positions.clone().into()),
+            );
+        }
+
+        let batch = builder.to_batch(vec![1.0]).unwrap();
+        assert!(batch.column_by_name(COMPRESSED_POSITION_COL).is_some());
+        assert!(batch.column_by_name(POSITION_COL).is_none());
+        assert_eq!(
+            batch.schema_ref().metadata().get(POSTING_TAIL_CODEC_KEY),
+            Some(&PostingTailCodec::VarintDelta.as_str().to_owned())
+        );
+        assert_eq!(
+            batch.schema_ref().metadata().get(POSITIONS_LAYOUT_KEY),
+            Some(&POSITIONS_LAYOUT_SHARED_STREAM_V2.to_owned())
+        );
+        assert_eq!(
+            batch.schema_ref().metadata().get(POSITIONS_CODEC_KEY),
+            Some(&PositionStreamCodec::PackedDelta.as_str().to_owned())
+        );
+
+        let posting =
+            PostingList::from_batch(&batch, Some(1.0), Some(entries.len() as u32)).unwrap();
+        let actual = posting
+            .iter()
+            .map(|(doc_id, freq, positions)| {
+                (doc_id as u32, freq, positions.unwrap().collect::<Vec<_>>())
+            })
+            .collect::<Vec<_>>();
+        let expected = entries
+            .iter()
+            .map(|(doc_id, positions)| (*doc_id, positions.len() as u32, positions.clone()))
+            .collect::<Vec<_>>();
+        assert_eq!(actual, expected);
+    }
+
+    #[test]
+    fn test_posting_builder_roundtrip_legacy_positions() {
+        let entries = vec![(0_u32, vec![1_u32, 5]), (2, vec![0, 4, 9]), (4, vec![7])];
+        let mut builder =
+            PostingListBuilder::new_with_posting_tail_codec(true, PostingTailCodec::Fixed32);
+        for (doc_id, positions) in &entries {
+            builder.add(
+                *doc_id,
+                PositionRecorder::Position(positions.clone().into()),
+            );
+        }
+
+        let batch = builder.to_batch(vec![1.0]).unwrap();
+        assert!(batch.column_by_name(POSITION_COL).is_some());
+        assert!(batch.column_by_name(COMPRESSED_POSITION_COL).is_none());
+        assert_eq!(
+            batch.schema_ref().metadata().get(POSTING_TAIL_CODEC_KEY),
+            None
+        );
+        assert_eq!(
+            batch.schema_ref().metadata().get(POSITIONS_LAYOUT_KEY),
+            None
+        );
+        assert_eq!(batch.schema_ref().metadata().get(POSITIONS_CODEC_KEY), None);
+
+        let posting =
+            PostingList::from_batch(&batch, Some(1.0), Some(entries.len() as u32)).unwrap();
+        let actual = posting
+            .iter()
+            .map(|(doc_id, freq, positions)| {
+                (doc_id as u32, freq, positions.unwrap().collect::<Vec<_>>())
+            })
+            .collect::<Vec<_>>();
+        let expected = entries
+            .iter()
+            .map(|(doc_id, positions)| (*doc_id, positions.len() as u32, positions.clone()))
+            .collect::<Vec<_>>();
+        assert_eq!(actual, expected);
+    }
+
+    #[test]
+    fn test_resolve_fts_format_version_defaults_to_v1() {
+        assert_eq!(
+            resolve_fts_format_version(None).unwrap(),
+            InvertedListFormatVersion::V1
+        );
+        assert_eq!(
+            resolve_fts_format_version(Some("2")).unwrap(),
+            InvertedListFormatVersion::V2
+        );
+    }
+
+    #[test]
+    fn test_legacy_compressed_positions_still_readable() {
+        let doc_ids = [1_u32, 3_u32];
+        let frequencies = [2_u32, 3_u32];
+        let posting = compress_posting_list_with_tail_codec(
+            doc_ids.len(),
+            doc_ids.iter(),
+            frequencies.iter(),
+            std::iter::once(1.0_f32),
+            PostingTailCodec::Fixed32,
+        )
+        .unwrap();
+
+        let mut posting_builder = ListBuilder::new(LargeBinaryBuilder::new());
+        for idx in 0..posting.len() {
+            posting_builder.values().append_value(posting.value(idx));
+        }
+        posting_builder.append(true);
+
+        let mut positions_builder = ListBuilder::new(ListBuilder::new(LargeBinaryBuilder::new()));
+        for positions in [vec![1_u32, 5_u32], vec![0_u32, 4_u32, 9_u32]] {
+            let compressed = compress_positions(&positions).unwrap();
+            let doc_builder = positions_builder.values();
+            for idx in 0..compressed.len() {
+                doc_builder.values().append_value(compressed.value(idx));
+            }
+            doc_builder.append(true);
+        }
+        positions_builder.append(true);
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new(
+                POSTING_COL,
+                DataType::List(Arc::new(Field::new("item", DataType::LargeBinary, true))),
+                false,
+            ),
+            Field::new(MAX_SCORE_COL, DataType::Float32, false),
+            Field::new(LENGTH_COL, DataType::UInt32, false),
+            Field::new(
+                POSITION_COL,
+                DataType::List(Arc::new(Field::new(
+                    "item",
+                    DataType::List(Arc::new(Field::new("item", DataType::LargeBinary, true))),
+                    true,
+                ))),
+                false,
+            ),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(posting_builder.finish()) as ArrayRef,
+                Arc::new(Float32Array::from(vec![1.0])) as ArrayRef,
+                Arc::new(UInt32Array::from(vec![doc_ids.len() as u32])) as ArrayRef,
+                Arc::new(positions_builder.finish()) as ArrayRef,
+            ],
+        )
+        .unwrap();
+
+        let posting =
+            PostingList::from_batch(&batch, Some(1.0), Some(doc_ids.len() as u32)).unwrap();
+        let actual = posting
+            .iter()
+            .map(|(doc_id, freq, positions)| {
+                (doc_id as u32, freq, positions.unwrap().collect::<Vec<_>>())
+            })
+            .collect::<Vec<_>>();
+        assert_eq!(actual, vec![(1, 2, vec![1, 5]), (3, 3, vec![0, 4, 9]),]);
+    }
+
+    #[test]
+    fn test_shared_stream_v2_without_codec_still_readable() {
+        let doc_ids = [1_u32, 3_u32];
+        let frequencies = [2_u32, 3_u32];
+        let posting = compress_posting_list_with_tail_codec(
+            doc_ids.len(),
+            doc_ids.iter(),
+            frequencies.iter(),
+            std::iter::once(1.0_f32),
+            PostingTailCodec::Fixed32,
+        )
+        .unwrap();
+
+        let mut posting_builder = ListBuilder::new(LargeBinaryBuilder::new());
+        for idx in 0..posting.len() {
+            posting_builder.values().append_value(posting.value(idx));
+        }
+        posting_builder.append(true);
+
+        let positions = vec![1_u32, 5_u32, 0_u32, 4_u32, 9_u32];
+        let mut encoded_positions = Vec::new();
+        encode_position_stream_block_into(
+            &positions,
+            &frequencies,
+            PositionStreamCodec::VarintDocDelta,
+            &mut encoded_positions,
+        )
+        .unwrap();
+
+        let mut position_offsets = ListBuilder::new(UInt32Builder::new());
+        position_offsets.values().append_value(0);
+        position_offsets.append(true);
+
+        let schema = Arc::new(Schema::new_with_metadata(
+            vec![
+                Field::new(
+                    POSTING_COL,
+                    DataType::List(Arc::new(Field::new("item", DataType::LargeBinary, true))),
+                    false,
+                ),
+                Field::new(MAX_SCORE_COL, DataType::Float32, false),
+                Field::new(LENGTH_COL, DataType::UInt32, false),
+                Field::new(COMPRESSED_POSITION_COL, DataType::LargeBinary, false),
+                Field::new(
+                    POSITION_BLOCK_OFFSET_COL,
+                    DataType::List(Arc::new(Field::new("item", DataType::UInt32, true))),
+                    false,
+                ),
+            ],
+            HashMap::from([(
+                POSITIONS_LAYOUT_KEY.to_owned(),
+                POSITIONS_LAYOUT_SHARED_STREAM_V2.to_owned(),
+            )]),
+        ));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(posting_builder.finish()) as ArrayRef,
+                Arc::new(Float32Array::from(vec![1.0])) as ArrayRef,
+                Arc::new(UInt32Array::from(vec![doc_ids.len() as u32])) as ArrayRef,
+                Arc::new(arrow_array::LargeBinaryArray::from(vec![Some(
+                    encoded_positions.as_slice(),
+                )])) as ArrayRef,
+                Arc::new(position_offsets.finish()) as ArrayRef,
+            ],
+        )
+        .unwrap();
 
-    use super::*;
+        let posting =
+            PostingList::from_batch(&batch, Some(1.0), Some(doc_ids.len() as u32)).unwrap();
+        let actual = posting
+            .iter()
+            .map(|(doc_id, freq, positions)| {
+                (doc_id as u32, freq, positions.unwrap().collect::<Vec<_>>())
+            })
+            .collect::<Vec<_>>();
+        assert_eq!(actual, vec![(1, 2, vec![1, 5]), (3, 3, vec![0, 4, 9]),]);
+    }
 
-    #[tokio::test]
-    async fn test_posting_builder_remap() {
-        let mut builder = PostingListBuilder::new(false);
-        let n = BLOCK_SIZE + 3;
-        for i in 0..n {
-            builder.add(i as u32, PositionRecorder::Count(1));
+    #[test]
+    fn test_shared_position_stream_is_smaller_for_sparse_positions() {
+        let mut builder =
+            PostingListBuilder::new_with_posting_tail_codec(true, PostingTailCodec::VarintDelta);
+        let mut legacy_positions = Vec::with_capacity(BLOCK_SIZE * 4);
+        for doc_id in 0..(BLOCK_SIZE * 4) as u32 {
+            let mut positions = vec![doc_id * 3 + 1];
+            if doc_id % 8 == 0 {
+                positions.push(doc_id * 3 + 2);
+            }
+            builder.add(doc_id, PositionRecorder::Position(positions.clone().into()));
+            legacy_positions.push(positions);
         }
-        let removed = vec![5, 7];
-        builder.remap(&removed);
 
-        let mut expected = PostingListBuilder::new(false);
-        for i in 0..n - removed.len() {
-            expected.add(i as u32, PositionRecorder::Count(1));
+        let batch = builder.to_batch(vec![1.0; 4]).unwrap();
+        let shared_positions_size = batch[COMPRESSED_POSITION_COL].get_buffer_memory_size()
+            + batch[POSITION_BLOCK_OFFSET_COL].get_buffer_memory_size();
+
+        let mut positions_builder = ListBuilder::new(ListBuilder::new(LargeBinaryBuilder::new()));
+        for positions in legacy_positions {
+            let compressed = compress_positions(&positions).unwrap();
+            let doc_builder = positions_builder.values();
+            for idx in 0..compressed.len() {
+                doc_builder.values().append_value(compressed.value(idx));
+            }
+            doc_builder.append(true);
         }
-        assert_eq!(builder.doc_ids, expected.doc_ids);
-        assert_eq!(builder.frequencies, expected.frequencies);
+        positions_builder.append(true);
+        let legacy_positions_size = positions_builder.finish().get_buffer_memory_size();
 
-        // BLOCK_SIZE + 3 elements should be reduced to BLOCK_SIZE + 1,
-        // there are still 2 blocks.
-        let batch = builder.to_batch(vec![1.0, 2.0]).unwrap();
-        let (doc_ids, freqs) = decompress_posting_list(
-            (n - removed.len()) as u32,
-            batch[POSTING_COL]
-                .as_list::<i32>()
-                .value(0)
-                .as_binary::<i64>(),
-        )
-        .unwrap();
-        assert!(
-            doc_ids
-                .iter()
-                .zip(expected.doc_ids.iter())
-                .all(|(a, b)| a == b)
-        );
         assert!(
-            freqs
-                .iter()
-                .zip(expected.frequencies.iter())
-                .all(|(a, b)| a == b)
+            shared_positions_size < legacy_positions_size,
+            "expected shared position stream to be smaller than legacy per-doc storage, shared={shared_positions_size}, legacy={legacy_positions_size}",
         );
     }
 
@@ -2893,21 +4715,310 @@ mod tests {
         assert_eq!(builder.docs.row_id(0), 1);
         assert_eq!(builder.docs.row_id(1), 3);
 
-        builder.write(store.as_ref()).await.unwrap();
+        builder.write(store.as_ref()).await.unwrap();
+
+        // remap to delete all docs
+        let mapping = HashMap::from([(1, None), (3, None)]);
+        builder.remap(&mapping).await.unwrap();
+
+        assert_eq!(builder.tokens.len(), 0);
+        assert_eq!(builder.posting_lists.len(), 0);
+        assert_eq!(builder.docs.len(), 0);
+
+        builder.write(store.as_ref()).await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_posting_cache_conflict_across_partitions() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // Create first partition with one token and posting list length 1
+        let mut builder1 = InnerBuilder::new(0, false, TokenSetFormat::default());
+        builder1.tokens.add("test".to_owned());
+        builder1.posting_lists.push(PostingListBuilder::new(false));
+        builder1.posting_lists[0].add(0, PositionRecorder::Count(1));
+        builder1.docs.append(100, 1); // row_id=100, num_tokens=1
+        builder1.write(store.as_ref()).await.unwrap();
+
+        // Create second partition with one token and posting list length 4
+        let mut builder2 = InnerBuilder::new(1, false, TokenSetFormat::default());
+        builder2.tokens.add("test".to_owned()); // Use same token to test cache prefix fix
+        builder2.posting_lists.push(PostingListBuilder::new(false));
+        builder2.posting_lists[0].add(0, PositionRecorder::Count(2));
+        builder2.posting_lists[0].add(1, PositionRecorder::Count(1));
+        builder2.posting_lists[0].add(2, PositionRecorder::Count(3));
+        builder2.posting_lists[0].add(3, PositionRecorder::Count(1));
+        builder2.docs.append(200, 2); // row_id=200, num_tokens=2
+        builder2.docs.append(201, 1); // row_id=201, num_tokens=1
+        builder2.docs.append(202, 3); // row_id=202, num_tokens=3
+        builder2.docs.append(203, 1); // row_id=203, num_tokens=1
+        builder2.write(store.as_ref()).await.unwrap();
+
+        // Create metadata file with both partitions
+        let metadata = std::collections::HashMap::from_iter(vec![
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&vec![0u64, 1u64]).unwrap(),
+            ),
+            (
+                "params".to_owned(),
+                serde_json::to_string(&InvertedIndexParams::default()).unwrap(),
+            ),
+            (
+                TOKEN_SET_FORMAT_KEY.to_owned(),
+                TokenSetFormat::default().to_string(),
+            ),
+        ]);
+        let mut writer = store
+            .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+
+        // Load the inverted index
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+
+        // Verify the index structure
+        assert_eq!(index.partitions.len(), 2);
+        assert_eq!(index.partitions[0].tokens.len(), 1);
+        assert_eq!(index.partitions[1].tokens.len(), 1);
+
+        // Verify the partitions were loaded correctly
+
+        // Verify posting list lengths (note: partition order may differ from creation order)
+        // Verify based on actual loading order
+        if index.partitions[0].id() == 0 {
+            // If partition[0] is ID=0, then it should have 1 document
+            assert_eq!(index.partitions[0].inverted_list.posting_len(0), 1);
+            assert_eq!(index.partitions[1].inverted_list.posting_len(0), 4);
+            assert_eq!(index.partitions[0].docs.len(), 1);
+            assert_eq!(index.partitions[1].docs.len(), 4);
+        } else {
+            // If partition[0] is ID=1, then it should have 4 documents
+            assert_eq!(index.partitions[0].inverted_list.posting_len(0), 4);
+            assert_eq!(index.partitions[1].inverted_list.posting_len(0), 1);
+            assert_eq!(index.partitions[0].docs.len(), 4);
+            assert_eq!(index.partitions[1].docs.len(), 1);
+        }
+
+        // Prewarm the inverted index (this loads posting lists into cache)
+        index.prewarm().await.unwrap();
+
+        let tokens = Arc::new(Tokens::new(vec!["test".to_string()], DocType::Text));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(10)));
+        let prefilter = Arc::new(NoFilter);
+        let metrics = Arc::new(NoOpMetricsCollector);
+
+        let (row_ids, scores) = index
+            .bm25_search(tokens, params, Operator::Or, prefilter, metrics)
+            .await
+            .unwrap();
+
+        // Verify that we got search results
+        // Expected to find 5 documents: 1 from first partition, 4 from second partition
+        assert_eq!(row_ids.len(), 5, "row_ids: {:?}", row_ids);
+        assert!(!row_ids.is_empty(), "Should find at least some documents");
+        assert_eq!(row_ids.len(), scores.len());
+
+        // All scores should be positive since all documents contain the search token
+        for &score in &scores {
+            assert!(score > 0.0, "All scores should be positive");
+        }
+
+        // Check that we got results from both partitions
+        assert!(
+            row_ids.contains(&100),
+            "Should contain row_id from partition 0"
+        );
+        assert!(
+            row_ids.iter().any(|&id| id >= 200),
+            "Should contain row_id from partition 1"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_modern_prewarm_shrinks_cached_posting_buffers() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        builder.tokens.add("alpha".to_owned());
+        builder.tokens.add("beta".to_owned());
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists[0].add(0, PositionRecorder::Count(1));
+        builder.posting_lists[0].add(1, PositionRecorder::Count(2));
+        builder.posting_lists[1].add(2, PositionRecorder::Count(3));
+        builder.posting_lists[1].add(3, PositionRecorder::Count(4));
+        builder.docs.append(100, 1);
+        builder.docs.append(101, 2);
+        builder.docs.append(102, 3);
+        builder.docs.append(103, 4);
+        builder.write(store.as_ref()).await.unwrap();
+
+        let metadata = std::collections::HashMap::from_iter(vec![
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&vec![0u64]).unwrap(),
+            ),
+            (
+                "params".to_owned(),
+                serde_json::to_string(&InvertedIndexParams::default()).unwrap(),
+            ),
+            (
+                TOKEN_SET_FORMAT_KEY.to_owned(),
+                TokenSetFormat::default().to_string(),
+            ),
+        ]);
+        let mut writer = store
+            .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+        let inverted_list = &index.partitions[0].inverted_list;
+        assert!(
+            inverted_list.offsets.is_none(),
+            "test should use modern posting layout"
+        );
+
+        inverted_list.prewarm_posting_lists(false).await.unwrap();
+
+        let alpha = inverted_list
+            .index_cache
+            .get_with_key(&PostingListKey { token_id: 0 })
+            .await
+            .unwrap();
+        let beta = inverted_list
+            .index_cache
+            .get_with_key(&PostingListKey { token_id: 1 })
+            .await
+            .unwrap();
+
+        let PostingList::Compressed(alpha) = alpha.as_ref() else {
+            panic!("expected compressed posting list for token 0");
+        };
+        let PostingList::Compressed(beta) = beta.as_ref() else {
+            panic!("expected compressed posting list for token 1");
+        };
+
+        assert_ne!(
+            alpha.blocks.values().as_ptr(),
+            beta.blocks.values().as_ptr(),
+            "prewarm should not leave cached posting lists sharing the same values buffer"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_prewarm_with_positions_populates_separate_position_cache() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new_with_format_version(
+            0,
+            true,
+            TokenSetFormat::default(),
+            InvertedListFormatVersion::V1,
+        );
+        builder.tokens.add("hello".to_owned());
+        builder.tokens.add("world".to_owned());
+        builder
+            .posting_lists
+            .push(PostingListBuilder::new_with_posting_tail_codec(
+                true,
+                PostingTailCodec::Fixed32,
+            ));
+        builder
+            .posting_lists
+            .push(PostingListBuilder::new_with_posting_tail_codec(
+                true,
+                PostingTailCodec::Fixed32,
+            ));
+        builder.posting_lists[0].add(0, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[1].add(0, PositionRecorder::Position(vec![1].into()));
+        builder.posting_lists[0].add(1, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[1].add(1, PositionRecorder::Position(vec![2].into()));
+        builder.docs.append(100, 2);
+        builder.docs.append(101, 2);
+        builder.write(store.as_ref()).await.unwrap();
+
+        let metadata = std::collections::HashMap::from_iter(vec![
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&vec![0_u64]).unwrap(),
+            ),
+            (
+                "params".to_owned(),
+                serde_json::to_string(&InvertedIndexParams::default().with_position(true)).unwrap(),
+            ),
+            (
+                TOKEN_SET_FORMAT_KEY.to_owned(),
+                TokenSetFormat::default().to_string(),
+            ),
+        ]);
+        let mut writer = store
+            .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
 
-        // remap to delete all docs
-        let mapping = HashMap::from([(1, None), (3, None)]);
-        builder.remap(&mapping).await.unwrap();
+        index
+            .prewarm_with_options(&FtsPrewarmOptions::new().with_position(true))
+            .await
+            .unwrap();
 
-        assert_eq!(builder.tokens.len(), 0);
-        assert_eq!(builder.posting_lists.len(), 0);
-        assert_eq!(builder.docs.len(), 0);
+        let inverted_list = &index.partitions[0].inverted_list;
+        let posting = inverted_list
+            .index_cache
+            .get_with_key(&PostingListKey { token_id: 0 })
+            .await
+            .unwrap();
+        assert!(
+            !posting.has_position(),
+            "posting cache should remain positions-free after prewarm"
+        );
 
-        builder.write(store.as_ref()).await.unwrap();
+        let positions = inverted_list
+            .index_cache
+            .get_with_key(&PositionKey { token_id: 0 })
+            .await
+            .unwrap();
+        assert!(
+            matches!(
+                positions.as_ref().0,
+                CompressedPositionStorage::LegacyPerDoc(_)
+            ),
+            "positions should be stored in the dedicated position cache"
+        );
     }
 
     #[tokio::test]
-    async fn test_posting_cache_conflict_across_partitions() {
+    async fn test_prewarm_with_v2_positions_preserves_shared_stream_codec() {
         let tmpdir = TempObjDir::default();
         let store = Arc::new(LanceIndexStore::new(
             ObjectStore::local().into(),
@@ -2915,42 +5026,54 @@ mod tests {
             Arc::new(LanceCache::no_cache()),
         ));
 
-        // Create first partition with one token and posting list length 1
-        let mut builder1 = InnerBuilder::new(0, false, TokenSetFormat::default());
-        builder1.tokens.add("test".to_owned());
-        builder1.posting_lists.push(PostingListBuilder::new(false));
-        builder1.posting_lists[0].add(0, PositionRecorder::Count(1));
-        builder1.docs.append(100, 1); // row_id=100, num_tokens=1
-        builder1.write(store.as_ref()).await.unwrap();
-
-        // Create second partition with one token and posting list length 4
-        let mut builder2 = InnerBuilder::new(1, false, TokenSetFormat::default());
-        builder2.tokens.add("test".to_owned()); // Use same token to test cache prefix fix
-        builder2.posting_lists.push(PostingListBuilder::new(false));
-        builder2.posting_lists[0].add(0, PositionRecorder::Count(2));
-        builder2.posting_lists[0].add(1, PositionRecorder::Count(1));
-        builder2.posting_lists[0].add(2, PositionRecorder::Count(3));
-        builder2.posting_lists[0].add(3, PositionRecorder::Count(1));
-        builder2.docs.append(200, 2); // row_id=200, num_tokens=2
-        builder2.docs.append(201, 1); // row_id=201, num_tokens=1
-        builder2.docs.append(202, 3); // row_id=202, num_tokens=3
-        builder2.docs.append(203, 1); // row_id=203, num_tokens=1
-        builder2.write(store.as_ref()).await.unwrap();
+        let format_version = InvertedListFormatVersion::V2;
+        let posting_tail_codec = format_version.posting_tail_codec();
+        let mut builder = InnerBuilder::new_with_format_version(
+            0,
+            true,
+            TokenSetFormat::default(),
+            format_version,
+        );
+        builder.tokens.add("body".to_owned());
+
+        let mut posting_list =
+            PostingListBuilder::new_with_posting_tail_codec(true, posting_tail_codec);
+        let expected = (0..(BLOCK_SIZE + 5) as u32)
+            .map(|doc_id| {
+                let positions = vec![doc_id % 3, doc_id % 3 + 2, doc_id % 3 + 5];
+                posting_list.add(doc_id, PositionRecorder::Position(positions.clone().into()));
+                builder.docs.append(30_000 + doc_id as u64, 20 + doc_id % 7);
+                (doc_id, positions.len() as u32, positions)
+            })
+            .collect::<Vec<_>>();
+        builder.posting_lists.push(posting_list);
+        builder.write(store.as_ref()).await.unwrap();
 
-        // Create metadata file with both partitions
-        let metadata = std::collections::HashMap::from_iter(vec![
+        let metadata = HashMap::from([
             (
                 "partitions".to_owned(),
-                serde_json::to_string(&vec![0u64, 1u64]).unwrap(),
+                serde_json::to_string(&vec![0_u64]).unwrap(),
             ),
             (
                 "params".to_owned(),
-                serde_json::to_string(&InvertedIndexParams::default()).unwrap(),
+                serde_json::to_string(&InvertedIndexParams::default().with_position(true)).unwrap(),
             ),
             (
                 TOKEN_SET_FORMAT_KEY.to_owned(),
                 TokenSetFormat::default().to_string(),
             ),
+            (
+                POSTING_TAIL_CODEC_KEY.to_owned(),
+                posting_tail_codec.as_str().to_owned(),
+            ),
+            (
+                POSITIONS_LAYOUT_KEY.to_owned(),
+                POSITIONS_LAYOUT_SHARED_STREAM_V2.to_owned(),
+            ),
+            (
+                POSITIONS_CODEC_KEY.to_owned(),
+                PositionStreamCodec::PackedDelta.as_str().to_owned(),
+            ),
         ]);
         let mut writer = store
             .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
@@ -2958,68 +5081,27 @@ mod tests {
             .unwrap();
         writer.finish_with_metadata(metadata).await.unwrap();
 
-        // Load the inverted index
         let cache = Arc::new(LanceCache::with_capacity(4096));
-        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+        let index = InvertedIndex::load(store, None, cache.as_ref())
             .await
             .unwrap();
-
-        // Verify the index structure
-        assert_eq!(index.partitions.len(), 2);
-        assert_eq!(index.partitions[0].tokens.len(), 1);
-        assert_eq!(index.partitions[1].tokens.len(), 1);
-
-        // Verify the partitions were loaded correctly
-
-        // Verify posting list lengths (note: partition order may differ from creation order)
-        // Verify based on actual loading order
-        if index.partitions[0].id() == 0 {
-            // If partition[0] is ID=0, then it should have 1 document
-            assert_eq!(index.partitions[0].inverted_list.posting_len(0), 1);
-            assert_eq!(index.partitions[1].inverted_list.posting_len(0), 4);
-            assert_eq!(index.partitions[0].docs.len(), 1);
-            assert_eq!(index.partitions[1].docs.len(), 4);
-        } else {
-            // If partition[0] is ID=1, then it should have 4 documents
-            assert_eq!(index.partitions[0].inverted_list.posting_len(0), 4);
-            assert_eq!(index.partitions[1].inverted_list.posting_len(0), 1);
-            assert_eq!(index.partitions[0].docs.len(), 4);
-            assert_eq!(index.partitions[1].docs.len(), 1);
-        }
-
-        // Prewarm the inverted index (this loads posting lists into cache)
-        index.prewarm().await.unwrap();
-
-        let tokens = Arc::new(Tokens::new(vec!["test".to_string()], DocType::Text));
-        let params = Arc::new(FtsSearchParams::new().with_limit(Some(10)));
-        let prefilter = Arc::new(NoFilter);
-        let metrics = Arc::new(NoOpMetricsCollector);
-
-        let (row_ids, scores) = index
-            .bm25_search(tokens, params, Operator::Or, prefilter, metrics)
+        index
+            .prewarm_with_options(&FtsPrewarmOptions::new().with_position(true))
             .await
             .unwrap();
 
-        // Verify that we got search results
-        // Expected to find 5 documents: 1 from first partition, 4 from second partition
-        assert_eq!(row_ids.len(), 5, "row_ids: {:?}", row_ids);
-        assert!(!row_ids.is_empty(), "Should find at least some documents");
-        assert_eq!(row_ids.len(), scores.len());
-
-        // All scores should be positive since all documents contain the search token
-        for &score in &scores {
-            assert!(score > 0.0, "All scores should be positive");
-        }
+        let actual = index.partitions[0]
+            .inverted_list
+            .posting_list(0, true, &NoOpMetricsCollector)
+            .await
+            .unwrap()
+            .iter()
+            .map(|(doc_id, freq, positions)| {
+                (doc_id as u32, freq, positions.unwrap().collect::<Vec<_>>())
+            })
+            .collect::<Vec<_>>();
 
-        // Check that we got results from both partitions
-        assert!(
-            row_ids.contains(&100),
-            "Should contain row_id from partition 0"
-        );
-        assert!(
-            row_ids.iter().any(|&id| id >= 200),
-            "Should contain row_id from partition 1"
-        );
+        assert_eq!(actual, expected);
     }
 
     #[test]
@@ -3120,4 +5202,221 @@ mod tests {
             );
         }
     }
+
+    #[tokio::test]
+    async fn test_phrase_query_reads_legacy_per_doc_positions() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new_with_format_version(
+            0,
+            true,
+            TokenSetFormat::default(),
+            InvertedListFormatVersion::V1,
+        );
+        builder.tokens.add("hello".to_owned());
+        builder.tokens.add("world".to_owned());
+        builder
+            .posting_lists
+            .push(PostingListBuilder::new_with_posting_tail_codec(
+                true,
+                PostingTailCodec::Fixed32,
+            ));
+        builder
+            .posting_lists
+            .push(PostingListBuilder::new_with_posting_tail_codec(
+                true,
+                PostingTailCodec::Fixed32,
+            ));
+        builder.posting_lists[0].add(0, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[1].add(0, PositionRecorder::Position(vec![1].into()));
+        builder.posting_lists[0].add(1, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[1].add(1, PositionRecorder::Position(vec![2].into()));
+        builder.docs.append(100, 2);
+        builder.docs.append(101, 2);
+        builder.write(store.as_ref()).await.unwrap();
+
+        let metadata = std::collections::HashMap::from_iter(vec![
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&vec![0_u64]).unwrap(),
+            ),
+            (
+                "params".to_owned(),
+                serde_json::to_string(&InvertedIndexParams::default().with_position(true)).unwrap(),
+            ),
+            (
+                TOKEN_SET_FORMAT_KEY.to_owned(),
+                TokenSetFormat::default().to_string(),
+            ),
+        ]);
+        let mut writer = store
+            .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::new(
+            vec!["hello".to_owned(), "world".to_owned()],
+            DocType::Text,
+        ));
+        let params = Arc::new(
+            FtsSearchParams::new()
+                .with_limit(Some(10))
+                .with_phrase_slop(Some(0)),
+        );
+        let prefilter = Arc::new(NoFilter);
+        let metrics = Arc::new(NoOpMetricsCollector);
+
+        let (row_ids, _scores) = index
+            .bm25_search(tokens, params, Operator::And, prefilter, metrics)
+            .await
+            .unwrap();
+
+        assert_eq!(row_ids, vec![100]);
+    }
+
+    #[tokio::test]
+    async fn test_update_preserves_loaded_v2_format_version() -> Result<()> {
+        let src_dir = TempObjDir::default();
+        let dest_dir = TempObjDir::default();
+        let src_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            src_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let dest_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            dest_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let format_version = InvertedListFormatVersion::V2;
+        let posting_tail_codec = format_version.posting_tail_codec();
+        let mut partition = InnerBuilder::new_with_format_version(
+            0,
+            false,
+            TokenSetFormat::default(),
+            format_version,
+        );
+        partition.tokens.add("hello".to_owned());
+        let mut posting_list =
+            PostingListBuilder::new_with_posting_tail_codec(false, posting_tail_codec);
+        posting_list.add(0, PositionRecorder::Count(1));
+        partition.posting_lists.push(posting_list);
+        partition.docs.append(100, 1);
+        partition.write(src_store.as_ref()).await?;
+
+        let metadata = HashMap::from([
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&vec![0_u64]).unwrap(),
+            ),
+            (
+                "params".to_owned(),
+                serde_json::to_string(&InvertedIndexParams::default()).unwrap(),
+            ),
+            (
+                TOKEN_SET_FORMAT_KEY.to_owned(),
+                TokenSetFormat::default().to_string(),
+            ),
+            (
+                POSTING_TAIL_CODEC_KEY.to_owned(),
+                posting_tail_codec.as_str().to_owned(),
+            ),
+        ]);
+        let mut writer = src_store
+            .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+
+        let index = InvertedIndex::load(src_store, None, &LanceCache::no_cache()).await?;
+        assert_eq!(index.index_version(), format_version.index_version());
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("doc", DataType::Utf8, true),
+            Field::new(ROW_ID, DataType::UInt64, false),
+        ]));
+        let docs = Arc::new(StringArray::from(vec![Some("hello again")]));
+        let row_ids = Arc::new(UInt64Array::from(vec![101u64]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![docs, row_ids])?;
+        let stream = RecordBatchStreamAdapter::new(schema, stream::iter(vec![Ok(batch)]));
+        let created = index
+            .update(Box::pin(stream), dest_store.as_ref(), None)
+            .await?;
+
+        assert_eq!(created.index_version, format_version.index_version());
+
+        let updated = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?;
+        assert_eq!(updated.index_version(), format_version.index_version());
+        assert_eq!(updated.partitions.len(), 2);
+        for partition in &updated.partitions {
+            assert_eq!(
+                partition.inverted_list.posting_tail_codec(),
+                posting_tail_codec
+            );
+        }
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_modern_index_without_deleted_col_has_empty_bitmap() {
+        // An index created before the deleted_fragments feature was added
+        // will have a metadata file with num_rows=0 (no record batch data).
+        // The load path should gracefully handle this with an empty bitmap.
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        builder.tokens.add("test".to_owned());
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists[0].add(0, PositionRecorder::Count(1));
+        builder.docs.append(100, 1);
+        builder.write(store.as_ref()).await.unwrap();
+
+        // Write a metadata file WITHOUT the deleted_fragments column
+        // (simulates an older index version)
+        let metadata = std::collections::HashMap::from_iter(vec![
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&vec![0u64]).unwrap(),
+            ),
+            (
+                "params".to_owned(),
+                serde_json::to_string(&InvertedIndexParams::default()).unwrap(),
+            ),
+            (
+                TOKEN_SET_FORMAT_KEY.to_owned(),
+                TokenSetFormat::default().to_string(),
+            ),
+        ]);
+        let mut writer = store
+            .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+
+        let index = InvertedIndex::load(store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        assert!(
+            index.deleted_fragments().is_empty(),
+            "index without deleted_fragments column should have empty bitmap"
+        );
+    }
 }
diff --git a/rust/lance-index/src/scalar/inverted/iter.rs b/rust/lance-index/src/scalar/inverted/iter.rs
index fcbb35fa5a8..dc07b15769c 100644
--- a/rust/lance-index/src/scalar/inverted/iter.rs
+++ b/rust/lance-index/src/scalar/inverted/iter.rs
@@ -2,12 +2,15 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 use arrow::array::AsArray;
-use arrow_array::{Array, LargeBinaryArray, ListArray};
+use arrow_array::{Array, LargeBinaryArray};
 
 use super::{
-    PostingList,
+    CompressedPositionStorage, PostingList, PostingTailCodec,
     builder::BLOCK_SIZE,
-    encoding::{decompress_positions, decompress_posting_block, decompress_posting_remainder},
+    encoding::{
+        decode_position_stream_block, decompress_positions, decompress_posting_block,
+        decompress_posting_remainder,
+    },
 };
 
 pub enum PostingListIterator<'a> {
@@ -23,6 +26,7 @@ impl<'a> PostingListIterator<'a> {
                 Self::Compressed(Box::new(CompressedPostingListIterator::new(
                     posting.length as usize,
                     posting.blocks.clone(),
+                    posting.posting_tail_codec,
                     posting.positions.clone(),
                 )))
             }
@@ -48,20 +52,52 @@ impl<'a> Iterator for PostingListIterator<'a> {
 pub type PlainPostingListIterator<'a> =
     Box<dyn Iterator<Item = (u64, f32, Option<Box<dyn Iterator<Item = u32> + 'a>>)> + 'a>;
 
+struct OwnedPositionsIter {
+    positions: Box<[u32]>,
+    index: usize,
+}
+
+impl OwnedPositionsIter {
+    fn new(positions: &[u32]) -> Self {
+        Self {
+            positions: Box::<[u32]>::from(positions),
+            index: 0,
+        }
+    }
+}
+
+impl Iterator for OwnedPositionsIter {
+    type Item = u32;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let position = self.positions.get(self.index).copied()?;
+        self.index += 1;
+        Some(position)
+    }
+}
+
 pub struct CompressedPostingListIterator {
     remainder: usize,
     blocks: LargeBinaryArray,
     next_block_idx: usize,
-    positions: Option<ListArray>,
-    idx: u32,
-    iter: InnerIterator,
+    posting_tail_codec: PostingTailCodec,
+    positions: Option<CompressedPositionStorage>,
+    idx: usize,
+    doc_ids: Vec<u32>,
+    frequencies: Vec<u32>,
+    doc_idx_in_block: usize,
+    decoded_positions: Vec<u32>,
+    position_offsets: Vec<usize>,
     buffer: [u32; BLOCK_SIZE],
 }
 
-type InnerIterator = std::iter::Zip<std::vec::IntoIter<u32>, std::vec::IntoIter<u32>>;
-
 impl CompressedPostingListIterator {
-    pub fn new(length: usize, blocks: LargeBinaryArray, positions: Option<ListArray>) -> Self {
+    pub fn new(
+        length: usize,
+        blocks: LargeBinaryArray,
+        posting_tail_codec: PostingTailCodec,
+        positions: Option<CompressedPositionStorage>,
+    ) -> Self {
         debug_assert!(length > 0, "length: {}", length);
         debug_assert_eq!(
             length.div_ceil(BLOCK_SIZE),
@@ -75,9 +111,14 @@ impl CompressedPostingListIterator {
             remainder: length % BLOCK_SIZE,
             blocks,
             next_block_idx: 0,
+            posting_tail_codec,
             positions,
             idx: 0,
-            iter: std::iter::zip(Vec::new(), Vec::new()),
+            doc_ids: Vec::new(),
+            frequencies: Vec::new(),
+            doc_idx_in_block: 0,
+            decoded_positions: Vec::new(),
+            position_offsets: Vec::new(),
             buffer: [0; BLOCK_SIZE],
         }
     }
@@ -87,13 +128,24 @@ impl Iterator for CompressedPostingListIterator {
     type Item = (u32, u32, Option<Box<dyn Iterator<Item = u32>>>);
 
     fn next(&mut self) -> Option<Self::Item> {
-        if let Some((doc_id, freq)) = self.iter.next() {
-            let positions = self.positions.as_ref().map(|p| {
-                let compressed = p.value(self.idx as usize);
-                let positions = decompress_positions(compressed.as_binary());
-                Box::new(positions.into_iter()) as _
+        if self.doc_idx_in_block < self.doc_ids.len() {
+            let doc_id = self.doc_ids[self.doc_idx_in_block];
+            let freq = self.frequencies[self.doc_idx_in_block];
+            let positions = self.positions.as_ref().map(|storage| match storage {
+                CompressedPositionStorage::LegacyPerDoc(list) => {
+                    let compressed = list.value(self.idx);
+                    let positions = decompress_positions(compressed.as_binary());
+                    Box::new(positions.into_iter()) as Box<dyn Iterator<Item = u32>>
+                }
+                CompressedPositionStorage::SharedStream(_) => {
+                    let start = self.position_offsets[self.doc_idx_in_block];
+                    let end = self.position_offsets[self.doc_idx_in_block + 1];
+                    Box::new(OwnedPositionsIter::new(&self.decoded_positions[start..end]))
+                        as Box<dyn Iterator<Item = u32>>
+                }
             });
             self.idx += 1;
+            self.doc_idx_in_block += 1;
             return Some((doc_id, freq, positions));
         }
 
@@ -104,19 +156,43 @@ impl Iterator for CompressedPostingListIterator {
         let compressed = self.blocks.value(self.next_block_idx);
         self.next_block_idx += 1;
 
-        let mut doc_ids = Vec::with_capacity(BLOCK_SIZE);
-        let mut frequencies = Vec::with_capacity(BLOCK_SIZE);
+        self.doc_ids.clear();
+        self.frequencies.clear();
         if self.next_block_idx == self.blocks.len() && self.remainder > 0 {
             decompress_posting_remainder(
                 compressed,
                 self.remainder,
-                &mut doc_ids,
-                &mut frequencies,
+                self.posting_tail_codec,
+                &mut self.doc_ids,
+                &mut self.frequencies,
             );
         } else {
-            decompress_posting_block(compressed, &mut self.buffer, &mut doc_ids, &mut frequencies);
+            decompress_posting_block(
+                compressed,
+                &mut self.buffer,
+                &mut self.doc_ids,
+                &mut self.frequencies,
+            );
+        }
+        self.doc_idx_in_block = 0;
+        self.decoded_positions.clear();
+        self.position_offsets.clear();
+        if let Some(CompressedPositionStorage::SharedStream(stream)) = self.positions.as_ref() {
+            decode_position_stream_block(
+                stream.block(self.next_block_idx - 1),
+                self.frequencies.as_slice(),
+                stream.codec(),
+                &mut self.decoded_positions,
+            )
+            .expect("shared position stream decoding should succeed");
+            self.position_offsets.reserve(self.frequencies.len() + 1);
+            self.position_offsets.push(0);
+            let mut offset = 0usize;
+            for &frequency in &self.frequencies {
+                offset += frequency as usize;
+                self.position_offsets.push(offset);
+            }
         }
-        self.iter = std::iter::zip(doc_ids, frequencies);
         self.next()
     }
 }
diff --git a/rust/lance-index/src/scalar/inverted/merger.rs b/rust/lance-index/src/scalar/inverted/merger.rs
deleted file mode 100644
index 500adacac93..00000000000
--- a/rust/lance-index/src/scalar/inverted/merger.rs
+++ /dev/null
@@ -1,426 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The Lance Authors
-
-use fst::Streamer;
-use futures::{StreamExt, TryStreamExt, stream};
-use lance_core::{Error, Result, cache::LanceCache, utils::tokio::get_num_compute_intensive_cpus};
-use std::sync::Arc;
-
-use crate::progress::IndexBuildProgress;
-use crate::scalar::IndexStore;
-
-use super::{
-    InvertedPartition, PostingListBuilder, TokenMap, TokenSetFormat,
-    builder::{InnerBuilder, PositionRecorder, doc_file_path, posting_file_path, token_file_path},
-};
-
-pub trait Merger {
-    // Merge the partitions and write new partitions,
-    // the new partitions are returned.
-    // This method streams partitions with bounded buffering to avoid
-    // loading all partitions into memory at once.
-    async fn merge(&mut self) -> Result<Vec<u64>>;
-}
-
-#[derive(Debug, Clone)]
-pub(super) struct PartitionSource {
-    store: std::sync::Arc<dyn IndexStore>,
-    id: u64,
-}
-
-impl PartitionSource {
-    pub(super) fn new(store: std::sync::Arc<dyn IndexStore>, id: u64) -> Self {
-        Self { store, id }
-    }
-
-    async fn load(
-        &self,
-        cache: &LanceCache,
-        token_set_format: TokenSetFormat,
-    ) -> Result<InvertedPartition> {
-        InvertedPartition::load(self.store.clone(), self.id, None, cache, token_set_format).await
-    }
-}
-
-// A merger that merges partitions based on their size,
-// it would read the posting lists for each token from
-// the partitions and write them to a new partition,
-// until the size of the new partition reaches the target size.
-pub struct SizeBasedMerger<'a> {
-    dest_store: &'a dyn IndexStore,
-    input: Vec<PartitionSource>,
-    with_position: Option<bool>,
-    target_size: u64,
-    token_set_format: TokenSetFormat,
-    progress: Arc<dyn IndexBuildProgress>,
-    builder: Option<InnerBuilder>,
-    next_id: u64,
-    partitions: Vec<u64>,
-}
-
-impl<'a> SizeBasedMerger<'a> {
-    // Create a new SizeBasedMerger with the target size,
-    // the size is compressed size in byte.
-    // Typically, just set the size to the memory limit,
-    // because less partitions means faster query.
-    pub fn new(
-        dest_store: &'a dyn IndexStore,
-        input: Vec<PartitionSource>,
-        target_size: u64,
-        token_set_format: TokenSetFormat,
-        progress: Arc<dyn IndexBuildProgress>,
-    ) -> Self {
-        let max_id = input.iter().map(|p| p.id).max().unwrap_or(0);
-
-        Self {
-            dest_store,
-            input,
-            with_position: None,
-            target_size,
-            token_set_format,
-            progress,
-            builder: None,
-            next_id: max_id + 1,
-            partitions: Vec::new(),
-        }
-    }
-
-    async fn flush(&mut self) -> Result<()> {
-        let Some(builder) = self.builder.as_mut() else {
-            return Ok(());
-        };
-
-        if !builder.tokens.is_empty() {
-            log::info!("flushing partition {}", builder.id());
-            let start = std::time::Instant::now();
-            builder.write(self.dest_store).await?;
-            log::info!(
-                "flushed partition {} in {:?}",
-                builder.id(),
-                start.elapsed()
-            );
-            self.partitions.push(builder.id());
-            let with_position = self.with_position.expect("with_position must be set");
-            let next_id = self.next_id;
-            self.builder = Some(InnerBuilder::new(
-                next_id,
-                with_position,
-                self.token_set_format,
-            ));
-            self.next_id += 1;
-        }
-        Ok(())
-    }
-
-    fn ensure_builder(&mut self, part: &InvertedPartition) -> Result<()> {
-        let with_position = part.inverted_list.has_positions();
-        match self.with_position {
-            Some(existing) => {
-                if existing != with_position {
-                    return Err(Error::index(
-                        "partition position settings do not match".to_string(),
-                    ));
-                }
-            }
-            None => {
-                self.with_position = Some(with_position);
-            }
-        }
-
-        if self.builder.is_none() {
-            let with_position = self.with_position.expect("with_position must be set");
-            self.builder = Some(InnerBuilder::new(
-                self.next_id,
-                with_position,
-                self.token_set_format,
-            ));
-            self.next_id += 1;
-        }
-        Ok(())
-    }
-
-    async fn merge_partition(
-        &mut self,
-        part: InvertedPartition,
-        estimated_size: &mut u64,
-    ) -> Result<()> {
-        self.ensure_builder(&part)?;
-
-        {
-            let builder = self.builder.as_ref().expect("builder must exist");
-            if builder.docs.len() + part.docs.len() > u32::MAX as usize
-                || *estimated_size >= self.target_size
-            {
-                self.flush().await?;
-                *estimated_size = 0;
-                self.ensure_builder(&part)?;
-            }
-        }
-
-        let builder = self.builder.as_mut().expect("builder must exist");
-        let mut token_id_map = vec![u32::MAX; part.tokens.len()];
-        match &part.tokens.tokens {
-            TokenMap::HashMap(map) => {
-                for (token, token_id) in map.iter() {
-                    let new_token_id = builder.tokens.get_or_add(token.as_str());
-                    let index = *token_id as usize;
-                    debug_assert!(index < token_id_map.len());
-                    token_id_map[index] = new_token_id;
-                }
-            }
-            TokenMap::Fst(map) => {
-                let mut stream = map.stream();
-                while let Some((token, token_id)) = stream.next() {
-                    let token_id = token_id as u32;
-                    let token = String::from_utf8_lossy(token);
-                    let new_token_id = builder.tokens.get_or_add(token.as_ref());
-                    let index = token_id as usize;
-                    debug_assert!(index < token_id_map.len());
-                    token_id_map[index] = new_token_id;
-                }
-            }
-        }
-        let doc_id_offset = builder.docs.len() as u32;
-        for (row_id, num_tokens) in part.docs.iter() {
-            builder.docs.append(*row_id, *num_tokens);
-        }
-        builder.posting_lists.resize_with(builder.tokens.len(), || {
-            PostingListBuilder::new(part.inverted_list.has_positions())
-        });
-
-        let postings = part
-            .inverted_list
-            .read_batch(part.inverted_list.has_positions())
-            .await?;
-        for token_id in 0..part.tokens.len() as u32 {
-            let posting_list = part
-                .inverted_list
-                .posting_list_from_batch(&postings.slice(token_id as usize, 1), token_id)?;
-            let new_token_id = token_id_map[token_id as usize];
-            debug_assert_ne!(new_token_id, u32::MAX);
-            let builder = &mut builder.posting_lists[new_token_id as usize];
-            let old_size = builder.size();
-            for (doc_id, freq, positions) in posting_list.iter() {
-                let new_doc_id = doc_id_offset + doc_id as u32;
-                let positions = match positions {
-                    Some(positions) => PositionRecorder::Position(positions.collect()),
-                    None => PositionRecorder::Count(freq),
-                };
-                builder.add(new_doc_id, positions);
-            }
-            let new_size = builder.size();
-            *estimated_size += new_size - old_size;
-        }
-        Ok(())
-    }
-}
-
-impl Merger for SizeBasedMerger<'_> {
-    async fn merge(&mut self) -> Result<Vec<u64>> {
-        if self.input.len() <= 1 {
-            let mut completed = 0;
-            for part in self.input.iter() {
-                part.store
-                    .copy_index_file(&token_file_path(part.id), self.dest_store)
-                    .await?;
-                part.store
-                    .copy_index_file(&posting_file_path(part.id), self.dest_store)
-                    .await?;
-                part.store
-                    .copy_index_file(&doc_file_path(part.id), self.dest_store)
-                    .await?;
-                completed += 1;
-                self.progress
-                    .stage_progress("merge_partitions", completed)
-                    .await?;
-            }
-
-            return Ok(self.input.iter().map(|p| p.id).collect());
-        }
-
-        // for token set, union the tokens,
-        // for doc set, concatenate the row ids, assign the doc id to offset + doc_id
-        // for posting list, concatenate the posting lists
-        log::info!(
-            "merging {} partitions with target size {} MiB",
-            self.input.len(),
-            self.target_size / 1024 / 1024
-        );
-        let mut estimated_size = 0;
-        let start = std::time::Instant::now();
-        let parts = std::mem::take(&mut self.input);
-        let num_parts = parts.len();
-        let buffer_size = std::cmp::max(
-            1,
-            std::cmp::min(get_num_compute_intensive_cpus(), num_parts),
-        );
-        let cache = LanceCache::no_cache();
-        let token_set_format = self.token_set_format;
-        let mut stream = stream::iter(parts.into_iter().map(|part| {
-            let cache = cache.clone();
-            tokio::task::spawn(async move { part.load(&cache, token_set_format).await })
-        }))
-        .buffered(buffer_size);
-
-        let mut idx = 0;
-        while let Some(part) = stream.try_next().await? {
-            let part = part?;
-            idx += 1;
-            self.merge_partition(part, &mut estimated_size).await?;
-            self.progress
-                .stage_progress("merge_partitions", idx as u64)
-                .await?;
-            log::info!(
-                "merged {}/{} partitions in {:?}",
-                idx,
-                num_parts,
-                start.elapsed()
-            );
-        }
-
-        self.flush().await?;
-        Ok(self.partitions.clone())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::metrics::NoOpMetricsCollector;
-    use crate::scalar::lance_format::LanceIndexStore;
-    use lance_core::cache::LanceCache;
-    use lance_core::utils::tempfile::TempObjDir;
-    use lance_io::object_store::ObjectStore;
-    use std::sync::Arc;
-
-    #[tokio::test]
-    async fn test_merge_reuses_token_ids_for_shared_tokens() -> Result<()> {
-        let src_dir = TempObjDir::default();
-        let dest_dir = TempObjDir::default();
-        let src_store = Arc::new(LanceIndexStore::new(
-            ObjectStore::local().into(),
-            src_dir.clone(),
-            Arc::new(LanceCache::no_cache()),
-        ));
-        let dest_store = Arc::new(LanceIndexStore::new(
-            ObjectStore::local().into(),
-            dest_dir.clone(),
-            Arc::new(LanceCache::no_cache()),
-        ));
-
-        let token_set_format = TokenSetFormat::default();
-
-        let mut builder0 = InnerBuilder::new(0, false, token_set_format);
-        let apple_id = builder0.tokens.add("apple".to_owned());
-        let banana_id = builder0.tokens.add("banana".to_owned());
-        builder0
-            .posting_lists
-            .resize_with(builder0.tokens.len(), || PostingListBuilder::new(false));
-        let doc_id = builder0.docs.append(10, 2);
-        builder0.posting_lists[apple_id as usize].add(doc_id, PositionRecorder::Count(1));
-        builder0.posting_lists[banana_id as usize].add(doc_id, PositionRecorder::Count(1));
-        builder0.write(src_store.as_ref()).await?;
-
-        let mut builder1 = InnerBuilder::new(1, false, token_set_format);
-        let banana_id = builder1.tokens.add("banana".to_owned());
-        let carrot_id = builder1.tokens.add("carrot".to_owned());
-        builder1
-            .posting_lists
-            .resize_with(builder1.tokens.len(), || PostingListBuilder::new(false));
-        let doc_id = builder1.docs.append(20, 2);
-        builder1.posting_lists[banana_id as usize].add(doc_id, PositionRecorder::Count(1));
-        builder1.posting_lists[carrot_id as usize].add(doc_id, PositionRecorder::Count(1));
-        builder1.write(src_store.as_ref()).await?;
-
-        let mut merger = SizeBasedMerger::new(
-            dest_store.as_ref(),
-            vec![
-                PartitionSource::new(src_store.clone(), 0),
-                PartitionSource::new(src_store.clone(), 1),
-            ],
-            u64::MAX,
-            token_set_format,
-            crate::progress::noop_progress(),
-        );
-        let merged_partitions = merger.merge().await?;
-        assert_eq!(merged_partitions, vec![2]);
-
-        let merged = InvertedPartition::load(
-            dest_store.clone(),
-            merged_partitions[0],
-            None,
-            &LanceCache::no_cache(),
-            token_set_format,
-        )
-        .await?;
-
-        assert_eq!(merged.tokens.len(), 3);
-        assert_eq!(merged.docs.len(), 2);
-        assert_eq!(merged.docs.row_id(0), 10);
-        assert_eq!(merged.docs.row_id(1), 20);
-
-        let banana_token_id = merged.tokens.get("banana").unwrap();
-        let posting = merged
-            .inverted_list
-            .posting_list(banana_token_id, false, &NoOpMetricsCollector)
-            .await?;
-        let doc_ids: Vec<u64> = posting.iter().map(|(doc_id, _, _)| doc_id).collect();
-        assert_eq!(doc_ids, vec![0, 1]);
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_merge_streams_partitions_in_batches() -> Result<()> {
-        let src_dir = TempObjDir::default();
-        let dest_dir = TempObjDir::default();
-        let src_store = Arc::new(LanceIndexStore::new(
-            ObjectStore::local().into(),
-            src_dir.clone(),
-            Arc::new(LanceCache::no_cache()),
-        ));
-        let dest_store = Arc::new(LanceIndexStore::new(
-            ObjectStore::local().into(),
-            dest_dir.clone(),
-            Arc::new(LanceCache::no_cache()),
-        ));
-
-        let token_set_format = TokenSetFormat::default();
-        let num_parts = get_num_compute_intensive_cpus().saturating_add(2);
-        let mut sources = Vec::with_capacity(num_parts);
-        for id in 0..num_parts as u64 {
-            let mut builder = InnerBuilder::new(id, false, token_set_format);
-            let token_id = builder.tokens.add(format!("token_{}", id));
-            builder
-                .posting_lists
-                .resize_with(builder.tokens.len(), || PostingListBuilder::new(false));
-            let doc_id = builder.docs.append(id * 10, 1);
-            builder.posting_lists[token_id as usize].add(doc_id, PositionRecorder::Count(1));
-            builder.write(src_store.as_ref()).await?;
-            sources.push(PartitionSource::new(src_store.clone(), id));
-        }
-
-        let mut merger = SizeBasedMerger::new(
-            dest_store.as_ref(),
-            sources,
-            u64::MAX,
-            token_set_format,
-            crate::progress::noop_progress(),
-        );
-        let merged_partitions = merger.merge().await?;
-        assert_eq!(merged_partitions, vec![num_parts as u64]);
-
-        let merged = InvertedPartition::load(
-            dest_store.clone(),
-            merged_partitions[0],
-            None,
-            &LanceCache::no_cache(),
-            token_set_format,
-        )
-        .await?;
-        assert_eq!(merged.tokens.len(), num_parts);
-        assert_eq!(merged.docs.len(), num_parts);
-
-        Ok(())
-    }
-}
diff --git a/rust/lance-index/src/scalar/inverted/query.rs b/rust/lance-index/src/scalar/inverted/query.rs
index a422a4524b3..21b631d12ad 100644
--- a/rust/lance-index/src/scalar/inverted/query.rs
+++ b/rust/lance-index/src/scalar/inverted/query.rs
@@ -1,8 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use crate::scalar::inverted::lance_tokenizer::DocType;
-use crate::scalar::inverted::tokenizer::lance_tokenizer::LanceTokenizer;
+use crate::scalar::inverted::document_tokenizer::DocType;
+use crate::scalar::inverted::tokenizer::document_tokenizer::LanceTokenizer;
 use lance_core::{Error, Result};
 use serde::ser::SerializeMap;
 use serde::{Deserialize, Serialize};
@@ -625,7 +625,7 @@ impl BooleanQuery {
 }
 
 #[derive(Debug, Clone, PartialEq)]
-#[allow(dead_code)]
+#[cfg(test)]
 pub(crate) struct BooleanMatchPlan {
     pub column: String,
     pub should: Vec<MatchQuery>,
@@ -633,7 +633,7 @@ pub(crate) struct BooleanMatchPlan {
     pub must_not: Vec<MatchQuery>,
 }
 
-#[allow(dead_code)]
+#[cfg(test)]
 impl BooleanMatchPlan {
     pub(crate) fn try_build(query: &FtsQuery) -> Option<Self> {
         match query {
@@ -719,12 +719,19 @@ impl FtsQueryNode for BooleanQuery {
 #[derive(Clone)]
 pub struct Tokens {
     tokens: Vec<String>,
+    positions: Vec<u32>,
     tokens_map: HashMap<String, usize>,
     token_type: DocType,
 }
 
 impl Tokens {
     pub fn new(tokens: Vec<String>, token_type: DocType) -> Self {
+        let positions = (0..tokens.len() as u32).collect();
+        Self::with_positions(tokens, positions, token_type)
+    }
+
+    pub fn with_positions(tokens: Vec<String>, positions: Vec<u32>, token_type: DocType) -> Self {
+        debug_assert_eq!(tokens.len(), positions.len());
         let mut tokens_vec = vec![];
         let mut tokens_map = HashMap::new();
         for (idx, token) in tokens.into_iter().enumerate() {
@@ -734,6 +741,7 @@ impl Tokens {
 
         Self {
             tokens: tokens_vec,
+            positions,
             tokens_map,
             token_type,
         }
@@ -762,6 +770,10 @@ impl Tokens {
     pub fn get_token(&self, index: usize) -> &str {
         &self.tokens[index]
     }
+
+    pub fn position(&self, index: usize) -> u32 {
+        self.positions[index]
+    }
 }
 
 impl IntoIterator for Tokens {
@@ -786,10 +798,12 @@ pub fn collect_query_tokens(text: &str, tokenizer: &mut Box<dyn LanceTokenizer>)
     let token_type = tokenizer.doc_type();
     let mut stream = tokenizer.token_stream_for_search(text);
     let mut tokens = Vec::new();
+    let mut positions = Vec::new();
     while let Some(token) = stream.next() {
         tokens.push(token.text.clone());
+        positions.push(token.position as u32);
     }
-    Tokens::new(tokens, token_type)
+    Tokens::with_positions(tokens, positions, token_type)
 }
 
 pub fn has_query_token(
diff --git a/rust/lance-index/src/scalar/inverted/scorer.rs b/rust/lance-index/src/scalar/inverted/scorer.rs
index f3914707e14..58c0471d262 100644
--- a/rust/lance-index/src/scalar/inverted/scorer.rs
+++ b/rust/lance-index/src/scalar/inverted/scorer.rs
@@ -10,13 +10,6 @@ use std::collections::HashMap;
 pub trait Scorer: Send + Sync {
     fn query_weight(&self, token: &str) -> f32;
     fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32;
-    // calculate the contribution of the token in the document
-    // token: the token to score
-    // freq: the frequency of the token in the document
-    // doc_tokens: the number of tokens in the document
-    fn score(&self, token: &str, freq: u32, doc_tokens: u32) -> f32 {
-        self.query_weight(token) * self.doc_weight(freq, doc_tokens)
-    }
 }
 
 // BM25 parameters
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs
index 0bb3ead446f..fb1e8cc3639 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs
@@ -8,7 +8,7 @@ use std::{env, path::PathBuf};
 #[cfg(feature = "tokenizer-jieba")]
 mod jieba;
 
-pub mod lance_tokenizer;
+pub mod document_tokenizer;
 #[cfg(feature = "tokenizer-lindera")]
 mod lindera;
 
@@ -19,9 +19,15 @@ use jieba::JiebaTokenizerBuilder;
 use lindera::LinderaTokenizerBuilder;
 
 use crate::pbold;
-use crate::scalar::inverted::tokenizer::lance_tokenizer::{
+use crate::scalar::inverted::tokenizer::document_tokenizer::{
     JsonTokenizer, LanceTokenizer, TextTokenizer,
 };
+pub use lance_tokenizer::Language;
+use lance_tokenizer::{
+    AsciiFoldingFilter, LowerCaser, NgramTokenizer, RawTokenizer, RemoveLongFilter,
+    SimpleTokenizer, Stemmer, StopWordFilter, TextAnalyzer, TextAnalyzerBuilder,
+    WhitespaceTokenizer,
+};
 
 /// Tokenizer configs
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
@@ -43,7 +49,7 @@ pub struct InvertedIndexParams {
 
     /// language for stemming and stop words
     /// this is only used when `stem` or `remove_stop_words` is true
-    pub(crate) language: tantivy::tokenizer::Language,
+    pub(crate) language: Language,
 
     /// If true, store the position of the term in the document
     /// This can significantly increase the size of the index
@@ -90,10 +96,28 @@ pub struct InvertedIndexParams {
     #[serde(default)]
     pub(crate) prefix_only: bool,
 
-    /// If true, skip the partition merge stage after indexing.
-    /// This can be useful for distributed indexing where merge is handled separately.
-    #[serde(default)]
-    pub(crate) skip_merge: bool,
+    /// Total memory limit in MiB for the build stage.
+    ///
+    /// This is split evenly across FTS workers at build time. By default Lance
+    /// uses roughly `num_cpus / 2` workers, unless `LANCE_FTS_NUM_SHARDS` is set.
+    /// If unset, each worker defaults to a 2 GiB build-time memory limit.
+    ///
+    /// This is a build-time only parameter and is not persisted with the index.
+    #[serde(
+        rename = "memory_limit",
+        skip_serializing,
+        default,
+        alias = "worker_memory_limit_mb"
+    )]
+    pub(crate) memory_limit_mb: Option<u64>,
+
+    /// Number of workers to use for FTS build.
+    ///
+    /// This is a build-time only parameter and is not persisted with the index.
+    /// By default Lance uses roughly `num_cpus / 2` workers.
+    /// The effective worker count is clamped to `[1, num_cpus - 2]`.
+    #[serde(rename = "num_workers", skip_serializing, default)]
+    pub(crate) num_workers: Option<usize>,
 }
 
 impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails {
@@ -139,7 +163,8 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams {
             min_ngram_length: details.min_ngram_length,
             max_ngram_length: details.max_ngram_length,
             prefix_only: details.prefix_only,
-            skip_merge: defaults.skip_merge,
+            memory_limit_mb: defaults.memory_limit_mb,
+            num_workers: defaults.num_workers,
         })
     }
 }
@@ -158,7 +183,7 @@ fn default_max_ngram_length() -> u32 {
 
 impl Default for InvertedIndexParams {
     fn default() -> Self {
-        Self::new("simple".to_owned(), tantivy::tokenizer::Language::English)
+        Self::new("simple".to_owned(), Language::English)
     }
 }
 
@@ -176,7 +201,7 @@ impl InvertedIndexParams {
     /// The `language` is used for stemming and removing stop words,
     /// this is not used for `lindera/*` and `jieba/*` tokenizers.
     /// Default to `English`.
-    pub fn new(base_tokenizer: String, language: tantivy::tokenizer::Language) -> Self {
+    pub fn new(base_tokenizer: String, language: Language) -> Self {
         Self {
             lance_tokenizer: None,
             base_tokenizer,
@@ -191,7 +216,8 @@ impl InvertedIndexParams {
             min_ngram_length: default_min_ngram_length(),
             max_ngram_length: default_max_ngram_length(),
             prefix_only: false,
-            skip_merge: false,
+            memory_limit_mb: None,
+            num_workers: None,
         }
     }
 
@@ -280,41 +306,66 @@ impl InvertedIndexParams {
         self
     }
 
-    /// Skip merging partitions after indexing.
-    pub fn skip_merge(mut self, skip_merge: bool) -> Self {
-        self.skip_merge = skip_merge;
+    pub fn memory_limit_mb(mut self, memory_limit_mb: u64) -> Self {
+        self.memory_limit_mb = Some(memory_limit_mb);
+        self
+    }
+
+    /// Set the number of workers to use for this build.
+    ///
+    /// By default Lance uses roughly `num_cpus / 2` workers.
+    /// The effective worker count is clamped to `[1, num_cpus - 2]`.
+    pub fn num_workers(mut self, num_workers: usize) -> Self {
+        self.num_workers = Some(num_workers);
         self
     }
 
+    /// Serialize params for the build/training path, including build-only fields.
+    pub fn to_training_json(&self) -> serde_json::Result<serde_json::Value> {
+        let mut value = serde_json::to_value(self)?;
+        let object = value
+            .as_object_mut()
+            .expect("inverted index params should serialize to a JSON object");
+        if let Some(memory_limit_mb) = self.memory_limit_mb {
+            object.insert(
+                "memory_limit".to_string(),
+                serde_json::Value::from(memory_limit_mb),
+            );
+        }
+        if let Some(num_workers) = self.num_workers {
+            object.insert(
+                "num_workers".to_string(),
+                serde_json::Value::from(num_workers),
+            );
+        }
+        Ok(value)
+    }
+
     pub fn build(&self) -> Result<Box<dyn LanceTokenizer>> {
         let mut builder = self.build_base_tokenizer()?;
         if let Some(max_token_length) = self.max_token_length {
-            builder = builder.filter_dynamic(tantivy::tokenizer::RemoveLongFilter::limit(
-                max_token_length,
-            ));
+            builder = builder.filter_dynamic(RemoveLongFilter::limit(max_token_length));
         }
         if self.lower_case {
-            builder = builder.filter_dynamic(tantivy::tokenizer::LowerCaser);
+            builder = builder.filter_dynamic(LowerCaser);
         }
         if self.stem {
-            builder = builder.filter_dynamic(tantivy::tokenizer::Stemmer::new(self.language));
+            builder = builder.filter_dynamic(Stemmer::new(self.language));
         }
         if self.remove_stop_words {
             let stop_word_filter = match &self.custom_stop_words {
-                Some(words) => tantivy::tokenizer::StopWordFilter::remove(words.iter().cloned()),
-                None => {
-                    tantivy::tokenizer::StopWordFilter::new(self.language).ok_or_else(|| {
-                        Error::invalid_input(format!(
-                            "removing stop words for language {:?} is not supported yet",
-                            self.language
-                        ))
-                    })?
-                }
+                Some(words) => StopWordFilter::remove(words.iter().cloned()),
+                None => StopWordFilter::new(self.language).ok_or_else(|| {
+                    Error::invalid_input(format!(
+                        "removing stop words for language {:?} is not supported yet",
+                        self.language
+                    ))
+                })?,
             };
             builder = builder.filter_dynamic(stop_word_filter);
         }
         if self.ascii_folding {
-            builder = builder.filter_dynamic(tantivy::tokenizer::AsciiFoldingFilter);
+            builder = builder.filter_dynamic(AsciiFoldingFilter);
         }
         let tokenizer = builder.build();
 
@@ -329,29 +380,20 @@ impl InvertedIndexParams {
         }
     }
 
-    fn build_base_tokenizer(&self) -> Result<tantivy::tokenizer::TextAnalyzerBuilder> {
+    fn build_base_tokenizer(&self) -> Result<TextAnalyzerBuilder> {
         match self.base_tokenizer.as_str() {
-            "simple" => Ok(tantivy::tokenizer::TextAnalyzer::builder(
-                tantivy::tokenizer::SimpleTokenizer::default(),
-            )
-            .dynamic()),
-            "whitespace" => Ok(tantivy::tokenizer::TextAnalyzer::builder(
-                tantivy::tokenizer::WhitespaceTokenizer::default(),
-            )
-            .dynamic()),
-            "raw" => Ok(tantivy::tokenizer::TextAnalyzer::builder(
-                tantivy::tokenizer::RawTokenizer::default(),
-            )
-            .dynamic()),
-            "ngram" => Ok(tantivy::tokenizer::TextAnalyzer::builder(
-                tantivy::tokenizer::NgramTokenizer::new(
+            "simple" => Ok(TextAnalyzer::builder(SimpleTokenizer::default()).dynamic()),
+            "whitespace" => Ok(TextAnalyzer::builder(WhitespaceTokenizer::default()).dynamic()),
+            "raw" => Ok(TextAnalyzer::builder(RawTokenizer::default()).dynamic()),
+            "ngram" => {
+                let tokenizer = NgramTokenizer::new(
                     self.min_ngram_length as usize,
                     self.max_ngram_length as usize,
                     self.prefix_only,
                 )
-                .map_err(|e| Error::invalid_input(e.to_string()))?,
-            )
-            .dynamic()),
+                .map_err(|e| Error::invalid_input(e.to_string()))?;
+                Ok(TextAnalyzer::builder(tokenizer).dynamic())
+            }
             #[cfg(feature = "tokenizer-lindera")]
             s if s.starts_with("lindera/") => {
                 let Some(home) = language_model_home() else {
@@ -391,3 +433,56 @@ pub fn language_model_home() -> Option<PathBuf> {
         Err(_) => dirs::data_local_dir().map(|p| p.join(LANCE_LANGUAGE_MODEL_DEFAULT_DIRECTORY)),
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::InvertedIndexParams;
+
+    #[test]
+    fn test_build_only_fields_are_not_serialized() {
+        let params = InvertedIndexParams::default()
+            .memory_limit_mb(4096)
+            .num_workers(7);
+        let json = serde_json::to_value(&params).unwrap();
+        assert!(json.get("memory_limit").is_none());
+        assert!(json.get("num_workers").is_none());
+    }
+
+    #[test]
+    fn test_memory_limit_serde_accepts_legacy_worker_field_name() {
+        let mut json = serde_json::to_value(InvertedIndexParams::default()).unwrap();
+        let obj = json.as_object_mut().unwrap();
+        obj.remove("memory_limit");
+        obj.insert(
+            "worker_memory_limit_mb".to_string(),
+            serde_json::Value::from(2048),
+        );
+        let params: InvertedIndexParams = serde_json::from_value(json).unwrap();
+        assert_eq!(params.memory_limit_mb, Some(2048));
+    }
+
+    #[test]
+    fn test_build_only_fields_deserialize_from_public_names() {
+        let mut json = serde_json::to_value(InvertedIndexParams::default()).unwrap();
+        let obj = json.as_object_mut().unwrap();
+        obj.insert("memory_limit".to_string(), serde_json::Value::from(4096));
+        obj.insert("num_workers".to_string(), serde_json::Value::from(3));
+
+        let params: InvertedIndexParams = serde_json::from_value(json).unwrap();
+        assert_eq!(params.memory_limit_mb, Some(4096));
+        assert_eq!(params.num_workers, Some(3));
+    }
+
+    #[test]
+    fn test_training_json_serializes_build_only_fields() {
+        let params = InvertedIndexParams::default()
+            .memory_limit_mb(4096)
+            .num_workers(3);
+        let json = params.to_training_json().unwrap();
+        assert_eq!(
+            json.get("memory_limit"),
+            Some(&serde_json::Value::from(4096))
+        );
+        assert_eq!(json.get("num_workers"), Some(&serde_json::Value::from(3)));
+    }
+}
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer/lance_tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer/document_tokenizer.rs
similarity index 92%
rename from rust/lance-index/src/scalar/inverted/tokenizer/lance_tokenizer.rs
rename to rust/lance-index/src/scalar/inverted/tokenizer/document_tokenizer.rs
index c7a6000c5b7..62dd7b1aa3b 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer/lance_tokenizer.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer/document_tokenizer.rs
@@ -4,8 +4,8 @@
 use arrow_schema::{DataType, Field};
 use lance_arrow::ARROW_EXT_NAME_KEY;
 use lance_arrow::json::JSON_EXT_NAME;
+use lance_tokenizer::{BoxTokenStream, TextAnalyzer, Token, TokenStream};
 use serde_json::Value;
-use tantivy::tokenizer::{BoxTokenStream, Token, TokenStream};
 
 /// Document type for full text search.
 #[derive(Debug, Clone)]
@@ -92,7 +92,7 @@ impl Clone for Box<dyn LanceTokenizer> {
 
 #[derive(Clone)]
 pub struct TextTokenizer {
-    tokenizer: tantivy::tokenizer::TextAnalyzer,
+    tokenizer: TextAnalyzer,
 }
 
 impl std::fmt::Debug for TextTokenizer {
@@ -102,7 +102,7 @@ impl std::fmt::Debug for TextTokenizer {
 }
 
 impl TextTokenizer {
-    pub fn new(tokenizer: tantivy::tokenizer::TextAnalyzer) -> Self {
+    pub fn new(tokenizer: TextAnalyzer) -> Self {
         Self { tokenizer }
     }
 }
@@ -127,11 +127,11 @@ impl LanceTokenizer for TextTokenizer {
 
 #[derive(Clone)]
 pub struct JsonTokenizer {
-    tokenizer: tantivy::tokenizer::TextAnalyzer,
+    tokenizer: TextAnalyzer,
 }
 
 impl JsonTokenizer {
-    pub fn new(tokenizer: tantivy::tokenizer::TextAnalyzer) -> Self {
+    pub fn new(tokenizer: TextAnalyzer) -> Self {
         Self { tokenizer }
     }
 }
@@ -170,10 +170,7 @@ impl LanceTokenizer for JsonTokenizer {
     }
 }
 
-fn flatten_triplet(
-    text: &str,
-    tokenizer: &mut tantivy::tokenizer::TextAnalyzer,
-) -> lance_core::Result<Vec<Token>> {
+fn flatten_triplet(text: &str, tokenizer: &mut TextAnalyzer) -> lance_core::Result<Vec<Token>> {
     let mut token_vec = Vec::new();
     let mut idx = 0;
 
@@ -228,7 +225,7 @@ fn flatten_json(
     prefix: &str,
     out: &mut Vec<Token>,
     position: &mut usize,
-    tokenizer: &mut tantivy::tokenizer::TextAnalyzer,
+    tokenizer: &mut TextAnalyzer,
 ) {
     match value {
         Value::Object(map) => {
@@ -306,11 +303,11 @@ impl TokenStream for TTStream {
 
 #[cfg(test)]
 mod tests {
-    use crate::scalar::inverted::tokenizer::lance_tokenizer::{
+    use crate::scalar::inverted::tokenizer::document_tokenizer::{
         JsonTokenizer, LanceTokenizer, flatten_json, flatten_triplet,
     };
+    use lance_tokenizer::{SimpleTokenizer, TextAnalyzer, Token};
     use serde_json::Value;
-    use tantivy::tokenizer::{SimpleTokenizer, Token};
 
     #[test]
     fn test_json_tokenizer() {
@@ -321,9 +318,8 @@ mod tests {
             {"c": "e"}
           ]
         }"#;
-        let mut tokenizer = JsonTokenizer::new(
-            tantivy::tokenizer::TextAnalyzer::builder(SimpleTokenizer::default()).build(),
-        );
+        let mut tokenizer =
+            JsonTokenizer::new(TextAnalyzer::builder(SimpleTokenizer::default()).build());
         let mut stream = tokenizer.token_stream_for_doc(text);
 
         let mut tokens: Vec<Token> = vec![];
@@ -354,8 +350,7 @@ mod tests {
         let value: Value = serde_json::from_str(json).unwrap();
 
         let mut tokens = vec![];
-        let mut tokenizer =
-            tantivy::tokenizer::TextAnalyzer::builder(SimpleTokenizer::default()).build();
+        let mut tokenizer = TextAnalyzer::builder(SimpleTokenizer::default()).build();
         let mut position = 0;
         flatten_json(&value, "", &mut tokens, &mut position, &mut tokenizer);
 
@@ -372,8 +367,7 @@ mod tests {
     #[test]
     fn test_flatten_triplet() {
         let text = r#"a,number,1;b.c,str,d;b.c,str,e;d,str,hello world;e,number,1.0"#;
-        let mut tokenizer =
-            tantivy::tokenizer::TextAnalyzer::builder(SimpleTokenizer::default()).build();
+        let mut tokenizer = TextAnalyzer::builder(SimpleTokenizer::default()).build();
         let tokens = flatten_triplet(text, &mut tokenizer).unwrap();
 
         assert_eq!(tokens.len(), 6);
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer/jieba.rs b/rust/lance-index/src/scalar/inverted/tokenizer/jieba.rs
index 43af65cd210..2bfd61ec7fc 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer/jieba.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer/jieba.rs
@@ -4,6 +4,7 @@
 use std::{fs::File, io::BufReader, path::Path, path::PathBuf};
 
 use lance_core::{Error, Result};
+use lance_tokenizer::{JiebaTokenizer, TextAnalyzerBuilder};
 use serde::{Deserialize, Serialize, de::DeserializeOwned};
 
 #[derive(Serialize, Deserialize, Default)]
@@ -37,7 +38,7 @@ pub trait JiebaTokenizerBuilder: Sized {
 
     fn new(config: Self::Config, root: &Path) -> Result<Self>;
 
-    fn build(&self) -> Result<tantivy::tokenizer::TextAnalyzerBuilder>;
+    fn build(&self) -> Result<TextAnalyzerBuilder>;
 }
 
 pub struct JiebaBuilder {
@@ -71,7 +72,7 @@ impl JiebaTokenizerBuilder for JiebaBuilder {
         })
     }
 
-    fn build(&self) -> Result<tantivy::tokenizer::TextAnalyzerBuilder> {
+    fn build(&self) -> Result<TextAnalyzerBuilder> {
         let main_dict_path = &self.main_dict_path();
         let file = std::fs::File::open(main_dict_path)?;
         let mut f = std::io::BufReader::new(file);
@@ -93,60 +94,6 @@ impl JiebaTokenizerBuilder for JiebaBuilder {
                 ))
             })?
         }
-        let tokenizer = JiebaTokenizer { jieba };
-        Ok(tantivy::tokenizer::TextAnalyzer::builder(tokenizer).dynamic())
-    }
-}
-
-#[derive(Clone)]
-struct JiebaTokenizer {
-    jieba: jieba_rs::Jieba,
-}
-
-struct JiebaTokenStream {
-    tokens: Vec<tantivy::tokenizer::Token>,
-    index: usize,
-}
-
-impl tantivy::tokenizer::TokenStream for JiebaTokenStream {
-    fn advance(&mut self) -> bool {
-        if self.index < self.tokens.len() {
-            self.index += 1;
-            true
-        } else {
-            false
-        }
-    }
-
-    fn token(&self) -> &tantivy::tokenizer::Token {
-        &self.tokens[self.index - 1]
-    }
-
-    fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {
-        &mut self.tokens[self.index - 1]
-    }
-}
-
-#[cfg(feature = "tokenizer-jieba")]
-impl tantivy::tokenizer::Tokenizer for JiebaTokenizer {
-    type TokenStream<'a> = JiebaTokenStream;
-
-    fn token_stream(&mut self, text: &str) -> JiebaTokenStream {
-        let mut indices = text.char_indices().collect::<Vec<_>>();
-        indices.push((text.len(), '\0'));
-        let orig_tokens = self
-            .jieba
-            .tokenize(text, jieba_rs::TokenizeMode::Search, true);
-        let mut tokens = Vec::new();
-        for token in orig_tokens {
-            tokens.push(tantivy::tokenizer::Token {
-                offset_from: indices[token.start].0,
-                offset_to: indices[token.end].0,
-                position: token.start,
-                text: String::from(&text[(indices[token.start].0)..(indices[token.end].0)]),
-                position_length: token.end - token.start,
-            });
-        }
-        JiebaTokenStream { tokens, index: 0 }
+        Ok(JiebaTokenizer::new(jieba).analyzer_builder())
     }
 }
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer/lindera.rs b/rust/lance-index/src/scalar/inverted/tokenizer/lindera.rs
index c39cab9ad68..e294819dca3 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer/lindera.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer/lindera.rs
@@ -4,7 +4,7 @@
 use std::path::{Path, PathBuf};
 
 use lance_core::{Error, Result};
-use lindera_tantivy::tokenizer::LinderaTokenizer;
+use lance_tokenizer::{LinderaTokenizer, TextAnalyzerBuilder};
 
 pub const LINDERA_LANGUAGE_MODEL_CONFIG_FILE: &str = "config.yml";
 
@@ -22,7 +22,7 @@ pub trait LinderaTokenizerBuilder: Sized {
 
     fn new(config_path: &Path) -> Result<Self>;
 
-    fn build(&self) -> Result<tantivy::tokenizer::TextAnalyzerBuilder>;
+    fn build(&self) -> Result<TextAnalyzerBuilder>;
 }
 
 pub struct LinderaBuilder {
@@ -36,7 +36,7 @@ impl LinderaTokenizerBuilder for LinderaBuilder {
         })
     }
 
-    fn build(&self) -> Result<tantivy::tokenizer::TextAnalyzerBuilder> {
+    fn build(&self) -> Result<TextAnalyzerBuilder> {
         let tokenizer = if self.config_path.exists() {
             match LinderaTokenizer::from_file(&self.config_path) {
                 Ok(tok) => tok,
@@ -56,6 +56,6 @@ impl LinderaTokenizerBuilder for LinderaBuilder {
             LinderaTokenizer::new()
                 .map_err(|e| Error::io(format!("Failed to initialize default tokenizer: {}", e)))?
         };
-        Ok(tantivy::tokenizer::TextAnalyzer::builder(tokenizer).dynamic())
+        Ok(tokenizer.analyzer_builder())
     }
 }
diff --git a/rust/lance-index/src/scalar/inverted/wand.rs b/rust/lance-index/src/scalar/inverted/wand.rs
index f813121fa42..b06c75c0021 100644
--- a/rust/lance-index/src/scalar/inverted/wand.rs
+++ b/rust/lance-index/src/scalar/inverted/wand.rs
@@ -7,9 +7,8 @@ use std::{cell::UnsafeCell, collections::BinaryHeap};
 use std::{cmp::Reverse, fmt::Debug};
 
 use arrow::array::AsArray;
-use arrow::datatypes::{Int32Type, UInt32Type};
-use arrow_array::{Array, UInt32Array};
-use arrow_schema::DataType;
+use arrow::datatypes::Int32Type;
+use arrow_array::Array;
 use itertools::Itertools;
 use lance_core::Result;
 use lance_core::utils::address::RowAddress;
@@ -17,18 +16,22 @@ use lance_core::utils::mask::RowAddrMask;
 
 use crate::metrics::MetricsCollector;
 
+use super::{
+    CompressedPositionStorage,
+    query::Operator,
+    scorer::{K1, idf},
+};
 use super::{
     CompressedPostingList, DocSet, PostingList, RawDocInfo,
     builder::ScoredDoc,
-    encoding::{decompress_positions, decompress_posting_block, decompress_posting_remainder},
+    encoding::{
+        decode_position_stream_block, decompress_positions, decompress_posting_block,
+        decompress_posting_remainder,
+    },
     query::FtsSearchParams,
     scorer::Scorer,
 };
 use super::{DocInfo, builder::BLOCK_SIZE};
-use super::{
-    query::Operator,
-    scorer::{K1, idf},
-};
 
 const TERMINATED_DOC_ID: u64 = u64::MAX;
 
@@ -43,6 +46,7 @@ pub struct PostingIterator {
     token: String,
     token_id: u32,
     position: u32,
+    query_weight: f32,
     list: PostingList,
     // the index of current doc, this can be changed only by `next()`
     index: usize,
@@ -60,6 +64,9 @@ struct CompressedState {
     doc_ids: Vec<u32>,
     freqs: Vec<u32>,
     buffer: Box<[u32; BLOCK_SIZE]>,
+    position_block_idx: Option<usize>,
+    position_values: Vec<u32>,
+    position_offsets: Vec<usize>,
 }
 
 impl CompressedState {
@@ -69,21 +76,40 @@ impl CompressedState {
             doc_ids: Vec::with_capacity(BLOCK_SIZE),
             freqs: Vec::with_capacity(BLOCK_SIZE),
             buffer: Box::new([0; BLOCK_SIZE]),
+            position_block_idx: None,
+            position_values: Vec::new(),
+            position_offsets: Vec::new(),
         }
     }
 
     #[inline]
-    fn decompress(&mut self, block: &[u8], block_idx: usize, num_blocks: usize, length: u32) {
+    fn decompress(
+        &mut self,
+        block: &[u8],
+        block_idx: usize,
+        num_blocks: usize,
+        length: u32,
+        tail_codec: super::PostingTailCodec,
+    ) {
         self.doc_ids.clear();
         self.freqs.clear();
 
         let remainder = length as usize % BLOCK_SIZE;
         if block_idx + 1 == num_blocks && remainder != 0 {
-            decompress_posting_remainder(block, remainder, &mut self.doc_ids, &mut self.freqs);
+            decompress_posting_remainder(
+                block,
+                remainder,
+                tail_codec,
+                &mut self.doc_ids,
+                &mut self.freqs,
+            );
         } else {
             decompress_posting_block(block, &mut self.buffer, &mut self.doc_ids, &mut self.freqs);
         }
         self.block_idx = block_idx;
+        self.position_block_idx = None;
+        self.position_values.clear();
+        self.position_offsets.clear();
     }
 }
 
@@ -157,17 +183,35 @@ impl PostingIterator {
         let compressed = unsafe { &mut *self.compressed_state_ptr() };
         if compressed.block_idx != block_idx || compressed.doc_ids.is_empty() {
             let block = list.blocks.value(block_idx);
-            compressed.decompress(block, block_idx, list.blocks.len(), list.length);
+            compressed.decompress(
+                block,
+                block_idx,
+                list.blocks.len(),
+                list.length,
+                list.posting_tail_codec,
+            );
         }
         compressed as *mut CompressedState
     }
 
+    #[cfg(test)]
     pub(crate) fn new(
         token: String,
         token_id: u32,
         position: u32,
         list: PostingList,
         num_doc: usize,
+    ) -> Self {
+        Self::with_query_weight(token, token_id, position, 1.0, list, num_doc)
+    }
+
+    pub(crate) fn with_query_weight(
+        token: String,
+        token_id: u32,
+        position: u32,
+        query_weight: f32,
+        list: PostingList,
+        num_doc: usize,
     ) -> Self {
         let approximate_upper_bound = match list.max_score() {
             Some(max_score) => max_score,
@@ -180,6 +224,7 @@ impl PostingIterator {
             token,
             token_id,
             position,
+            query_weight,
             list,
             index: 0,
             block_idx: 0,
@@ -203,6 +248,16 @@ impl PostingIterator {
         self.approximate_upper_bound
     }
 
+    #[inline]
+    fn score<S: Scorer>(&self, scorer: &S, freq: u32, doc_length: u32) -> f32 {
+        self.query_weight * scorer.doc_weight(freq, doc_length)
+    }
+
+    #[inline]
+    fn cost(&self) -> usize {
+        self.list.len()
+    }
+
     #[inline]
     fn empty(&self) -> bool {
         self.index >= self.list.len()
@@ -230,14 +285,63 @@ impl PostingIterator {
         }
     }
 
-    fn positions(&self) -> Option<Arc<dyn Array>> {
+    fn position_cursor(&self) -> Option<PositionCursor<'_>> {
         match self.list {
-            PostingList::Plain(ref list) => list.positions(self.index),
-            PostingList::Compressed(ref list) => list.positions.as_ref().map(|p| {
-                let positions = p.value(self.index);
-                let positions = decompress_positions(positions.as_binary());
-                Arc::new(UInt32Array::from(positions)) as Arc<dyn Array>
+            PostingList::Plain(ref list) => list.positions.as_ref().map(|positions| {
+                let start = positions.value_offsets()[self.index] as usize;
+                let end = positions.value_offsets()[self.index + 1] as usize;
+                PositionCursor::new(
+                    PositionValues::Owned(
+                        positions.values().as_primitive::<Int32Type>().values()[start..end]
+                            .iter()
+                            .map(|value| *value as u32)
+                            .collect(),
+                    ),
+                    self.position as i32,
+                )
             }),
+            PostingList::Compressed(ref list) => match list.positions.as_ref()? {
+                CompressedPositionStorage::LegacyPerDoc(positions) => {
+                    let positions = positions.value(self.index);
+                    let positions = decompress_positions(positions.as_binary());
+                    Some(PositionCursor::new(
+                        PositionValues::Owned(positions),
+                        self.position as i32,
+                    ))
+                }
+                CompressedPositionStorage::SharedStream(stream) => {
+                    let block_idx = self.index / BLOCK_SIZE;
+                    let block_offset = self.index % BLOCK_SIZE;
+                    let compressed =
+                        unsafe { &mut *self.ensure_compressed_block_ptr(list, block_idx) };
+                    if compressed.position_block_idx != Some(block_idx) {
+                        decode_position_stream_block(
+                            stream.block(block_idx),
+                            compressed.freqs.as_slice(),
+                            stream.codec(),
+                            &mut compressed.position_values,
+                        )
+                        .expect("shared position stream decoding should succeed");
+                        compressed.position_offsets.clear();
+                        compressed
+                            .position_offsets
+                            .reserve(compressed.freqs.len() + 1);
+                        compressed.position_offsets.push(0);
+                        let mut offset = 0usize;
+                        for &freq in &compressed.freqs {
+                            offset += freq as usize;
+                            compressed.position_offsets.push(offset);
+                        }
+                        compressed.position_block_idx = Some(block_idx);
+                    }
+                    let start = compressed.position_offsets[block_offset];
+                    let end = compressed.position_offsets[block_offset + 1];
+                    Some(PositionCursor::new(
+                        PositionValues::Borrowed(&compressed.position_values[start..end]),
+                        self.position as i32,
+                    ))
+                }
+            },
         }
     }
 
@@ -338,17 +442,138 @@ pub struct DocCandidate {
     pub doc_length: u32,
 }
 
+struct HeadPosting {
+    // Iterators that are already positioned on or after the next candidate doc.
+    // The heap is ordered by smallest doc id so the top element determines
+    // the next target doc to consider.
+    posting: Box<PostingIterator>,
+}
+
+impl HeadPosting {
+    fn new(posting: Box<PostingIterator>) -> Self {
+        Self { posting }
+    }
+
+    fn doc_id(&self) -> u64 {
+        self.posting
+            .doc()
+            .map(|doc| doc.doc_id())
+            .unwrap_or(TERMINATED_DOC_ID)
+    }
+}
+
+impl PartialEq for HeadPosting {
+    fn eq(&self, other: &Self) -> bool {
+        self.doc_id() == other.doc_id()
+            && self.posting.approximate_upper_bound().to_bits()
+                == other.posting.approximate_upper_bound().to_bits()
+            && self.posting.token_id == other.posting.token_id
+            && self.posting.position == other.posting.position
+    }
+}
+
+impl Eq for HeadPosting {}
+
+impl PartialOrd for HeadPosting {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for HeadPosting {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        other
+            .doc_id()
+            .cmp(&self.doc_id())
+            .then_with(|| {
+                self.posting
+                    .approximate_upper_bound()
+                    .total_cmp(&other.posting.approximate_upper_bound())
+            })
+            .then_with(|| self.posting.token_id.cmp(&other.posting.token_id))
+            .then_with(|| self.posting.position.cmp(&other.posting.position))
+    }
+}
+
+struct TailPosting {
+    // Iterators that lag behind the current target doc but may still help the
+    // target beat the threshold if advanced to that doc.
+    upper_bound: f32,
+    // Used as a tie-breaker when upper bounds are equal. Lower-cost iterators
+    // are cheaper to advance, so they are preferred.
+    cost: usize,
+    posting: Box<PostingIterator>,
+}
+
+impl TailPosting {
+    fn new(upper_bound: f32, cost: usize, posting: Box<PostingIterator>) -> Self {
+        Self {
+            upper_bound,
+            cost,
+            posting,
+        }
+    }
+}
+
+impl PartialEq for TailPosting {
+    fn eq(&self, other: &Self) -> bool {
+        self.upper_bound.to_bits() == other.upper_bound.to_bits()
+            && self.cost == other.cost
+            && self.posting.token_id == other.posting.token_id
+            && self.posting.position == other.posting.position
+    }
+}
+
+impl Eq for TailPosting {}
+
+impl PartialOrd for TailPosting {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for TailPosting {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.upper_bound
+            .total_cmp(&other.upper_bound)
+            .then_with(|| other.cost.cmp(&self.cost))
+            .then_with(|| other.posting.token_id.cmp(&self.posting.token_id))
+            .then_with(|| other.posting.position.cmp(&self.posting.position))
+    }
+}
+
 pub struct Wand<'a, S: Scorer> {
     threshold: f32, // multiple of factor and the minimum score of the top-k documents
     operator: Operator,
     num_terms: usize,
-    // we need to sort the posting iterators frequently,
-    // so wrap them in `Box` to avoid the cost of copying
+    // Posting iterators whose current doc id is >= the next target doc.
+    // The heap top gives the smallest current doc id.
+    head: BinaryHeap<HeadPosting>,
     #[allow(clippy::vec_box)]
-    postings: Vec<Box<PostingIterator>>,
+    // Posting iterators that already match the current target doc.
+    // Only these iterators participate in scoring / phrase checks for the
+    // current candidate.
+    lead: Vec<Box<PostingIterator>>,
+    // Posting iterators that are behind the current target doc but still kept
+    // in play because their score upper bound could affect the decision for the
+    // current candidate.
+    tail: BinaryHeap<TailPosting>,
+    // Sum of upper bounds for all iterators currently held in `tail`.
+    // This lets us cheaply decide whether the current candidate can still beat
+    // the threshold before fully advancing every lagging iterator.
+    tail_max_score: f32,
+    // Block-max scores are valid for all candidate docs up to this doc id.
+    // `None` means the window has not been initialized yet and the next
+    // candidate must refresh block-max state before making pruning decisions.
+    up_to: Option<u64>,
+    // For conjunctions, this is the maximum attainable score for the current
+    // block-max window `[target, up_to]`.
+    and_max_score: f32,
+    // Last conjunction doc returned to the caller. The next conjunction search
+    // resumes strictly after this doc, like Lucene's `nextDoc()/advance()`.
+    and_last_doc: Option<u64>,
     docs: &'a DocSet,
     scorer: S,
-    cur_doc: Option<DocInfo>,
 }
 
 // we were using row id as doc id in the past, which is u64,
@@ -361,17 +586,40 @@ impl<'a, S: Scorer> Wand<'a, S> {
         docs: &'a DocSet,
         scorer: S,
     ) -> Self {
-        let mut posting_lists = postings.collect::<Vec<_>>();
-        posting_lists.sort_unstable();
+        let mut head = BinaryHeap::new();
+        let mut lead = Vec::new();
+        for posting in postings {
+            if posting.doc().is_none() {
+                continue;
+            }
+            let posting = Box::new(posting);
+            if operator == Operator::And {
+                lead.push(posting);
+            } else {
+                head.push(HeadPosting::new(posting));
+            }
+        }
+        if operator == Operator::And {
+            lead.sort_unstable_by_key(|posting| posting.cost());
+        }
 
         Self {
             threshold: 0.0,
             operator,
-            num_terms: posting_lists.len(),
-            postings: posting_lists.into_iter().map(Box::new).collect(),
+            num_terms: if operator == Operator::And {
+                lead.len()
+            } else {
+                head.len()
+            },
+            head,
+            lead,
+            tail: BinaryHeap::new(),
+            tail_max_score: 0.0,
+            up_to: None,
+            and_max_score: f32::INFINITY,
+            and_last_doc: None,
             docs,
             scorer,
-            cur_doc: None,
         }
     }
 
@@ -390,8 +638,9 @@ impl<'a, S: Scorer> Wand<'a, S> {
 
         match (mask.max_len(), mask.iter_addrs()) {
             (Some(num_rows_matched), Some(row_ids))
-                if num_rows_matched * 100
-                    <= FLAT_SEARCH_PERCENT_THRESHOLD.deref() * self.docs.len() as u64 =>
+                if self.operator == Operator::Or
+                    && num_rows_matched * 100
+                        <= FLAT_SEARCH_PERCENT_THRESHOLD.deref() * self.docs.len() as u64 =>
             {
                 return self.flat_search(params, row_ids, metrics);
             }
@@ -400,13 +649,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
 
         let mut candidates = BinaryHeap::with_capacity(std::cmp::min(limit, BLOCK_SIZE * 10));
         let mut num_comparisons = 0;
-        while let Some((pivot, doc)) = self.next()? {
-            if let Some(cur_doc) = self.cur_doc
-                && cur_doc.doc_id() >= doc.doc_id()
-            {
-                continue;
-            }
-            self.cur_doc = Some(doc);
+        while let Some((doc, mut score)) = self.next()? {
             num_comparisons += 1;
 
             let row_id = match &doc {
@@ -417,14 +660,9 @@ impl<'a, S: Scorer> Wand<'a, S> {
                 DocInfo::Located(doc) => doc.row_id,
             };
             if !mask.selected(row_id) {
-                self.move_preceding(pivot, doc.doc_id() + 1);
-                continue;
-            }
-
-            if params.phrase_slop.is_some()
-                && !self.check_positions(params.phrase_slop.unwrap() as i32)
-            {
-                self.move_preceding(pivot, doc.doc_id() + 1);
+                if self.operator == Operator::Or {
+                    self.push_back_leads(doc.doc_id() + 1);
+                }
                 continue;
             }
 
@@ -432,8 +670,27 @@ impl<'a, S: Scorer> Wand<'a, S> {
                 DocInfo::Raw(doc) => self.docs.num_tokens(doc.doc_id),
                 DocInfo::Located(doc) => self.docs.num_tokens_by_row_id(doc.row_id),
             };
-            let score = self.score(pivot, doc_length);
-            let freqs = self.iter_term_freqs(pivot).collect();
+
+            let score = if self.operator == Operator::Or {
+                self.advance_all_tail(doc.doc_id(), Some(doc_length), Some(&mut score));
+                if params.phrase_slop.is_some()
+                    && !self.check_positions(params.phrase_slop.unwrap() as i32)
+                {
+                    self.push_back_leads(doc.doc_id() + 1);
+                    continue;
+                }
+                score
+            } else {
+                self.advance_all_tail(doc.doc_id(), None, None);
+                if params.phrase_slop.is_some()
+                    && !self.check_positions(params.phrase_slop.unwrap() as i32)
+                {
+                    continue;
+                }
+                self.score(doc_length)
+            };
+
+            let freqs = self.iter_term_freqs().collect();
             if candidates.len() < limit {
                 candidates.push(Reverse((ScoredDoc::new(row_id, score), freqs, doc_length)));
                 if candidates.len() == limit {
@@ -444,7 +701,9 @@ impl<'a, S: Scorer> Wand<'a, S> {
                 candidates.push(Reverse((ScoredDoc::new(row_id, score), freqs, doc_length)));
                 self.threshold = candidates.peek().unwrap().0.0.score.0 * params.wand_factor;
             }
-            self.move_preceding(pivot, doc.doc_id() + 1);
+            if self.operator == Operator::Or {
+                self.push_back_leads(doc.doc_id() + 1);
+            }
         }
         metrics.record_comparisons(num_comparisons);
 
@@ -478,76 +737,45 @@ impl<'a, S: Scorer> Wand<'a, S> {
             })
             .sorted_unstable()
             .collect::<Vec<_>>();
-        let is_compressed = matches!(self.postings[0].list, PostingList::Compressed(_));
+        let is_compressed = self
+            .head
+            .peek()
+            .map(|posting| matches!(posting.posting.list, PostingList::Compressed(_)))
+            .or_else(|| {
+                self.lead
+                    .first()
+                    .map(|posting| matches!(posting.list, PostingList::Compressed(_)))
+            })
+            .unwrap_or(false);
 
         let mut num_comparisons = 0;
         let mut candidates = BinaryHeap::new();
-        let mut current_doc = 0;
         for (doc_id, row_id) in doc_ids {
             num_comparisons += 1;
+            self.move_head_before_target_to_tail(doc_id);
+            self.move_head_doc_to_lead(doc_id);
 
-            if doc_id < current_doc {
-                continue;
-            }
-            current_doc = doc_id;
-
-            // even we already know the candidate doc id, we still need to know how many terms are required to hit the threshold
-            let mut pivot = 0;
-            let mut approximate_upper_bound = self.postings[0].approximate_upper_bound();
-            while pivot + 1 < self.postings.len() && approximate_upper_bound < self.threshold {
-                approximate_upper_bound += self.postings[pivot + 1].approximate_upper_bound();
-                pivot += 1;
-            }
-
-            if let Some(least_id) = self.postings[0].block_first_doc()
-                && least_id > doc_id
-            {
-                current_doc = least_id;
+            if self.lead.is_empty() && self.tail.is_empty() {
                 continue;
             }
-            let mut max_pivot = 0;
-            while max_pivot + 1 < self.postings.len() {
-                self.postings[max_pivot + 1].shallow_next(doc_id);
-                match self.postings[max_pivot + 1].block_first_doc() {
-                    Some(block_doc_id) if block_doc_id <= doc_id => {
-                        max_pivot += 1;
-                    }
-                    _ => break,
-                }
-            }
 
-            if !self.check_block_max(max_pivot, doc_id) {
-                // the current block max score is less than the threshold,
-                // which means we have to skip at least the current block
-                let (_, least_id) = self.get_new_candidate(max_pivot);
-                if least_id == TERMINATED_DOC_ID {
-                    break;
-                }
-                current_doc = std::cmp::max(doc_id, least_id);
-                self.move_preceding(max_pivot, least_id);
+            if !self.can_target_beat_threshold(doc_id) {
+                self.advance_tail_and_lead_to_head(doc_id + 1);
                 continue;
             }
 
-            // move all postings to this doc id
-            if !self.check_pivot_aligned(pivot, doc_id) {
-                if self.postings.is_empty() {
-                    break;
-                } else {
-                    continue;
-                }
-            }
+            self.collect_tail_matches(doc_id);
 
-            max_pivot = 0;
-            while max_pivot + 1 < self.postings.len()
-                && self.postings[max_pivot + 1].doc().map(|d| d.doc_id()) == Some(doc_id)
-            {
-                max_pivot += 1;
+            if self.operator == Operator::And && self.lead.len() < self.num_terms {
+                self.advance_lead_to_head(doc_id + 1);
+                continue;
             }
 
             // check positions
             if params.phrase_slop.is_some()
                 && !self.check_positions(params.phrase_slop.unwrap() as i32)
             {
+                self.advance_lead_to_head(doc_id + 1);
                 continue;
             }
 
@@ -556,9 +784,18 @@ impl<'a, S: Scorer> Wand<'a, S> {
                 true => self.docs.num_tokens(doc_id as u32),
                 false => self.docs.num_tokens_by_row_id(row_id),
             };
+            if self.operator == Operator::Or && !self.refine_or_candidate(doc_id, doc_length) {
+                // `flat_search` evaluates an explicit allow-list of doc ids. Unlike the
+                // regular WAND path, skipping to the next block boundary is unsafe here
+                // because later doc ids from the same block may still be present in the
+                // allow-list and need to be evaluated individually.
+                self.advance_tail_and_lead_to_head(doc_id + 1);
+                continue;
+            }
 
-            let score = self.score(max_pivot, doc_length);
-            let freqs = self.iter_term_freqs(max_pivot).collect();
+            self.collect_tail_matches(doc_id);
+            let score = self.score(doc_length);
+            let freqs = self.iter_term_freqs().collect();
 
             if candidates.len() < limit {
                 candidates.push(Reverse((ScoredDoc::new(row_id, score), freqs, doc_length)));
@@ -570,6 +807,8 @@ impl<'a, S: Scorer> Wand<'a, S> {
                 candidates.push(Reverse((ScoredDoc::new(row_id, score), freqs, doc_length)));
                 self.threshold = candidates.peek().unwrap().0.0.score.0 * params.wand_factor;
             }
+
+            self.advance_lead_to_head(doc_id + 1);
         }
         metrics.record_comparisons(num_comparisons);
 
@@ -584,26 +823,19 @@ impl<'a, S: Scorer> Wand<'a, S> {
     }
 
     // calculate the score of the current document
-    fn score(&self, pivot: usize, doc_length: u32) -> f32 {
+    fn score(&self, doc_length: u32) -> f32 {
         let mut score = 0.0;
-        for (token, freq) in self.iter_token_freqs(pivot) {
-            score += self.scorer.score(token, freq, doc_length);
+        for posting in &self.lead {
+            if let Some(doc) = posting.doc() {
+                score += posting.score(&self.scorer, doc.frequency(), doc_length);
+            }
         }
         score
     }
 
-    // iterate over all the preceding terms and collect the token and frequency
-    fn iter_token_freqs(&self, pivot: usize) -> impl Iterator<Item = (&str, u32)> + '_ {
-        self.postings[..=pivot].iter().filter_map(|posting| {
-            posting
-                .doc()
-                .map(|doc| (posting.token.as_str(), doc.frequency()))
-        })
-    }
-
     // iterate over all the preceding terms and collect the term index and frequency
-    fn iter_term_freqs(&self, pivot: usize) -> impl Iterator<Item = (u32, u32)> + '_ {
-        self.postings[..=pivot].iter().filter_map(|posting| {
+    fn iter_term_freqs(&self) -> impl Iterator<Item = (u32, u32)> + '_ {
+        self.lead.iter().filter_map(|posting| {
             posting
                 .doc()
                 .map(|doc| (posting.term_index(), doc.frequency()))
@@ -611,281 +843,650 @@ impl<'a, S: Scorer> Wand<'a, S> {
     }
 
     // find the next doc candidate
-    fn next(&mut self) -> Result<Option<(usize, DocInfo)>> {
-        while let Some((pivot, max_pivot)) = self.find_pivot_term() {
-            let posting = &self.postings[pivot];
-            let doc = posting.doc().unwrap();
-            let doc_id = doc.doc_id();
-
-            if !self.check_block_max(max_pivot, doc_id) {
-                // the current block max score is less than the threshold,
-                // which means we have to skip at least the current block
-                let (picked_term, least_id) = self.get_new_candidate(max_pivot);
-                if least_id == TERMINATED_DOC_ID {
-                    return Ok(None);
-                }
-                self.move_term(picked_term, least_id);
+    // Find the next term-level candidate doc. The returned score is the exact
+    // contribution from the current `lead` set; additional score can still come
+    // from `tail` iterators that are advanced to the same doc later.
+    fn next(&mut self) -> Result<Option<(DocInfo, f32)>> {
+        if self.operator == Operator::And {
+            return Ok(self.next_and_candidate().map(|doc| (doc, 0.0)));
+        }
+
+        while let Some(target) = self.head_doc() {
+            if self.up_to.is_none_or(|up_to| target > up_to) {
+                self.update_max_scores(target);
+            }
+            self.move_head_doc_to_lead(target);
+            if self.lead.is_empty() {
                 continue;
             }
 
-            if !self.check_pivot_aligned(pivot, doc_id) {
+            let Some(doc) = self.lead.first().and_then(|posting| posting.doc()) else {
+                self.push_back_leads(target + 1);
                 continue;
+            };
+            let doc_length = match &doc {
+                DocInfo::Raw(doc) => self.docs.num_tokens(doc.doc_id),
+                DocInfo::Located(doc) => self.docs.num_tokens_by_row_id(doc.row_id),
+            };
+            let mut lead_score = self
+                .lead
+                .iter()
+                .filter_map(|posting| {
+                    posting.doc().map(|lead_doc| {
+                        posting.score(&self.scorer, lead_doc.frequency(), doc_length)
+                    })
+                })
+                .sum::<f32>();
+
+            while lead_score <= self.threshold {
+                if lead_score + self.tail_max_score <= self.threshold {
+                    self.push_back_leads(doc.doc_id() + 1);
+                    break;
+                }
+                if !self.advance_tail_top(target, doc_length, &mut lead_score) {
+                    self.push_back_leads(doc.doc_id() + 1);
+                    break;
+                }
             }
 
-            // all the posting iterators preceding pivot have reached this doc id,
-            // this means the sum of upper bound of all terms is not less than the threshold,
-            // this document is a candidate, but we still need to check filters, positions, etc.
-            return Ok(Some((max_pivot, doc)));
+            if !self.lead.is_empty() {
+                return Ok(self
+                    .lead
+                    .first()
+                    .and_then(|posting| posting.doc())
+                    .map(|doc| (doc, lead_score)));
+            }
         }
+
         Ok(None)
     }
 
-    fn check_block_max(&mut self, pivot: usize, pivot_doc: u64) -> bool {
-        let mut sum = 0.0;
-        for posting in self.postings[..=pivot].iter_mut() {
-            posting.shallow_next(pivot_doc);
-            sum += posting.block_max_score();
-        }
-        sum > self.threshold
-    }
-
-    // find the term and new doc_id to move / move to,
-    // the term should be the one with the maximum score,
-    // the new doc_id should be the one is the minimum among:
-    // 1. for the terms preceding the pivot, the next block first doc id
-    // 2. for the terms after the pivot, the doc id of the term
-    fn get_new_candidate(&self, pivot: usize) -> (usize, u64) {
-        let mut picked_term = pivot;
-        let mut max_score = self.postings[pivot].approximate_upper_bound();
-        let mut least_id = self.postings[pivot]
-            .next_block_first_doc()
-            .unwrap_or(TERMINATED_DOC_ID);
-        for (i, posting) in self.postings[..pivot].iter().enumerate().rev() {
-            let next_block_first_doc = posting.next_block_first_doc().unwrap_or(TERMINATED_DOC_ID);
-            if next_block_first_doc < least_id {
-                least_id = next_block_first_doc;
-            }
-            if posting.approximate_upper_bound() > max_score {
-                max_score = posting.approximate_upper_bound();
-                picked_term = i;
+    fn next_and_candidate(&mut self) -> Option<DocInfo> {
+        if self.lead.len() < self.num_terms {
+            return None;
+        }
+        if let Some(last_doc) = self.and_last_doc
+            && self
+                .lead
+                .first()
+                .and_then(|posting| posting.doc())
+                .map(|doc| doc.doc_id())
+                == Some(last_doc)
+        {
+            let next_target = self.and_advance_target(last_doc + 1);
+            if next_target == TERMINATED_DOC_ID {
+                return None;
             }
+            self.lead[0].next(next_target);
         }
 
-        for posting in self.postings[pivot + 1..].iter() {
-            let doc = posting
-                .doc()
-                .map(|d| d.doc_id())
-                .unwrap_or(TERMINATED_DOC_ID);
-            if doc < least_id {
-                least_id = doc;
+        'advance_head: loop {
+            let doc = self
+                .lead
+                .first()
+                .and_then(|posting| posting.doc())?
+                .doc_id();
+            if self.up_to.is_none_or(|up_to| doc > up_to) {
+                let next_target = self.and_advance_target(doc);
+                if next_target == TERMINATED_DOC_ID {
+                    return None;
+                }
+                if next_target != doc {
+                    self.lead[0].next(next_target);
+                    continue;
+                }
+            }
+
+            for posting in self.lead.iter_mut().skip(1) {
+                if posting.doc()?.doc_id() < doc {
+                    posting.next(doc);
+                }
+                let next = posting.doc()?.doc_id();
+                if next > doc {
+                    let next_target = self.and_advance_target(next);
+                    if next_target == TERMINATED_DOC_ID {
+                        return None;
+                    }
+                    self.lead[0].next(next_target);
+                    continue 'advance_head;
+                }
             }
+
+            self.and_last_doc = Some(doc);
+            return self.lead.first().and_then(|posting| posting.doc());
+        }
+    }
+
+    fn and_move_to_next_block(&mut self, target: u64) {
+        if self.threshold <= 0.0 {
+            self.up_to = Some(target);
+            self.and_max_score = f32::INFINITY;
+            return;
         }
 
-        (picked_term, least_id)
+        let mut up_to = TERMINATED_DOC_ID;
+        let mut max_score = 0.0;
+        for posting in &mut self.lead {
+            posting.shallow_next(target);
+            let block_end = posting
+                .next_block_first_doc()
+                .map(|doc| doc.saturating_sub(1))
+                .unwrap_or(TERMINATED_DOC_ID);
+            up_to = up_to.min(block_end.max(target));
+            max_score += posting.block_max_score();
+        }
+        self.up_to = Some(up_to);
+        self.and_max_score = max_score;
     }
 
-    // find the first term that the sum of upper bound of all preceding terms and itself,
-    // are greater than or equal to the threshold.
-    // returns the least pivot and the max index of the terms that have the same doc id.
-    fn find_pivot_term(&self) -> Option<(usize, usize)> {
-        if self.operator == Operator::And {
-            // for AND query, we always require all terms to be present in the document,
-            // so the pivot is always the last term as long as no posting list is exhausted
-            if self.postings.len() == self.num_terms {
-                return Some((self.num_terms - 1, self.num_terms - 1));
-            }
-            return None;
+    fn and_advance_target(&mut self, mut target: u64) -> u64 {
+        if self.up_to.is_none_or(|up_to| target > up_to) {
+            self.and_move_to_next_block(target);
         }
 
-        let mut acc = 0.0;
-        let mut pivot = None;
-        for (idx, posting) in self.postings.iter().enumerate() {
-            acc += posting.approximate_upper_bound();
-            if acc >= self.threshold {
-                pivot = Some(idx);
-                break;
+        loop {
+            let Some(up_to) = self.up_to else {
+                return TERMINATED_DOC_ID;
+            };
+            if self.and_max_score >= self.threshold {
+                return target;
             }
+            if up_to == TERMINATED_DOC_ID {
+                return TERMINATED_DOC_ID;
+            }
+            target = up_to + 1;
+            self.and_move_to_next_block(target);
         }
-        let pivot = pivot?;
-        let mut max_pivot = pivot;
-        let doc_id = self.postings[pivot].doc().unwrap().doc_id();
-        while max_pivot + 1 < self.postings.len()
-            && self.postings[max_pivot + 1].doc().unwrap().doc_id() == doc_id
-        {
-            max_pivot += 1;
-        }
+    }
 
-        Some((pivot, max_pivot))
+    #[allow(clippy::vec_box)]
+    fn head_doc(&self) -> Option<u64> {
+        self.head.peek().map(HeadPosting::doc_id)
     }
 
-    // pick the term that has the maximum upper bound and the current doc id is less than the given doc id
-    // so that we can move the posting iterator to the next doc id that is possible to be candidate
-    fn move_term(&mut self, picked_term: usize, least_id: u64) {
-        self.postings[picked_term].next(least_id);
-        let doc_id = self.postings[picked_term]
-            .doc()
-            .map(|d| d.doc_id())
-            .unwrap_or(TERMINATED_DOC_ID);
-        if doc_id == TERMINATED_DOC_ID {
-            self.postings.swap_remove(picked_term);
+    fn push_head(&mut self, posting: Box<PostingIterator>) {
+        if posting.doc().is_some() {
+            self.head.push(HeadPosting::new(posting));
         }
-        self.bubble_up(picked_term);
     }
 
-    fn check_pivot_aligned(&mut self, pivot: usize, pivot_doc: u64) -> bool {
-        for i in (0..=pivot).rev() {
-            self.postings[i].next(pivot_doc);
-            let doc_id = self.postings[i]
-                .doc()
-                .map(|d| d.doc_id())
-                .unwrap_or(TERMINATED_DOC_ID);
-            if doc_id != pivot_doc {
-                if doc_id == TERMINATED_DOC_ID {
-                    self.postings.swap_remove(i);
+    fn move_head_doc_to_lead(&mut self, target: u64) {
+        while self.head_doc() == Some(target) {
+            if let Some(posting) = self.head.pop() {
+                self.lead.push(posting.posting);
+            }
+        }
+    }
+
+    // Move all head iterators that are already known to be behind `target`
+    // into `tail`, possibly overflowing low-value entries back into `head`.
+    fn move_head_before_target_to_tail(&mut self, target: u64) {
+        while matches!(self.head_doc(), Some(doc_id) if doc_id < target) {
+            if let Some(posting) = self.head.pop() {
+                let upper_bound = posting.posting.approximate_upper_bound();
+                if let Some(mut evicted) =
+                    self.insert_tail_with_overflow(posting.posting, upper_bound)
+                {
+                    evicted.next(target);
+                    self.push_head(evicted);
                 }
-                self.bubble_up(i);
-                return false;
-            } else {
-                self.bubble_up(i);
             }
         }
-        true
     }
 
-    fn move_preceding(&mut self, pivot: usize, least_id: u64) {
-        for i in 0..=pivot {
-            self.postings[i].next(least_id);
+    fn can_target_beat_threshold(&mut self, target: u64) -> bool {
+        if self.up_to.is_none_or(|up_to| target > up_to) {
+            self.update_max_scores(target);
         }
 
-        let mut i = 0;
-        while i < self.postings.len() {
-            if self.postings[i].doc().is_none() {
-                self.postings.swap_remove(i);
-            } else {
-                i += 1;
+        let mut sum = self
+            .lead
+            .iter()
+            .map(|posting| posting.block_max_score())
+            .sum::<f32>();
+        let mut possible_matches = self.lead.len();
+        for posting in &self.tail {
+            if matches!(posting.posting.block_first_doc(), Some(block_doc) if block_doc <= target) {
+                sum += posting.posting.block_max_score();
+                possible_matches += 1;
             }
         }
-        self.postings.sort_unstable();
-    }
 
-    fn bubble_up(&mut self, index: usize) {
-        if index >= self.postings.len() {
-            return;
+        match self.operator {
+            Operator::And => possible_matches >= self.num_terms && sum > self.threshold,
+            Operator::Or => sum > self.threshold,
         }
+    }
 
-        for i in index + 1..self.postings.len() {
-            if self.postings[i].cmp(&self.postings[i - 1]) >= std::cmp::Ordering::Equal {
-                break;
+    fn update_max_scores(&mut self, target: u64) {
+        // Refresh the block-max window for the current target. The resulting
+        // `up_to` is the furthest doc id for which this block-max view remains
+        // valid.
+        let lead_cost = self
+            .lead
+            .iter()
+            .map(|posting| posting.cost())
+            .min()
+            .unwrap_or(usize::MAX);
+        let mut up_to = TERMINATED_DOC_ID;
+        for posting in &mut self.lead {
+            posting.shallow_next(target);
+            let block_end = posting
+                .next_block_first_doc()
+                .map(|doc| doc.saturating_sub(1))
+                .unwrap_or(TERMINATED_DOC_ID);
+            up_to = up_to.min(block_end);
+        }
+        let head = std::mem::take(&mut self.head);
+        let mut rebuilt_head = BinaryHeap::with_capacity(head.len());
+        for mut posting in head.into_vec() {
+            if posting.posting.cost() <= lead_cost {
+                posting.posting.shallow_next(posting.doc_id());
+                let block_end = posting
+                    .posting
+                    .next_block_first_doc()
+                    .map(|doc| doc.saturating_sub(1))
+                    .unwrap_or(TERMINATED_DOC_ID);
+                up_to = up_to.min(block_end);
+            }
+            rebuilt_head.push(posting);
+        }
+        self.head = rebuilt_head;
+        if up_to == TERMINATED_DOC_ID
+            && let Some(top) = self.tail.peek()
+            && top.cost <= lead_cost
+        {
+            let block_end = top
+                .posting
+                .next_block_first_doc()
+                .map(|doc| doc.saturating_sub(1))
+                .unwrap_or(TERMINATED_DOC_ID);
+            up_to = up_to.min(block_end.max(target));
+        }
+        self.up_to = Some(up_to);
+
+        let tail = std::mem::take(&mut self.tail);
+        self.tail_max_score = 0.0;
+        for mut tail_posting in tail.into_vec() {
+            tail_posting.posting.shallow_next(target);
+            let upper_bound = match tail_posting.posting.block_first_doc() {
+                Some(block_doc) if block_doc <= target => tail_posting.posting.block_max_score(),
+                _ => 0.0,
+            };
+            if let Some(mut evicted) =
+                self.insert_tail_with_overflow(tail_posting.posting, upper_bound)
+            {
+                evicted.next(target);
+                self.push_head(evicted);
             }
-            self.postings.swap(i - 1, i);
         }
     }
 
-    fn check_positions(&self, slop: i32) -> bool {
-        let mut position_iters = self
-            .postings
+    fn refine_or_candidate(&mut self, target: u64, doc_length: u32) -> bool {
+        if self.threshold <= 0.0 {
+            return true;
+        }
+
+        let mut lead_score = self
+            .lead
             .iter()
-            .map(|posting| {
-                PositionIterator::new(
-                    posting.positions().expect("positions must exist"),
-                    posting.position as i32,
-                )
+            .filter_map(|posting| {
+                posting
+                    .doc()
+                    .map(|doc| posting.score(&self.scorer, doc.frequency(), doc_length))
             })
-            .collect::<Vec<_>>();
-        position_iters.sort_unstable_by_key(|iter| iter.position_in_query);
-
-        loop {
-            let mut max_relative_pos = None;
-            let mut all_same = true;
-            for window in position_iters.windows(2) {
-                let last = window[0].relative_position();
-                let next = window[1].relative_position();
-                let (Some(last), Some(next)) = (last, next) else {
-                    return false;
-                };
+            .sum::<f32>();
 
-                let move_to = if last > next {
-                    last
-                } else {
-                    std::cmp::max(last + 1, next - slop)
-                };
-                max_relative_pos = max_relative_pos.max(Some(move_to));
-                if !(last <= next && next <= last + slop) {
-                    all_same = false;
-                    break;
-                }
+        while lead_score <= self.threshold {
+            if lead_score + self.tail_max_score <= self.threshold {
+                return false;
             }
-
-            if all_same {
-                return true;
+            if !self.advance_tail_top(target, doc_length, &mut lead_score) {
+                return false;
             }
-
-            position_iters.iter_mut().for_each(|iter| {
-                iter.next(max_relative_pos.unwrap());
-            });
         }
+
+        true
     }
-}
 
-#[derive(Debug)]
-struct PositionIterator {
-    // It's Int32Array for legacy index,
-    // UInt32Array for new index
-    positions: Arc<dyn Array>,
-    pub position_in_query: i32,
-    index: usize,
-}
+    fn collect_tail_matches(&mut self, target: u64) {
+        let mut remaining = Vec::with_capacity(self.tail.len());
+        let tail = std::mem::take(&mut self.tail);
+        self.tail_max_score = 0.0;
+        for tail_posting in tail.into_vec() {
+            let mut posting = tail_posting.posting;
+            posting.next(target);
+            match posting.doc().map(|doc| doc.doc_id()) {
+                Some(doc_id) if doc_id == target => self.lead.push(posting),
+                Some(_) => remaining.push(posting),
+                None => {}
+            }
+        }
 
-impl PositionIterator {
-    fn new(positions: Arc<dyn Array>, position_in_query: i32) -> Self {
-        let mut iter = Self {
-            positions,
-            position_in_query,
-            index: 0,
-        };
-        iter.next(0);
-        iter
+        for posting in remaining {
+            self.push_head(posting);
+        }
     }
 
-    // get the current relative position
-    fn relative_position(&self) -> Option<i32> {
-        if self.index < self.positions.len() {
-            match self.positions.data_type() {
-                DataType::Int32 => Some(
-                    self.positions.as_primitive::<Int32Type>().value(self.index)
-                        - self.position_in_query,
-                ),
-                DataType::UInt32 => Some(
-                    self.positions
-                        .as_primitive::<UInt32Type>()
-                        .value(self.index) as i32
-                        - self.position_in_query,
-                ),
-                _ => {
-                    unreachable!("position iterator only supports Int32 and UInt32");
+    fn advance_tail_and_lead_to_head(&mut self, least_id: u64) {
+        let mut postings = Vec::with_capacity(self.tail.len() + self.lead.len());
+        while let Some(tail) = self.tail.pop() {
+            postings.push(tail.posting);
+        }
+        self.tail_max_score = 0.0;
+        postings.append(&mut self.lead);
+        for mut posting in postings {
+            posting.next(least_id);
+            self.push_head(posting);
+        }
+    }
+
+    fn advance_lead_to_head(&mut self, least_id: u64) {
+        let lead = std::mem::take(&mut self.lead);
+        for mut posting in lead {
+            posting.next(least_id);
+            self.push_head(posting);
+        }
+        // In the flat-search path this is only called after `collect_tail_matches`,
+        // which drains the current tail into either `lead` or `head`. At this
+        // point `tail` is expected to be empty, so clearing it is a no-op that
+        // just resets the cached `tail_max_score`.
+        debug_assert!(self.tail.is_empty());
+        self.clear_tail();
+    }
+
+    fn clear_tail(&mut self) {
+        self.tail.clear();
+        self.tail_max_score = 0.0;
+    }
+
+    fn insert_tail(&mut self, posting: Box<PostingIterator>, upper_bound: f32) {
+        self.tail_max_score += upper_bound;
+        self.tail
+            .push(TailPosting::new(upper_bound, posting.cost(), posting));
+    }
+
+    fn insert_tail_with_overflow(
+        &mut self,
+        posting: Box<PostingIterator>,
+        upper_bound: f32,
+    ) -> Option<Box<PostingIterator>> {
+        // Keep only the lagging iterators that are most useful for deciding the
+        // current candidate. If a stronger tail entry arrives, evict the weakest
+        // one back to the caller so it can be advanced into `head`.
+        if self.threshold <= 0.0 || upper_bound <= 0.0 {
+            return Some(posting);
+        }
+
+        if self.tail_max_score + upper_bound < self.threshold {
+            self.insert_tail(posting, upper_bound);
+            return None;
+        }
+
+        if self.tail.is_empty() {
+            return Some(posting);
+        }
+
+        let candidate = TailPosting::new(upper_bound, posting.cost(), posting);
+        if let Some(top) = self.tail.peek()
+            && top > &candidate
+        {
+            let evicted = self.tail.pop().expect("peeked tail posting should exist");
+            self.tail_max_score = self.tail_max_score - evicted.upper_bound + upper_bound;
+            self.tail.push(candidate);
+            return Some(evicted.posting);
+        }
+
+        Some(candidate.posting)
+    }
+
+    fn push_back_leads(&mut self, target: u64) {
+        // After finishing a candidate doc, convert the aligned iterators back
+        // into lagging iterators. Entries that do not stay in `tail` are
+        // advanced to `target` and returned to `head`.
+        let leads = std::mem::take(&mut self.lead);
+        for posting in leads {
+            let upper_bound = posting.approximate_upper_bound();
+            if let Some(mut evicted) = self.insert_tail_with_overflow(posting, upper_bound) {
+                evicted.next(target);
+                self.push_head(evicted);
+            }
+        }
+    }
+
+    fn advance_tail_top(&mut self, target: u64, doc_length: u32, lead_score: &mut f32) -> bool {
+        // Advance the most promising lagging iterator to the current target.
+        // If it lands on the target, fold its exact contribution into
+        // `lead_score`; otherwise put it back into `head`.
+        let Some(TailPosting {
+            upper_bound,
+            cost: _,
+            mut posting,
+        }) = self.tail.pop()
+        else {
+            return false;
+        };
+        self.tail_max_score -= upper_bound;
+        posting.next(target);
+        match posting.doc().map(|doc| doc.doc_id()) {
+            Some(doc_id) if doc_id == target => {
+                let frequency = posting.doc().expect("posting must exist").frequency();
+                *lead_score += posting.score(&self.scorer, frequency, doc_length);
+                self.lead.push(posting);
+            }
+            Some(_) => self.push_head(posting),
+            None => {}
+        }
+        true
+    }
+
+    fn advance_all_tail(
+        &mut self,
+        target: u64,
+        doc_length: Option<u32>,
+        mut score: Option<&mut f32>,
+    ) {
+        // Materialize all remaining lagging iterators for `target`. This is
+        // only done once we have already decided to fully score / validate the
+        // candidate.
+        let tail = std::mem::take(&mut self.tail);
+        self.tail_max_score = 0.0;
+        for tail_posting in tail.into_vec() {
+            let mut posting = tail_posting.posting;
+            posting.next(target);
+            match posting.doc().map(|doc| doc.doc_id()) {
+                Some(doc_id) if doc_id == target => {
+                    if let (Some(doc_length), Some(score)) = (doc_length, score.as_deref_mut()) {
+                        let frequency = posting
+                            .doc()
+                            .expect("posting moved to target should have doc")
+                            .frequency();
+                        *score += posting.score(&self.scorer, frequency, doc_length);
+                    }
+                    self.lead.push(posting)
                 }
+                Some(_) => self.push_head(posting),
+                None => {}
             }
-        } else {
-            None
         }
     }
 
-    // move to the next position that the relative position is greater than or equal to least_pos
-    fn next(&mut self, least_relative_pos: i32) {
-        let least_pos = least_relative_pos + self.position_in_query;
-        self.index = match self.positions.data_type() {
-            DataType::Int32 => self
-                .positions
-                .as_primitive::<Int32Type>()
-                .values()
-                .partition_point(|&pos| pos < least_pos),
-            DataType::UInt32 => self
-                .positions
-                .as_primitive::<UInt32Type>()
-                .values()
-                .partition_point(|&pos| (pos as i32) < least_pos),
-            _ => unreachable!("position iterator only supports Int32 and UInt32"),
+    fn current_doc_postings(&self) -> Vec<&PostingIterator> {
+        if !self.lead.is_empty() {
+            return self.lead.iter().map(|posting| posting.as_ref()).collect();
+        }
+
+        let Some(target) = self.head_doc() else {
+            return Vec::new();
         };
+        self.head
+            .iter()
+            .filter(|posting| posting.doc_id() == target)
+            .map(|posting| posting.posting.as_ref())
+            .collect()
+    }
+
+    fn check_positions(&self, slop: i32) -> bool {
+        if slop == 0 {
+            return self.check_exact_positions();
+        }
+
+        let mut position_iters = self
+            .current_doc_postings()
+            .into_iter()
+            .map(|posting| posting.position_cursor().expect("positions must exist"))
+            .collect::<Vec<_>>();
+        position_iters.sort_unstable_by_key(|iter| iter.position_in_query);
+
+        loop {
+            let mut max_relative_pos = None;
+            let mut all_same = true;
+            for window in position_iters.windows(2) {
+                let last = window[0].relative_position();
+                let next = window[1].relative_position();
+                let (Some(last), Some(next)) = (last, next) else {
+                    return false;
+                };
+
+                let move_to = if last > next {
+                    last
+                } else {
+                    std::cmp::max(last + 1, next - slop)
+                };
+                max_relative_pos = max_relative_pos.max(Some(move_to));
+                if !(last <= next && next <= last + slop) {
+                    all_same = false;
+                    break;
+                }
+            }
+
+            if all_same {
+                return true;
+            }
+
+            position_iters.iter_mut().for_each(|iter| {
+                iter.advance_to_relative(max_relative_pos.unwrap());
+            });
+        }
+    }
+
+    fn check_exact_positions(&self) -> bool {
+        let mut position_iters = self
+            .current_doc_postings()
+            .into_iter()
+            .map(|posting| posting.position_cursor().expect("positions must exist"))
+            .collect::<Vec<_>>();
+        position_iters.sort_unstable_by_key(|iter| iter.len());
+        let Some(lead) = position_iters.first() else {
+            return false;
+        };
+        let lead_position = lead.position_in_query;
+
+        loop {
+            let Some(anchor) = position_iters[0].absolute_position() else {
+                return false;
+            };
+            let Some(base) = anchor.checked_sub(lead_position as u32) else {
+                position_iters[0].advance_next();
+                continue;
+            };
+
+            let mut next_lead_relative = None;
+            let mut matched = true;
+            for follower in position_iters.iter_mut().skip(1) {
+                let Some(target) = base.checked_add(follower.position_in_query as u32) else {
+                    return false;
+                };
+                let Some(position) = follower.advance_to_absolute(target) else {
+                    return false;
+                };
+                if position != target {
+                    next_lead_relative = Some(position as i32 - follower.position_in_query);
+                    matched = false;
+                    break;
+                }
+            }
+
+            if matched {
+                return true;
+            }
+
+            position_iters[0].advance_to_relative(next_lead_relative.unwrap());
+        }
+    }
+}
+
+#[derive(Debug)]
+enum PositionValues<'a> {
+    Borrowed(&'a [u32]),
+    Owned(Vec<u32>),
+}
+
+impl<'a> PositionValues<'a> {
+    fn as_slice(&self) -> &[u32] {
+        match self {
+            Self::Borrowed(values) => values,
+            Self::Owned(values) => values.as_slice(),
+        }
+    }
+
+    fn len(&self) -> usize {
+        self.as_slice().len()
+    }
+}
+
+#[derive(Debug)]
+struct PositionCursor<'a> {
+    positions: PositionValues<'a>,
+    pub position_in_query: i32,
+    index: usize,
+}
+
+impl<'a> PositionCursor<'a> {
+    fn new(positions: PositionValues<'a>, position_in_query: i32) -> Self {
+        Self {
+            positions,
+            position_in_query,
+            index: 0,
+        }
+    }
+
+    fn len(&self) -> usize {
+        self.positions.len()
+    }
+
+    fn absolute_position(&self) -> Option<u32> {
+        self.positions.as_slice().get(self.index).copied()
+    }
+
+    fn relative_position(&self) -> Option<i32> {
+        self.positions
+            .as_slice()
+            .get(self.index)
+            .map(|position| *position as i32 - self.position_in_query)
+    }
+
+    fn advance_to_relative(&mut self, least_relative_pos: i32) {
+        if self.index >= self.len() {
+            return;
+        }
+        let least_pos = least_relative_pos + self.position_in_query;
+        let least_pos = least_pos.max(0) as u32;
+        let values = self.positions.as_slice();
+        self.index += values[self.index..].partition_point(|&pos| pos < least_pos);
+    }
+
+    fn advance_to_absolute(&mut self, least_pos: u32) -> Option<u32> {
+        if self.index >= self.len() {
+            return None;
+        }
+        let values = self.positions.as_slice();
+        self.index += values[self.index..].partition_point(|&pos| pos < least_pos);
+        self.absolute_position()
+    }
+
+    fn advance_next(&mut self) {
+        self.index = self.index.saturating_add(1).min(self.len());
     }
 }
 
@@ -899,10 +1500,47 @@ mod tests {
     use crate::{
         metrics::NoOpMetricsCollector,
         scalar::inverted::{
-            CompressedPostingList, PlainPostingList, encoding::compress_posting_list,
+            CompressedPostingList, PlainPostingList, PostingListBuilder, builder::PositionRecorder,
+            encoding::compress_posting_list,
         },
     };
 
+    struct UnitScorer;
+
+    impl Scorer for UnitScorer {
+        fn query_weight(&self, _token: &str) -> f32 {
+            1.0
+        }
+
+        fn doc_weight(&self, freq: u32, _doc_tokens: u32) -> f32 {
+            freq as f32
+        }
+    }
+
+    struct PanicQueryWeightScorer;
+
+    impl Scorer for PanicQueryWeightScorer {
+        fn query_weight(&self, _token: &str) -> f32 {
+            panic!("query_weight should be precomputed before WAND construction");
+        }
+
+        fn doc_weight(&self, freq: u32, _doc_tokens: u32) -> f32 {
+            freq as f32
+        }
+    }
+
+    struct InverseDocLengthScorer;
+
+    impl Scorer for InverseDocLengthScorer {
+        fn query_weight(&self, _token: &str) -> f32 {
+            1.0
+        }
+
+        fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32 {
+            freq as f32 / doc_tokens as f32
+        }
+    }
+
     fn generate_posting_list(
         doc_ids: Vec<u32>,
         max_score: f32,
@@ -923,6 +1561,7 @@ mod tests {
                 blocks,
                 max_score,
                 doc_ids.len() as u32,
+                crate::scalar::inverted::PostingTailCodec::VarintDelta,
                 None,
             ))
         } else {
@@ -935,6 +1574,43 @@ mod tests {
         }
     }
 
+    fn generate_posting_list_with_positions(
+        doc_ids: Vec<u32>,
+        positions_by_doc: Vec<Vec<u32>>,
+        max_score: f32,
+        is_compressed: bool,
+    ) -> PostingList {
+        let freqs = positions_by_doc
+            .iter()
+            .map(|positions| positions.len() as u32)
+            .collect::<Vec<_>>();
+        if is_compressed {
+            let mut builder = PostingListBuilder::new(true);
+            for (doc_id, positions) in doc_ids.iter().copied().zip(positions_by_doc) {
+                builder.add(doc_id, PositionRecorder::Position(positions.into()));
+            }
+            let batch = builder
+                .to_batch(vec![max_score; doc_ids.len().div_ceil(BLOCK_SIZE)])
+                .unwrap();
+            PostingList::from_batch(&batch, Some(max_score), Some(doc_ids.len() as u32)).unwrap()
+        } else {
+            let mut position_builder =
+                arrow::array::ListBuilder::new(arrow::array::Int32Builder::new());
+            for positions in positions_by_doc {
+                for position in positions {
+                    position_builder.values().append_value(position as i32);
+                }
+                position_builder.append(true);
+            }
+            PostingList::Plain(PlainPostingList::new(
+                ScalarBuffer::from_iter(doc_ids.iter().map(|id| *id as u64)),
+                ScalarBuffer::from_iter(freqs.iter().map(|freq| *freq as f32)),
+                Some(max_score),
+                Some(position_builder.finish()),
+            ))
+        }
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_wand(#[values(false, true)] is_compressed: bool) {
@@ -1044,6 +1720,208 @@ mod tests {
         assert!(result.is_ok());
     }
 
+    #[test]
+    fn test_wand_new_uses_precomputed_query_weight() {
+        let mut docs = DocSet::default();
+        docs.append(1, 1);
+
+        let postings = vec![PostingIterator::with_query_weight(
+            String::from("term"),
+            0,
+            0,
+            2.0,
+            generate_posting_list(vec![0], 1.0, None, false),
+            docs.len(),
+        )];
+
+        let wand = Wand::new(
+            Operator::Or,
+            postings.into_iter(),
+            &docs,
+            PanicQueryWeightScorer,
+        );
+        assert_eq!(wand.head.len(), 1);
+    }
+
+    #[test]
+    fn test_and_search_terminates_for_disjoint_postings() {
+        let mut docs = DocSet::default();
+        for i in 0..6 {
+            docs.append(i, 1);
+        }
+
+        let postings = vec![
+            PostingIterator::with_query_weight(
+                String::from("a"),
+                0,
+                0,
+                1.0,
+                generate_posting_list(vec![0, 2, 4], 1.0, None, false),
+                docs.len(),
+            ),
+            PostingIterator::with_query_weight(
+                String::from("b"),
+                1,
+                1,
+                1.0,
+                generate_posting_list(vec![1, 3, 5], 1.0, None, false),
+                docs.len(),
+            ),
+        ];
+
+        let mut wand = Wand::new(Operator::And, postings.into_iter(), &docs, UnitScorer);
+        assert!(wand.next().unwrap().is_none());
+    }
+
+    #[test]
+    fn test_up_to_refreshes_on_first_candidate() {
+        let mut docs = DocSet::default();
+        for i in 0..=(BLOCK_SIZE as u64 + 1) {
+            docs.append(i, 1);
+        }
+
+        let postings = vec![PostingIterator::with_query_weight(
+            String::from("term"),
+            0,
+            0,
+            1.0,
+            generate_posting_list(
+                (0..=(BLOCK_SIZE as u32 + 1)).collect(),
+                1.0,
+                Some(vec![1.0, 1.0]),
+                true,
+            ),
+            docs.len(),
+        )];
+
+        let mut wand = Wand::new(Operator::Or, postings.into_iter(), &docs, UnitScorer);
+        assert!(wand.up_to.is_none());
+        let _ = wand.next().unwrap();
+        assert!(wand.up_to.is_some());
+    }
+
+    #[test]
+    fn test_and_search_prunes_with_threshold_and_keeps_candidate() {
+        let mut docs = DocSet::default();
+        for i in 0..(2 * BLOCK_SIZE as u64) {
+            let doc_tokens = if i < BLOCK_SIZE as u64 { 100 } else { 1 };
+            docs.append(i, doc_tokens);
+        }
+        let all_docs = (0..2 * BLOCK_SIZE as u32).collect::<Vec<_>>();
+
+        let postings = vec![
+            PostingIterator::with_query_weight(
+                String::from("a"),
+                0,
+                0,
+                1.0,
+                generate_posting_list(all_docs.clone(), 1.0, Some(vec![0.02, 1.0]), true),
+                docs.len(),
+            ),
+            PostingIterator::with_query_weight(
+                String::from("b"),
+                1,
+                1,
+                1.0,
+                generate_posting_list(all_docs, 1.0, Some(vec![0.02, 1.0]), true),
+                docs.len(),
+            ),
+        ];
+
+        let mut wand = Wand::new(
+            Operator::And,
+            postings.into_iter(),
+            &docs,
+            InverseDocLengthScorer,
+        );
+        wand.threshold = 0.5;
+
+        let candidate = wand.next().unwrap().unwrap();
+        assert_eq!(candidate.0.doc_id(), BLOCK_SIZE as u64);
+    }
+
+    #[rstest]
+    fn test_wand_batches_lagging_iterators(#[values(false, true)] is_compressed: bool) {
+        let mut docs = DocSet::default();
+        for i in 0..16 {
+            docs.append(i as u64, 1);
+        }
+
+        let postings = vec![
+            PostingIterator::new(
+                String::from("a"),
+                0,
+                0,
+                generate_posting_list(vec![1, 10], 1.0, None, is_compressed),
+                docs.len(),
+            ),
+            PostingIterator::new(
+                String::from("b"),
+                1,
+                1,
+                generate_posting_list(vec![2, 10], 1.0, None, is_compressed),
+                docs.len(),
+            ),
+            PostingIterator::new(
+                String::from("c"),
+                2,
+                2,
+                generate_posting_list(vec![10], 1.0, None, is_compressed),
+                docs.len(),
+            ),
+        ];
+
+        let mut wand = Wand::new(Operator::Or, postings.into_iter(), &docs, UnitScorer);
+        wand.threshold = 2.5;
+
+        let candidate = wand.next().unwrap().unwrap();
+        assert_eq!(candidate.0.doc_id(), 10);
+        assert_eq!(wand.lead.len(), 3);
+    }
+
+    #[test]
+    fn test_flat_search_or_keeps_masked_docs_in_same_block() {
+        let mut docs = DocSet::default();
+        for i in 0..=(BLOCK_SIZE as u64 + 1) {
+            let doc_tokens = if i == 1 { 100 } else { 1 };
+            docs.append(i, doc_tokens);
+        }
+
+        let posting = PostingIterator::with_query_weight(
+            String::from("term"),
+            0,
+            0,
+            1.0,
+            generate_posting_list(
+                (1..=(BLOCK_SIZE as u32 + 1)).collect(),
+                1.0,
+                Some(vec![1.0, 1.0]),
+                true,
+            ),
+            docs.len(),
+        );
+
+        let mut wand = Wand::new(
+            Operator::Or,
+            vec![posting].into_iter(),
+            &docs,
+            InverseDocLengthScorer,
+        );
+        wand.threshold = 0.5;
+
+        let selected = vec![RowAddress::from(1_u64), RowAddress::from(2_u64)];
+        let result = wand
+            .flat_search(
+                &FtsSearchParams::default(),
+                Box::new(selected.into_iter()),
+                &NoOpMetricsCollector,
+            )
+            .unwrap();
+
+        let matched = result.into_iter().map(|doc| doc.row_id).collect::<Vec<_>>();
+        assert_eq!(matched, vec![2]);
+    }
+
     #[test]
     fn test_block_max_score_matches_stored_value() {
         let doc_ids = vec![0_u32];
@@ -1062,4 +1940,138 @@ mod tests {
             "block max score should match stored value"
         );
     }
+
+    #[rstest]
+    fn test_exact_phrase_with_repeated_terms(#[values(false, true)] is_compressed: bool) {
+        let mut docs = DocSet::default();
+        docs.append(0, 16);
+
+        let token_a_positions = vec![vec![1_u32, 3, 10]];
+        let token_b_positions = vec![vec![2_u32, 11]];
+        let postings = vec![
+            PostingIterator::new(
+                String::from("a"),
+                0,
+                0,
+                generate_posting_list_with_positions(
+                    vec![0],
+                    token_a_positions.clone(),
+                    1.0,
+                    is_compressed,
+                ),
+                docs.len(),
+            ),
+            PostingIterator::new(
+                String::from("b"),
+                1,
+                1,
+                generate_posting_list_with_positions(
+                    vec![0],
+                    token_b_positions,
+                    1.0,
+                    is_compressed,
+                ),
+                docs.len(),
+            ),
+            PostingIterator::new(
+                String::from("a"),
+                2,
+                2,
+                generate_posting_list_with_positions(
+                    vec![0],
+                    token_a_positions,
+                    1.0,
+                    is_compressed,
+                ),
+                docs.len(),
+            ),
+        ];
+
+        let bm25 = IndexBM25Scorer::new(std::iter::empty());
+        let wand = Wand::new(Operator::And, postings.into_iter(), &docs, bm25);
+        assert!(wand.check_exact_positions());
+        assert!(wand.check_positions(0));
+    }
+
+    #[rstest]
+    fn test_exact_phrase_respects_query_position_gaps(#[values(false, true)] is_compressed: bool) {
+        let mut docs = DocSet::default();
+        docs.append(0, 16);
+
+        let postings = vec![
+            PostingIterator::new(
+                String::from("want"),
+                0,
+                0,
+                generate_posting_list_with_positions(
+                    vec![0],
+                    vec![vec![0_u32]],
+                    1.0,
+                    is_compressed,
+                ),
+                docs.len(),
+            ),
+            PostingIterator::new(
+                String::from("apple"),
+                1,
+                2,
+                generate_posting_list_with_positions(
+                    vec![0],
+                    vec![vec![2_u32]],
+                    1.0,
+                    is_compressed,
+                ),
+                docs.len(),
+            ),
+        ];
+
+        let bm25 = IndexBM25Scorer::new(std::iter::empty());
+        let wand = Wand::new(Operator::And, postings.into_iter(), &docs, bm25);
+        assert!(wand.check_exact_positions());
+        assert!(wand.check_positions(0));
+    }
+
+    #[rstest]
+    fn test_and_phrase_miss_advances_to_next_candidate(#[values(false, true)] is_compressed: bool) {
+        let mut docs = DocSet::default();
+        docs.append(0, 8);
+        docs.append(1, 8);
+
+        let postings = vec![
+            PostingIterator::new(
+                String::from("a"),
+                0,
+                0,
+                generate_posting_list_with_positions(
+                    vec![0, 1],
+                    vec![vec![1_u32], vec![10_u32]],
+                    1.0,
+                    is_compressed,
+                ),
+                docs.len(),
+            ),
+            PostingIterator::new(
+                String::from("b"),
+                1,
+                1,
+                generate_posting_list_with_positions(
+                    vec![0, 1],
+                    vec![vec![3_u32], vec![11_u32]],
+                    1.0,
+                    is_compressed,
+                ),
+                docs.len(),
+            ),
+        ];
+
+        let mut wand = Wand::new(Operator::And, postings.into_iter(), &docs, UnitScorer);
+        let first = wand.next().unwrap().unwrap();
+        assert_eq!(first.0.doc_id(), 0);
+        assert!(!wand.check_positions(0));
+
+        wand.threshold = 1.5;
+        let second = wand.next().unwrap().unwrap();
+        assert_eq!(second.0.doc_id(), 1);
+        assert!(wand.check_positions(0));
+    }
 }
diff --git a/rust/lance-index/src/scalar/json.rs b/rust/lance-index/src/scalar/json.rs
index 9025ab05c3e..63465547c8f 100644
--- a/rust/lance-index/src/scalar/json.rs
+++ b/rust/lance-index/src/scalar/json.rs
@@ -130,6 +130,7 @@ impl ScalarIndex for JsonIndex {
             index_details: prost_types::Any::from_msg(&json_details)?,
             // TODO: We should store the target index version in the details
             index_version: JSON_INDEX_VERSION,
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -151,6 +152,7 @@ impl ScalarIndex for JsonIndex {
             index_details: prost_types::Any::from_msg(&json_details)?,
             // TODO: We should store the target index version in the details
             index_version: JSON_INDEX_VERSION,
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -779,6 +781,7 @@ impl ScalarIndexPlugin for JsonIndexPlugin {
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&index_details)?,
             index_version: JSON_INDEX_VERSION,
+            files: Some(index_store.list_files_with_sizes().await?),
         })
     }
 
diff --git a/rust/lance-index/src/scalar/label_list.rs b/rust/lance-index/src/scalar/label_list.rs
index 9bd77dae218..e6238d9d5a8 100644
--- a/rust/lance-index/src/scalar/label_list.rs
+++ b/rust/lance-index/src/scalar/label_list.rs
@@ -1,20 +1,28 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::{any::Any, collections::HashMap, fmt::Debug, pin::Pin, sync::Arc};
+use std::{
+    any::Any,
+    collections::HashMap,
+    fmt::Debug,
+    pin::Pin,
+    sync::{Arc, Mutex},
+};
 
 use arrow::array::AsArray;
 use arrow_array::{Array, RecordBatch, UInt64Array};
 use arrow_schema::{DataType, Field, Fields, Schema, SchemaRef};
 use async_trait::async_trait;
+use bytes::Bytes;
 use datafusion::execution::RecordBatchStream;
 use datafusion::physical_plan::{SendableRecordBatchStream, stream::RecordBatchStreamAdapter};
 use datafusion_common::ScalarValue;
 use deepsize::DeepSizeOf;
 use futures::{StreamExt, TryStream, TryStreamExt, stream::BoxStream};
 use lance_core::cache::LanceCache;
-use lance_core::utils::mask::{NullableRowAddrSet, RowAddrTreeMap};
-use lance_core::{Error, Result};
+use lance_core::error::LanceOptionExt;
+use lance_core::utils::mask::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps};
+use lance_core::{Error, ROW_ID, Result};
 use roaring::RoaringBitmap;
 use tracing::instrument;
 
@@ -33,7 +41,9 @@ use crate::scalar::{CreatedIndex, UpdateCriteria};
 use crate::{Index, IndexType};
 
 pub const BITMAP_LOOKUP_NAME: &str = "bitmap_page_lookup.lance";
-const LABEL_LIST_INDEX_VERSION: u32 = 0;
+pub const LABEL_LIST_NULLS_METADATA_KEY: &str = "lance:label_list_nulls";
+pub const LABEL_LIST_NULLS_MIN_VERSION: i32 = 1;
+const LABEL_LIST_INDEX_VERSION: u32 = 1;
 
 #[async_trait]
 trait LabelListSubIndex: ScalarIndex + DeepSizeOf {
@@ -64,12 +74,16 @@ impl<T: ScalarIndex + DeepSizeOf> LabelListSubIndex for T {}
 /// and `array_has` / `array_contains`, using an underlying bitmap index.
 #[derive(Clone, Debug, DeepSizeOf)]
 pub struct LabelListIndex {
-    values_index: Arc<dyn LabelListSubIndex>,
+    values_index: Arc<BitmapIndex>,
+    list_nulls: Arc<RowAddrTreeMap>,
 }
 
 impl LabelListIndex {
-    fn new(values_index: Arc<dyn LabelListSubIndex>) -> Self {
-        Self { values_index }
+    fn new(values_index: Arc<BitmapIndex>, list_nulls: Arc<RowAddrTreeMap>) -> Self {
+        Self {
+            values_index,
+            list_nulls,
+        }
     }
 
     async fn load(
@@ -77,9 +91,10 @@ impl LabelListIndex {
         frag_reuse_index: Option<Arc<FragReuseIndex>>,
         index_cache: &LanceCache,
     ) -> Result<Arc<Self>> {
-        BitmapIndex::load(store, frag_reuse_index, index_cache)
-            .await
-            .map(|index| Arc::new(Self::new(index)))
+        let values_index =
+            BitmapIndex::load(store.clone(), frag_reuse_index.clone(), index_cache).await?;
+        let list_nulls = read_list_nulls(store, frag_reuse_index).await?;
+        Ok(Arc::new(Self::new(values_index, Arc::new(list_nulls))))
     }
 }
 
@@ -182,6 +197,13 @@ impl ScalarIndex for LabelListIndex {
                 self.set_union(values_results, labels.len() == 1).await
             }
         }?;
+        let row_ids = if self.list_nulls.as_ref().is_empty() {
+            row_ids
+        } else {
+            let mut nulls = row_ids.null_rows().clone();
+            nulls |= self.list_nulls.as_ref();
+            row_ids.with_nulls(nulls)
+        };
         Ok(SearchResult::Exact(row_ids))
     }
 
@@ -195,12 +217,29 @@ impl ScalarIndex for LabelListIndex {
         mapping: &HashMap<u64, Option<u64>>,
         dest_store: &dyn IndexStore,
     ) -> Result<CreatedIndex> {
-        self.values_index.remap(mapping, dest_store).await?;
+        let state = self.values_index.load_bitmap_index_state().await?;
+        let remapped_state = BitmapIndexPlugin::remap_bitmap_state(state, mapping);
+        let remapped_nulls =
+            RowAddrTreeMap::from_iter(self.list_nulls.row_addrs().unwrap().filter_map(|addr| {
+                let addr_as_u64 = u64::from(addr);
+                mapping
+                    .get(&addr_as_u64)
+                    .copied()
+                    .unwrap_or(Some(addr_as_u64))
+            }));
+        write_label_list_bitmap_index(
+            remapped_state,
+            dest_store,
+            self.values_index.value_type(),
+            &remapped_nulls,
+        )
+        .await?;
 
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::LabelListIndexDetails::default())
                 .unwrap(),
             index_version: LABEL_LIST_INDEX_VERSION,
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -211,14 +250,24 @@ impl ScalarIndex for LabelListIndex {
         dest_store: &dyn IndexStore,
         old_data_filter: Option<super::OldIndexDataFilter>,
     ) -> Result<CreatedIndex> {
-        self.values_index
-            .update(unnest_chunks(new_data)?, dest_store, old_data_filter)
-            .await?;
+        let state = self.values_index.load_bitmap_index_state().await?;
+        let list_nulls = Arc::new(Mutex::new(RowAddrTreeMap::new()));
+        let new_data = track_list_nulls(new_data, list_nulls.clone());
+        let (merged_state, value_type) =
+            BitmapIndexPlugin::build_bitmap_index_state(unnest_chunks(new_data)?, state).await?;
+        let _ = old_data_filter;
+        let mut merged_nulls = (*self.list_nulls).clone();
+        let new_nulls = list_nulls.lock().unwrap().clone();
+        if !new_nulls.is_empty() {
+            merged_nulls |= &new_nulls;
+        }
+        write_label_list_bitmap_index(merged_state, dest_store, &value_type, &merged_nulls).await?;
 
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::LabelListIndexDetails::default())
                 .unwrap(),
             index_version: LABEL_LIST_INDEX_VERSION,
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -256,6 +305,44 @@ fn extract_flatten_indices(list_arr: &dyn Array) -> UInt64Array {
     }
 }
 
+/// Collect row_ids for list-level NULLs before unnest; unnest drops NULL lists entirely.
+fn track_list_nulls(
+    source: SendableRecordBatchStream,
+    list_nulls: Arc<Mutex<RowAddrTreeMap>>,
+) -> SendableRecordBatchStream {
+    let schema = source.schema();
+    let stream = source.try_filter_map(move |batch| {
+        let list_nulls = list_nulls.clone();
+        async move {
+            record_list_nulls(&batch, &list_nulls)?;
+            Ok(Some(batch))
+        }
+    });
+
+    Box::pin(RecordBatchStreamAdapter::new(schema, stream))
+}
+
+fn record_list_nulls(
+    batch: &RecordBatch,
+    list_nulls: &Arc<Mutex<RowAddrTreeMap>>,
+) -> datafusion_common::Result<()> {
+    let values = batch.column_by_name(VALUE_COLUMN_NAME).expect_ok()?;
+    let row_ids = batch.column_by_name(ROW_ID).expect_ok()?;
+    let row_ids = row_ids.as_any().downcast_ref::<UInt64Array>().unwrap();
+
+    let mut local_nulls = RowAddrTreeMap::new();
+    for i in 0..values.len() {
+        if values.is_null(i) {
+            local_nulls.insert(row_ids.value(i));
+        }
+    }
+    if !local_nulls.is_empty() {
+        let mut guard = list_nulls.lock().unwrap();
+        *guard |= &local_nulls;
+    }
+    Ok(())
+}
+
 fn unnest_schema(schema: &Schema) -> SchemaRef {
     let mut fields_iter = schema.fields.iter().cloned();
     let key_field = fields_iter.next().unwrap();
@@ -353,6 +440,54 @@ fn unnest_chunks(
     )))
 }
 
+async fn read_list_nulls(
+    store: Arc<dyn IndexStore>,
+    frag_reuse_index: Option<Arc<FragReuseIndex>>,
+) -> Result<RowAddrTreeMap> {
+    let reader = store.open_index_file(BITMAP_LOOKUP_NAME).await?;
+    if let Some(buffer_idx_str) = reader.schema().metadata.get(LABEL_LIST_NULLS_METADATA_KEY) {
+        let buffer_idx = buffer_idx_str.parse::<u32>().map_err(|err| {
+            Error::internal(format!(
+                "LabelList metadata key {} had invalid global buffer index {}: {}",
+                LABEL_LIST_NULLS_METADATA_KEY, buffer_idx_str, err
+            ))
+        })?;
+        let bytes = reader.read_global_buffer(buffer_idx).await?;
+        let null_map = RowAddrTreeMap::deserialize_from(bytes.as_ref())?;
+        return if let Some(frag_reuse_index) = frag_reuse_index {
+            Ok(frag_reuse_index.remap_row_addrs_tree_map(&null_map))
+        } else {
+            Ok(null_map)
+        };
+    }
+    Ok(RowAddrTreeMap::default())
+}
+
+fn serialize_list_nulls(null_map: &RowAddrTreeMap) -> Result<Bytes> {
+    let mut bytes = Vec::new();
+    null_map.serialize_into(&mut bytes)?;
+    Ok(Bytes::from(bytes))
+}
+
+async fn write_label_list_bitmap_index(
+    state: HashMap<ScalarValue, RowAddrTreeMap>,
+    store: &dyn IndexStore,
+    value_type: &DataType,
+    list_nulls: &RowAddrTreeMap,
+) -> Result<()> {
+    BitmapIndexPlugin::write_bitmap_index_with_extras(
+        state,
+        store,
+        value_type,
+        HashMap::new(),
+        vec![(
+            LABEL_LIST_NULLS_METADATA_KEY.to_string(),
+            serialize_list_nulls(list_nulls)?,
+        )],
+    )
+    .await
+}
+
 #[derive(Debug, Default)]
 pub struct LabelListIndexPlugin;
 
@@ -407,9 +542,9 @@ impl ScalarIndexPlugin for LabelListIndexPlugin {
         &self,
         data: SendableRecordBatchStream,
         index_store: &dyn IndexStore,
-        request: Box<dyn TrainingRequest>,
+        _request: Box<dyn TrainingRequest>,
         fragment_ids: Option<Vec<u32>>,
-        progress: Arc<dyn crate::progress::IndexBuildProgress>,
+        _progress: Arc<dyn crate::progress::IndexBuildProgress>,
     ) -> Result<CreatedIndex> {
         if fragment_ids.is_some() {
             return Err(Error::invalid_input_source(
@@ -440,15 +575,18 @@ impl ScalarIndexPlugin for LabelListIndexPlugin {
             .into()));
         }
 
+        let list_nulls = Arc::new(Mutex::new(RowAddrTreeMap::new()));
+        let data = track_list_nulls(data, list_nulls.clone());
         let data = unnest_chunks(data)?;
-        let bitmap_plugin = BitmapIndexPlugin;
-        bitmap_plugin
-            .train_index(data, index_store, request, fragment_ids, progress)
-            .await?;
+        let (state, value_type) =
+            BitmapIndexPlugin::build_bitmap_index_state(data, HashMap::new()).await?;
+        let list_nulls = list_nulls.lock().unwrap().clone();
+        write_label_list_bitmap_index(state, index_store, &value_type, &list_nulls).await?;
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::LabelListIndexDetails::default())
                 .unwrap(),
             index_version: LABEL_LIST_INDEX_VERSION,
+            files: Some(index_store.list_files_with_sizes().await?),
         })
     }
 
diff --git a/rust/lance-index/src/scalar/lance_format.rs b/rust/lance-index/src/scalar/lance_format.rs
index 078f00bc8e4..42bef5d7244 100644
--- a/rust/lance-index/src/scalar/lance_format.rs
+++ b/rust/lance-index/src/scalar/lance_format.rs
@@ -7,10 +7,12 @@ use super::{IndexReader, IndexStore, IndexWriter};
 use arrow_array::RecordBatch;
 use arrow_schema::Schema;
 use async_trait::async_trait;
+use bytes::Bytes;
 use deepsize::DeepSizeOf;
 use futures::TryStreamExt;
 use lance_core::{Error, Result, cache::LanceCache};
 use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
+use lance_encoding::version::LanceFileVersion;
 use lance_file::previous::{
     reader::FileReader as PreviousFileReader,
     writer::{FileWriter as PreviousFileWriter, ManifestProvider as PreviousManifestProvider},
@@ -21,6 +23,7 @@ use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
 use lance_io::utils::CachedFileSize;
 use lance_io::{ReadBatchParams, object_store::ObjectStore};
 use lance_table::format::SelfDescribingFileReader;
+use lance_table::format::{IndexFile, list_index_files_with_sizes};
 use object_store::path::Path;
 use std::cmp::min;
 use std::collections::HashMap;
@@ -31,12 +34,16 @@ use std::{any::Any, sync::Arc};
 /// Scalar indices are made up of named collections of record batches.  This
 /// struct relies on there being a dedicated directory for the index and stores
 /// each collection in a file in the lance format.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct LanceIndexStore {
     object_store: Arc<ObjectStore>,
     index_dir: Path,
     metadata_cache: Arc<LanceCache>,
     scheduler: Arc<ScanScheduler>,
+    /// Cached file sizes (filename -> size in bytes)
+    /// When set, used to avoid HEAD calls when opening files
+    file_sizes: HashMap<String, u64>,
+    format_version: LanceFileVersion,
 }
 
 impl DeepSizeOf for LanceIndexStore {
@@ -53,6 +60,21 @@ impl LanceIndexStore {
         object_store: Arc<ObjectStore>,
         index_dir: Path,
         metadata_cache: Arc<LanceCache>,
+    ) -> Self {
+        Self::with_format_version(
+            object_store,
+            index_dir,
+            metadata_cache,
+            LanceFileVersion::V2_0,
+        )
+    }
+
+    /// Create a new index store at the given directory with a specific format version
+    pub fn with_format_version(
+        object_store: Arc<ObjectStore>,
+        index_dir: Path,
+        metadata_cache: Arc<LanceCache>,
+        format_version: LanceFileVersion,
     ) -> Self {
         let scheduler = ScanScheduler::new(
             object_store.clone(),
@@ -63,8 +85,19 @@ impl LanceIndexStore {
             index_dir,
             metadata_cache,
             scheduler,
+            file_sizes: HashMap::new(),
+            format_version,
         }
     }
+
+    /// Set cached file sizes to avoid HEAD calls when opening files.
+    ///
+    /// The map should contain relative paths (e.g., "index.idx") as keys
+    /// and file sizes in bytes as values.
+    pub fn with_file_sizes(mut self, file_sizes: HashMap<String, u64>) -> Self {
+        self.file_sizes = file_sizes;
+        self
+    }
 }
 
 #[async_trait]
@@ -94,6 +127,10 @@ impl IndexWriter for current_writer::FileWriter {
         Ok(offset)
     }
 
+    async fn add_global_buffer(&mut self, data: Bytes) -> Result<u32> {
+        Self::add_global_buffer(self, data).await
+    }
+
     async fn finish(&mut self) -> Result<()> {
         Self::finish(self).await.map(|_| ())
     }
@@ -147,6 +184,10 @@ impl IndexReader for current_reader::FileReader {
         self.read_range(start as usize..end as usize, None).await
     }
 
+    async fn read_global_buffer(&self, n: u32) -> Result<Bytes> {
+        Self::read_global_buffer(self, n).await
+    }
+
     async fn read_range(
         &self,
         range: std::ops::Range<usize>,
@@ -201,6 +242,10 @@ impl IndexStore for LanceIndexStore {
         self
     }
 
+    fn clone_arc(&self) -> Arc<dyn IndexStore> {
+        Arc::new(self.clone())
+    }
+
     fn io_parallelism(&self) -> usize {
         self.object_store.io_parallelism()
     }
@@ -216,17 +261,23 @@ impl IndexStore for LanceIndexStore {
         let writer = current_writer::FileWriter::try_new(
             writer,
             schema,
-            current_writer::FileWriterOptions::default(),
+            current_writer::FileWriterOptions {
+                format_version: Some(self.format_version),
+                ..Default::default()
+            },
         )?;
         Ok(Box::new(writer))
     }
 
     async fn open_index_file(&self, name: &str) -> Result<Arc<dyn IndexReader>> {
         let path = self.index_dir.child(name);
-        let file_scheduler = self
-            .scheduler
-            .open_file(&path, &CachedFileSize::unknown())
-            .await?;
+        // Use cached file size if available, otherwise unknown (requires HEAD call)
+        let cached_size = self
+            .file_sizes
+            .get(name)
+            .map(|&size| CachedFileSize::new(size))
+            .unwrap_or_else(CachedFileSize::unknown);
+        let file_scheduler = self.scheduler.open_file(&path, &cached_size).await?;
         match current_reader::FileReader::try_open(
             file_scheduler,
             None,
@@ -296,10 +347,14 @@ impl IndexStore for LanceIndexStore {
         let path = self.index_dir.child(name);
         self.object_store.delete(&path).await
     }
+
+    async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>> {
+        list_index_files_with_sizes(&self.object_store, &self.index_dir).await
+    }
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
 
     use std::{collections::HashMap, ops::Bound};
 
@@ -325,6 +380,7 @@ pub mod tests {
     use arrow_schema::Schema as ArrowSchema;
     use arrow_schema::{DataType, Field, TimeUnit};
     use arrow_select::take::TakeOptions;
+    use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
     use datafusion_common::ScalarValue;
     use futures::FutureExt;
     use lance_core::ROW_ID;
@@ -380,6 +436,28 @@ pub mod tests {
         prost_types::Any::from_msg(&T::default()).unwrap()
     }
 
+    #[tokio::test]
+    async fn test_global_buffer_round_trip() {
+        let tempdir = TempDir::default();
+        let index_store = test_store(&tempdir);
+
+        let mut writer = index_store
+            .new_index_file("global-buffer.lance", Arc::new(Schema::empty()))
+            .await
+            .unwrap();
+        let expected = bytes::Bytes::from_static(b"scalar-global-buffer");
+        let buffer_idx = writer.add_global_buffer(expected.clone()).await.unwrap();
+        writer.finish().await.unwrap();
+
+        let reader = index_store
+            .open_index_file("global-buffer.lance")
+            .await
+            .unwrap();
+        let actual = reader.read_global_buffer(buffer_idx).await.unwrap();
+
+        assert_eq!(actual, expected);
+    }
+
     #[tokio::test]
     async fn test_basic_btree() {
         let tempdir = TempDir::default();
@@ -909,13 +987,37 @@ pub mod tests {
         index_store: &Arc<dyn IndexStore>,
         data: impl RecordBatchReader + Send + Sync + 'static,
     ) {
-        let data = lance_datafusion::utils::reader_to_stream(Box::new(data));
+        // Sort the data by value column (nulls first) to match the production
+        // scanner behavior (TrainingOrdering::Values).
+        let schema = data.schema();
+        let batches: Vec<_> = data
+            .into_iter()
+            .collect::<std::result::Result<Vec<_>, _>>()
+            .unwrap();
+        let combined = arrow::compute::concat_batches(&schema, &batches).unwrap();
+        let options = arrow::compute::SortOptions {
+            descending: false,
+            nulls_first: true,
+        };
+        let indices =
+            arrow::compute::sort_to_indices(combined.column(0), Some(options), None).unwrap();
+        let sorted_columns: Vec<_> = combined
+            .columns()
+            .iter()
+            .map(|col| arrow::compute::take(col.as_ref(), &indices, None).unwrap())
+            .collect();
+        let sorted_batch = RecordBatch::try_new(schema.clone(), sorted_columns).unwrap();
+        let stream = Box::pin(RecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move { Ok(sorted_batch) }),
+        ));
+
         let request = BitmapIndexPlugin
             .new_training_request("{}", &Field::new(VALUE_COLUMN_NAME, DataType::Int32, false))
             .unwrap();
         BitmapIndexPlugin
             .train_index(
-                data,
+                stream,
                 index_store.as_ref(),
                 request,
                 None,
@@ -1604,4 +1706,57 @@ pub mod tests {
             _ => panic!("Expected Exact search result"),
         }
     }
+
+    #[tokio::test]
+    async fn test_label_list_bitmap_only_layout_is_compatible() {
+        let tempdir = TempDir::default();
+        let index_store = test_store(&tempdir);
+
+        // Simulate an older released layout that only had the bitmap lookup file.
+        let values = arrow_array::UInt8Array::from(vec![1, 2]);
+        let row_ids = UInt64Array::from(vec![0, 2]);
+        let schema = Arc::new(Schema::new(vec![
+            Field::new(VALUE_COLUMN_NAME, DataType::UInt8, true),
+            Field::new(ROW_ID, DataType::UInt64, false),
+        ]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(values), Arc::new(row_ids)])
+            .unwrap();
+
+        BitmapIndexPlugin::train_bitmap_index(
+            lance_datafusion::utils::reader_to_stream(Box::new(RecordBatchIterator::new(
+                vec![Ok(batch)],
+                schema,
+            ))),
+            index_store.as_ref(),
+        )
+        .await
+        .unwrap();
+
+        let index = LabelListIndexPlugin
+            .load_index(
+                index_store,
+                &default_details::<pbold::LabelListIndexDetails>(),
+                None,
+                &LanceCache::no_cache(),
+            )
+            .await
+            .unwrap();
+
+        let query = LabelListQuery::HasAnyLabel(vec![ScalarValue::UInt8(Some(1))]);
+        let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
+
+        match result {
+            SearchResult::Exact(row_ids) => {
+                assert!(row_ids.null_rows().is_empty());
+                let actual_rows: Vec<u64> = row_ids
+                    .true_rows()
+                    .row_addrs()
+                    .unwrap()
+                    .map(u64::from)
+                    .collect();
+                assert_eq!(actual_rows, vec![0]);
+            }
+            _ => panic!("Expected Exact search result"),
+        }
+    }
 }
diff --git a/rust/lance-index/src/scalar/ngram.rs b/rust/lance-index/src/scalar/ngram.rs
index ce992af70ab..d14f1f2b4ae 100644
--- a/rust/lance-index/src/scalar/ngram.rs
+++ b/rust/lance-index/src/scalar/ngram.rs
@@ -41,10 +41,12 @@ use lance_core::utils::tracing::{IO_TYPE_LOAD_SCALAR_PART, TRACE_IO_EVENTS};
 use lance_core::{Error, utils::mask::RowAddrTreeMap};
 use lance_core::{ROW_ID, Result};
 use lance_io::object_store::ObjectStore;
+use lance_tokenizer::{
+    AlphaNumOnlyFilter, AsciiFoldingFilter, LowerCaser, NgramTokenizer, RawTokenizer, TextAnalyzer,
+};
 use log::info;
 use roaring::{RoaringBitmap, RoaringTreemap};
 use serde::Serialize;
-use tantivy::tokenizer::TextAnalyzer;
 use tracing::instrument;
 
 const TOKENS_COL: &str = "tokens";
@@ -65,15 +67,15 @@ pub static POSTINGS_SCHEMA: LazyLock<SchemaRef> = LazyLock::new(|| {
     ]))
 });
 pub static TEXT_PREPPER: LazyLock<TextAnalyzer> = LazyLock::new(|| {
-    TextAnalyzer::builder(tantivy::tokenizer::RawTokenizer::default())
-        .filter(tantivy::tokenizer::LowerCaser)
-        .filter(tantivy::tokenizer::AsciiFoldingFilter)
+    TextAnalyzer::builder(RawTokenizer::default())
+        .filter(LowerCaser)
+        .filter(AsciiFoldingFilter)
         .build()
 });
 /// Currently we ALWAYS use trigrams with ascii folding and lower casing.  We may want to make this configurable in the future.
 pub static NGRAM_TOKENIZER: LazyLock<TextAnalyzer> = LazyLock::new(|| {
-    TextAnalyzer::builder(tantivy::tokenizer::NgramTokenizer::all_ngrams(3, 3).unwrap())
-        .filter(tantivy::tokenizer::AlphaNumOnlyFilter)
+    TextAnalyzer::builder(NgramTokenizer::all_ngrams(3, 3).unwrap())
+        .filter(AlphaNumOnlyFilter)
         .build()
 });
 
@@ -170,6 +172,10 @@ impl CacheKey for NGramPostingListKey {
     fn key(&self) -> std::borrow::Cow<'_, str> {
         format!("posting-list-{}", self.row_offset).into()
     }
+
+    fn type_name() -> &'static str {
+        "NGramPostingList"
+    }
 }
 
 impl NGramPostingList {
@@ -504,6 +510,7 @@ impl ScalarIndex for NGramIndex {
             index_details: prost_types::Any::from_msg(&pbold::NGramIndexDetails::default())
                 .unwrap(),
             index_version: NGRAM_INDEX_VERSION,
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -524,6 +531,7 @@ impl ScalarIndex for NGramIndex {
             index_details: prost_types::Any::from_msg(&pbold::NGramIndexDetails::default())
                 .unwrap(),
             index_version: NGRAM_INDEX_VERSION,
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -1291,6 +1299,7 @@ impl ScalarIndexPlugin for NGramIndexPlugin {
             index_details: prost_types::Any::from_msg(&pbold::NGramIndexDetails::default())
                 .unwrap(),
             index_version: NGRAM_INDEX_VERSION,
+            files: Some(index_store.list_files_with_sizes().await?),
         })
     }
 
@@ -1328,7 +1337,7 @@ mod tests {
     };
     use lance_datagen::{BatchCount, ByteCount, RowCount};
     use lance_io::object_store::ObjectStore;
-    use tantivy::tokenizer::TextAnalyzer;
+    use lance_tokenizer::TextAnalyzer;
 
     use crate::scalar::{
         ScalarIndex, SearchResult, TextQuery,
diff --git a/rust/lance-index/src/scalar/rtree.rs b/rust/lance-index/src/scalar/rtree.rs
index 598bef43f50..920a59bb4b2 100644
--- a/rust/lance-index/src/scalar/rtree.rs
+++ b/rust/lance-index/src/scalar/rtree.rs
@@ -249,6 +249,10 @@ impl CacheKey for RTreeCacheKey {
             Self::Nulls => "nulls".into(),
         }
     }
+
+    fn type_name() -> &'static str {
+        "RTree"
+    }
 }
 
 #[derive(Clone)]
@@ -600,6 +604,7 @@ impl ScalarIndex for RTreeIndex {
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pb::RTreeIndexDetails::default())?,
             index_version: RTREE_INDEX_VERSION,
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -964,6 +969,7 @@ impl ScalarIndexPlugin for RTreeIndexPlugin {
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pb::RTreeIndexDetails::default())?,
             index_version: RTREE_INDEX_VERSION,
+            files: Some(index_store.list_files_with_sizes().await?),
         })
     }
 
diff --git a/rust/lance-index/src/scalar/zonemap.rs b/rust/lance-index/src/scalar/zonemap.rs
index 8db7c771cac..9156322b1dc 100644
--- a/rust/lance-index/src/scalar/zonemap.rs
+++ b/rust/lance-index/src/scalar/zonemap.rs
@@ -20,6 +20,7 @@ use crate::scalar::registry::{
 };
 use crate::scalar::{
     BuiltinIndexType, CreatedIndex, SargableQuery, ScalarIndexParams, UpdateCriteria,
+    compute_next_prefix,
 };
 use datafusion::functions_aggregate::min_max::{MaxAccumulator, MinAccumulator};
 use datafusion_expr::Accumulator;
@@ -325,6 +326,53 @@ impl ZoneMapIndex {
             SargableQuery::FullTextSearch(_) => Err(Error::not_supported_source(
                 "full text search is not supported for zonemap indexes".into(),
             )),
+            SargableQuery::LikePrefix(prefix) => {
+                // For prefix matching, a zone can match if:
+                // - zone.max >= prefix (there could be values >= prefix)
+                // - zone.min < next_prefix (there could be values < next_prefix)
+                //
+                // For example, prefix "foo":
+                // - Zone [aaa, azz]: max="azz" < "foo", so no match
+                // - Zone [fa, foz]: min="fa" < "fop", max="foz" >= "foo", so potential match
+                // - Zone [fop, fzz]: min="fop" >= "fop", so no match
+
+                let prefix_str = match prefix {
+                    ScalarValue::Utf8(Some(s)) => s.as_str(),
+                    ScalarValue::LargeUtf8(Some(s)) => s.as_str(),
+                    _ => return Ok(true), // Conservative: include zone if not a string prefix
+                };
+
+                // Empty prefix matches everything
+                if prefix_str.is_empty() {
+                    return Ok(true);
+                }
+
+                // Check zone.max >= prefix
+                let max_check = &zone.max >= prefix;
+                if !max_check {
+                    return Ok(false);
+                }
+
+                // Compute next_prefix by incrementing the last byte
+                // If the prefix ends with 0xFF bytes, we need to handle overflow
+                let next_prefix = compute_next_prefix(prefix_str);
+
+                match next_prefix {
+                    Some(next) => {
+                        // Check zone.min < next_prefix
+                        let next_scalar = match prefix {
+                            ScalarValue::Utf8(_) => ScalarValue::Utf8(Some(next)),
+                            ScalarValue::LargeUtf8(_) => ScalarValue::LargeUtf8(Some(next)),
+                            _ => return Ok(true),
+                        };
+                        Ok(zone.min < next_scalar)
+                    }
+                    None => {
+                        // No upper bound (prefix is all 0xFF), so any zone with max >= prefix matches
+                        Ok(true)
+                    }
+                }
+            }
         }
     }
 
@@ -557,6 +605,7 @@ impl ScalarIndex for ZoneMapIndex {
             index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default())
                 .unwrap(),
             index_version: ZONEMAP_INDEX_VERSION,
+            files: Some(dest_store.list_files_with_sizes().await?),
         })
     }
 
@@ -890,6 +939,7 @@ impl ScalarIndexPlugin for ZoneMapIndexPlugin {
             index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default())
                 .unwrap(),
             index_version: ZONEMAP_INDEX_VERSION,
+            files: Some(index_store.list_files_with_sizes().await?),
         })
     }
 
@@ -921,7 +971,11 @@ mod tests {
     use futures::{StreamExt, TryStreamExt, stream};
     use lance_core::utils::mask::NullableRowAddrSet;
     use lance_core::utils::tempfile::TempObjDir;
-    use lance_core::{ROW_ADDR, cache::LanceCache, utils::mask::RowAddrTreeMap};
+    use lance_core::{
+        ROW_ADDR,
+        cache::{LanceCache, WeakLanceCache},
+        utils::mask::RowAddrTreeMap,
+    };
     use lance_datafusion::datagen::DatafusionDatagenExt;
     use lance_datagen::ArrayGeneratorExt;
     use lance_datagen::{BatchCount, RowCount, array};
@@ -2197,4 +2251,289 @@ mod tests {
             &[4294967296, 4294967297, 4294967298, 4294967299, 4294967300]
         );
     }
+
+    #[tokio::test]
+    async fn test_like_prefix_query() {
+        let tmpdir = TempObjDir::default();
+        let test_store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // Create zones with different string ranges
+        // Zone 0: ["aaa", "azz"] - should NOT match "foo%"
+        // Zone 1: ["bar", "baz"] - should NOT match "foo%"
+        // Zone 2: ["fa", "foz"]  - should match "foo%" (contains potential matches)
+        // Zone 3: ["fop", "fzz"] - should NOT match "foo%" (all values >= "fop")
+        // Zone 4: ["foo", "foobar"] - should match "foo%"
+        // Zone 5: ["gaa", "gzz"] - should NOT match "foo%"
+
+        let zones = vec![
+            ZoneMapStatistics {
+                min: ScalarValue::Utf8(Some("aaa".to_string())),
+                max: ScalarValue::Utf8(Some("azz".to_string())),
+                null_count: 0,
+                nan_count: 0,
+                bound: ZoneBound {
+                    fragment_id: 0,
+                    start: 0,
+                    length: 100,
+                },
+            },
+            ZoneMapStatistics {
+                min: ScalarValue::Utf8(Some("bar".to_string())),
+                max: ScalarValue::Utf8(Some("baz".to_string())),
+                null_count: 0,
+                nan_count: 0,
+                bound: ZoneBound {
+                    fragment_id: 1,
+                    start: 0,
+                    length: 100,
+                },
+            },
+            ZoneMapStatistics {
+                min: ScalarValue::Utf8(Some("fa".to_string())),
+                max: ScalarValue::Utf8(Some("foz".to_string())),
+                null_count: 0,
+                nan_count: 0,
+                bound: ZoneBound {
+                    fragment_id: 2,
+                    start: 0,
+                    length: 100,
+                },
+            },
+            ZoneMapStatistics {
+                min: ScalarValue::Utf8(Some("fop".to_string())),
+                max: ScalarValue::Utf8(Some("fzz".to_string())),
+                null_count: 0,
+                nan_count: 0,
+                bound: ZoneBound {
+                    fragment_id: 3,
+                    start: 0,
+                    length: 100,
+                },
+            },
+            ZoneMapStatistics {
+                min: ScalarValue::Utf8(Some("foo".to_string())),
+                max: ScalarValue::Utf8(Some("foobar".to_string())),
+                null_count: 0,
+                nan_count: 0,
+                bound: ZoneBound {
+                    fragment_id: 4,
+                    start: 0,
+                    length: 100,
+                },
+            },
+            ZoneMapStatistics {
+                min: ScalarValue::Utf8(Some("gaa".to_string())),
+                max: ScalarValue::Utf8(Some("gzz".to_string())),
+                null_count: 0,
+                nan_count: 0,
+                bound: ZoneBound {
+                    fragment_id: 5,
+                    start: 0,
+                    length: 100,
+                },
+            },
+        ];
+
+        let index = ZoneMapIndex {
+            zones,
+            data_type: DataType::Utf8,
+            rows_per_zone: ROWS_PER_ZONE_DEFAULT,
+            store: test_store,
+            fri: None,
+            index_cache: WeakLanceCache::from(&LanceCache::no_cache()),
+        };
+
+        // Test LikePrefix query for "foo"
+        let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("foo".to_string())));
+        let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
+
+        // Should match zones 2 and 4 only
+        let mut expected = RowAddrTreeMap::new();
+        // Zone 2: fragment 2
+        expected.insert_range((2u64 << 32)..((2u64 << 32) + 100));
+        // Zone 4: fragment 4
+        expected.insert_range((4u64 << 32)..((4u64 << 32) + 100));
+
+        assert_eq!(result, SearchResult::at_most(expected));
+    }
+
+    #[tokio::test]
+    async fn test_like_prefix_edge_cases() {
+        let tmpdir = TempObjDir::default();
+        let test_store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // Test edge cases for LIKE prefix
+        let zones = vec![
+            // Zone with values that contain the prefix exactly
+            ZoneMapStatistics {
+                min: ScalarValue::Utf8(Some("test".to_string())),
+                max: ScalarValue::Utf8(Some("test".to_string())),
+                null_count: 0,
+                nan_count: 0,
+                bound: ZoneBound {
+                    fragment_id: 0,
+                    start: 0,
+                    length: 100,
+                },
+            },
+            // Zone with values that span across the prefix boundary
+            ZoneMapStatistics {
+                min: ScalarValue::Utf8(Some("te".to_string())),
+                max: ScalarValue::Utf8(Some("tf".to_string())),
+                null_count: 0,
+                nan_count: 0,
+                bound: ZoneBound {
+                    fragment_id: 1,
+                    start: 0,
+                    length: 100,
+                },
+            },
+            // Zone completely before prefix
+            ZoneMapStatistics {
+                min: ScalarValue::Utf8(Some("abc".to_string())),
+                max: ScalarValue::Utf8(Some("def".to_string())),
+                null_count: 0,
+                nan_count: 0,
+                bound: ZoneBound {
+                    fragment_id: 2,
+                    start: 0,
+                    length: 100,
+                },
+            },
+        ];
+
+        let index = ZoneMapIndex {
+            zones,
+            data_type: DataType::Utf8,
+            rows_per_zone: ROWS_PER_ZONE_DEFAULT,
+            store: test_store,
+            fri: None,
+            index_cache: WeakLanceCache::from(&LanceCache::no_cache()),
+        };
+
+        // Test LikePrefix "test"
+        let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("test".to_string())));
+        let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
+
+        // Should match zones 0 and 1
+        let mut expected = RowAddrTreeMap::new();
+        expected.insert_range(0..100); // Zone 0: fragment 0
+        expected.insert_range((1u64 << 32)..((1u64 << 32) + 100));
+
+        assert_eq!(result, SearchResult::at_most(expected));
+
+        // Test empty prefix - should match all zones
+        let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("".to_string())));
+        let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
+
+        let mut expected = RowAddrTreeMap::new();
+        expected.insert_range(0..100); // Zone 0: fragment 0
+        expected.insert_range((1u64 << 32)..((1u64 << 32) + 100));
+        expected.insert_range((2u64 << 32)..((2u64 << 32) + 100));
+
+        assert_eq!(result, SearchResult::at_most(expected));
+    }
+
+    #[tokio::test]
+    async fn test_like_prefix_large_utf8() {
+        let tmpdir = TempObjDir::default();
+        let test_store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // Test with LargeUtf8 type
+        let zones = vec![
+            ZoneMapStatistics {
+                min: ScalarValue::LargeUtf8(Some("aaa".to_string())),
+                max: ScalarValue::LargeUtf8(Some("azz".to_string())),
+                null_count: 0,
+                nan_count: 0,
+                bound: ZoneBound {
+                    fragment_id: 0,
+                    start: 0,
+                    length: 100,
+                },
+            },
+            ZoneMapStatistics {
+                min: ScalarValue::LargeUtf8(Some("foo".to_string())),
+                max: ScalarValue::LargeUtf8(Some("foobar".to_string())),
+                null_count: 0,
+                nan_count: 0,
+                bound: ZoneBound {
+                    fragment_id: 1,
+                    start: 0,
+                    length: 100,
+                },
+            },
+        ];
+
+        let index = ZoneMapIndex {
+            zones,
+            data_type: DataType::LargeUtf8,
+            rows_per_zone: ROWS_PER_ZONE_DEFAULT,
+            store: test_store,
+            fri: None,
+            index_cache: WeakLanceCache::from(&LanceCache::no_cache()),
+        };
+
+        // Test LikePrefix with LargeUtf8
+        let query = SargableQuery::LikePrefix(ScalarValue::LargeUtf8(Some("foo".to_string())));
+        let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
+
+        // Should match only zone 1
+        let mut expected = RowAddrTreeMap::new();
+        expected.insert_range((1u64 << 32)..((1u64 << 32) + 100));
+
+        assert_eq!(result, SearchResult::at_most(expected));
+    }
+
+    #[test]
+    fn test_compute_next_prefix() {
+        use super::compute_next_prefix;
+
+        // Basic cases
+        assert_eq!(compute_next_prefix("foo"), Some("fop".to_string()));
+        assert_eq!(compute_next_prefix("abc"), Some("abd".to_string()));
+        assert_eq!(compute_next_prefix("a"), Some("b".to_string()));
+        assert_eq!(compute_next_prefix("z"), Some("{".to_string())); // 'z' + 1 = '{'
+
+        // Edge case: prefix with 'z' at the end
+        assert_eq!(compute_next_prefix("abz"), Some("ab{".to_string()));
+
+        // Edge case with tilde (~) which is 0x7E
+        assert_eq!(compute_next_prefix("ab~"), Some("ab\x7f".to_string()));
+
+        // Empty prefix
+        assert_eq!(compute_next_prefix(""), None);
+
+        // Non-ASCII: works correctly by incrementing Unicode code points
+        // é (U+00E9) -> ê (U+00EA)
+        assert_eq!(compute_next_prefix("café"), Some("cafê".to_string()));
+        // 中 (U+4E2D) -> 丮 (U+4E2E)
+        assert_eq!(compute_next_prefix("abc中"), Some("abc丮".to_string()));
+        // ÿ (U+00FF) -> Ā (U+0100) - crosses byte boundary but works
+        assert_eq!(compute_next_prefix("cafÿ"), Some("cafĀ".to_string()));
+
+        // Edge case: character just before surrogate range
+        // U+D7FF -> U+E000 (skips surrogate range U+D800-U+DFFF)
+        assert_eq!(
+            compute_next_prefix("a\u{D7FF}"),
+            Some("a\u{E000}".to_string())
+        );
+
+        // Edge case: max Unicode character U+10FFFF, falls back to previous char
+        assert_eq!(compute_next_prefix("ab\u{10FFFF}"), Some("ac".to_string()));
+        // All max characters
+        assert_eq!(compute_next_prefix("\u{10FFFF}\u{10FFFF}"), None);
+    }
 }
diff --git a/rust/lance-index/src/traits.rs b/rust/lance-index/src/traits.rs
index 305ec6892c5..5cb5c830666 100644
--- a/rust/lance-index/src/traits.rs
+++ b/rust/lance-index/src/traits.rs
@@ -1,15 +1,9 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::sync::Arc;
+use lance_core::Result;
 
-use async_trait::async_trait;
-use datafusion::execution::SendableRecordBatchStream;
-use lance_core::{Error, Result};
-
-use crate::{IndexParams, IndexType, optimize::OptimizeOptions};
 use lance_table::format::IndexMetadata;
-use uuid::Uuid;
 
 /// A set of criteria used to filter potential indices to use for a query
 #[derive(Debug, Default)]
@@ -58,6 +52,32 @@ impl<'a> IndexCriteria<'a> {
 #[deprecated(since = "0.39.0", note = "Use IndexCriteria instead")]
 pub type ScalarIndexCriteria<'a> = IndexCriteria<'a>;
 
+/// Options for prewarming an inverted index.
+#[non_exhaustive]
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct FtsPrewarmOptions {
+    /// If true, prewarm positions along with posting lists.
+    pub with_position: bool,
+}
+
+impl FtsPrewarmOptions {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn with_position(mut self, with_position: bool) -> Self {
+        self.with_position = with_position;
+        self
+    }
+}
+
+/// Options for prewarming an index.
+#[non_exhaustive]
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum PrewarmOptions {
+    Fts(FtsPrewarmOptions),
+}
+
 /// Additional information about an index
 ///
 /// Note that a single index might consist of multiple segments.  Each segment has its own
@@ -78,6 +98,13 @@ pub trait IndexDescription: Send + Sync {
     /// IndexMetadata for each segment of the index.
     fn metadata(&self) -> &[IndexMetadata];
 
+    /// Returns the physical index segments that make up this logical index.
+    ///
+    /// This is an alias for [`Self::metadata`] with a less ambiguous name.
+    fn segments(&self) -> &[IndexMetadata] {
+        self.metadata()
+    }
+
     /// Returns the index type URL
     ///
     /// This is extracted from the type url of the index details
@@ -115,171 +142,10 @@ pub trait IndexDescription: Send + Sync {
     /// plugin.  As a result, this method may fail if there is no plugin
     /// available for the index.
     fn details(&self) -> Result<String>;
-}
-
-// Extends Lance Dataset with secondary index.
-#[async_trait]
-pub trait DatasetIndexExt {
-    type IndexBuilder<'a>
-    where
-        Self: 'a;
-
-    /// Create a builder for creating an index on columns.
-    ///
-    /// This returns a builder that can be configured with additional options
-    /// like `name()`, `replace()`, and `train()` before awaiting to execute.
-    ///
-    /// # Parameters
-    /// - `columns`: the columns to build the indices on.
-    /// - `index_type`: specify [`IndexType`].
-    /// - `params`: index parameters.
-    fn create_index_builder<'a>(
-        &'a mut self,
-        columns: &'a [&'a str],
-        index_type: IndexType,
-        params: &'a dyn IndexParams,
-    ) -> Self::IndexBuilder<'a>;
-
-    /// Create indices on columns.
-    ///
-    /// Upon finish, a new dataset version is generated.
-    ///
-    /// Parameters:
-    ///
-    ///  - `columns`: the columns to build the indices on.
-    ///  - `index_type`: specify [`IndexType`].
-    ///  - `name`: optional index name. Must be unique in the dataset.
-    ///            if not provided, it will auto-generate one.
-    ///  - `params`: index parameters.
-    ///  - `replace`: replace the existing index if it exists.
-    ///
-    /// Returns the metadata of the created index.
-    async fn create_index(
-        &mut self,
-        columns: &[&str],
-        index_type: IndexType,
-        name: Option<String>,
-        params: &dyn IndexParams,
-        replace: bool,
-    ) -> Result<IndexMetadata>;
-
-    /// Drop indices by name.
-    ///
-    /// Upon finish, a new dataset version is generated.
-    ///
-    /// Parameters:
-    ///
-    /// - `name`: the name of the index to drop.
-    async fn drop_index(&mut self, name: &str) -> Result<()>;
-
-    /// Prewarm an index by name.
-    ///
-    /// This will load the index into memory and cache it.
-    ///
-    /// Generally, this should only be called when it is known the entire index will
-    /// fit into the index cache.
-    ///
-    /// This is a hint that is not enforced by all indices today.  Some indices may choose
-    /// to ignore this hint.
-    async fn prewarm_index(&self, name: &str) -> Result<()>;
-
-    /// Read all indices of this Dataset version.
-    ///
-    /// The indices are lazy loaded and cached in memory within the `Dataset` instance.
-    /// The cache is invalidated when the dataset version (Manifest) is changed.
-    async fn load_indices(&self) -> Result<Arc<Vec<IndexMetadata>>>;
-
-    /// Loads all the indies of a given UUID.
-    ///
-    /// Note that it is possible to have multiple indices with the same UUID,
-    /// as they are the deltas of the same index.
-    async fn load_index(&self, uuid: &str) -> Result<Option<IndexMetadata>> {
-        self.load_indices().await.map(|indices| {
-            indices
-                .iter()
-                .find(|idx| idx.uuid.to_string() == uuid)
-                .cloned()
-        })
-    }
-
-    /// Loads a specific index with the given index name
-    ///
-    /// Returns
-    /// -------
-    /// - `Ok(indices)`: if the index exists, returns the index.
-    /// - `Ok(vec![])`: if the index does not exist.
-    /// - `Err(e)`: if there is an error loading indices.
-    ///
-    async fn load_indices_by_name(&self, name: &str) -> Result<Vec<IndexMetadata>> {
-        self.load_indices().await.map(|indices| {
-            indices
-                .iter()
-                .filter(|idx| idx.name == name)
-                .cloned()
-                .collect()
-        })
-    }
 
-    /// Loads a specific index with the given index name.
-    /// This function only works for indices that are unique.
-    /// If there are multiple indices sharing the same name, please use [`Self::load_indices_by_name`]
-    ///
-    /// Returns
-    /// -------
-    /// - `Ok(Some(index))`: if the index exists, returns the index.
-    /// - `Ok(None)`: if the index does not exist.
-    /// - `Err(e)`: Index error if there are multiple indexes sharing the same name.
-    ///
-    async fn load_index_by_name(&self, name: &str) -> Result<Option<IndexMetadata>> {
-        let indices = self.load_indices_by_name(name).await?;
-        if indices.is_empty() {
-            Ok(None)
-        } else if indices.len() == 1 {
-            Ok(Some(indices[0].clone()))
-        } else {
-            Err(Error::index(format!(
-                "Found multiple indices of the same name: {:?}, please use load_indices_by_name",
-                indices.iter().map(|idx| &idx.name).collect::<Vec<_>>()
-            )))
-        }
-    }
-
-    /// Describes indexes in a dataset
-    ///
-    /// This method should only access the index metadata and should not load the index into memory.
+    /// Returns the total size in bytes of all files across all segments.
     ///
-    /// More detailed information may be available from `index_statistics` but that will require
-    /// loading the index into memory.
-    async fn describe_indices<'a, 'b>(
-        &'a self,
-        criteria: Option<IndexCriteria<'b>>,
-    ) -> Result<Vec<Arc<dyn IndexDescription>>>;
-
-    /// Loads a specific index with the given index name.
-    async fn load_scalar_index<'a, 'b>(
-        &'a self,
-        criteria: IndexCriteria<'b>,
-    ) -> Result<Option<IndexMetadata>>;
-
-    /// Optimize indices.
-    async fn optimize_indices(&mut self, options: &OptimizeOptions) -> Result<()>;
-
-    /// Find index with a given index_name and return its serialized statistics.
-    ///
-    /// If the index does not exist, return Error.
-    async fn index_statistics(&self, index_name: &str) -> Result<String>;
-
-    async fn commit_existing_index(
-        &mut self,
-        index_name: &str,
-        column: &str,
-        index_id: Uuid,
-    ) -> Result<()>;
-
-    async fn read_index_partition(
-        &self,
-        index_name: &str,
-        partition_id: usize,
-        with_vector: bool,
-    ) -> Result<SendableRecordBatchStream>;
+    /// Returns `None` if file size information is not available for any segment
+    /// (for backward compatibility with indices created before file tracking was added).
+    fn total_size_bytes(&self) -> Option<u64>;
 }
diff --git a/rust/lance-index/src/vector/bq/storage.rs b/rust/lance-index/src/vector/bq/storage.rs
index c47dd0211eb..38c194ce66b 100644
--- a/rust/lance-index/src/vector/bq/storage.rs
+++ b/rust/lance-index/src/vector/bq/storage.rs
@@ -348,17 +348,8 @@ impl DistCalculator for RabitDistCalculator<'_> {
         let id = id as usize;
         let code_len = self.dim * (self.num_bits as usize) / u8::BITS as usize;
         let num_vectors = self.codes.len() / code_len;
-        let code = get_rq_code(self.codes, id, num_vectors, code_len);
-        let dist = code
-            .zip(self.dist_table.chunks_exact(SEGMENT_NUM_CODES).tuples())
-            .map(|(code_byte, (dist_table, next_dist_table))| {
-                // code is a bit vector, we iterate over 8 bits at a time,
-                // every 4 bits is a sub-vector, we need to extract the bits
-                let current_code = (code_byte & 0x0F) as usize;
-                let next_code = (code_byte >> 4) as usize;
-                dist_table[current_code] + next_dist_table[next_code]
-            })
-            .sum::<f32>();
+        let dist =
+            compute_single_rq_distance(self.codes, id, num_vectors, code_len, &self.dist_table);
 
         // distance between quantized vector and query vector
         let dist_vq_qr = (2.0 * dist - self.sum_q) / self.sqrt_d;
@@ -799,6 +790,60 @@ impl QuantizerStorage for RabitQuantizationStorage {
     }
 }
 
+/// Compute the raw distance for a single vector without allocating.
+///
+/// Fuses code extraction from the packed layout with distance accumulation
+/// in a single pass, avoiding the intermediate `Vec` allocation that
+/// `get_rq_code` + iterator would require.
+#[inline]
+fn compute_single_rq_distance(
+    codes: &[u8],
+    id: usize,
+    num_vectors: usize,
+    num_code_bytes: usize,
+    dist_table: &[f32],
+) -> f32 {
+    let remainder = num_vectors % BATCH_SIZE;
+    let mut dist_table_iter = dist_table.chunks_exact(SEGMENT_NUM_CODES).tuples();
+
+    if id < num_vectors - remainder {
+        let batch_codes = &codes[id / BATCH_SIZE * BATCH_SIZE * num_code_bytes
+            ..(id / BATCH_SIZE + 1) * BATCH_SIZE * num_code_bytes];
+
+        let id_in_batch = id % BATCH_SIZE;
+        let idx = PERM0_INVERSE[id_in_batch % 16];
+        let is_lower = id_in_batch < 16;
+
+        let mut dist = 0.0f32;
+        for block in batch_codes.chunks_exact(BATCH_SIZE) {
+            let code_byte = if is_lower {
+                (block[idx] & 0xF) | (block[idx + 16] << 4)
+            } else {
+                (block[idx] >> 4) | (block[idx + 16] & 0xF0)
+            };
+            if let Some((current_dt, next_dt)) = dist_table_iter.next() {
+                let current_code = (code_byte & 0x0F) as usize;
+                let next_code = (code_byte >> 4) as usize;
+                dist += current_dt[current_code] + next_dt[next_code];
+            }
+        }
+        dist
+    } else {
+        let offset_id = id - (num_vectors - remainder);
+        let remainder_codes = &codes[(num_vectors - remainder) * num_code_bytes..];
+
+        let mut dist = 0.0f32;
+        for &code_byte in remainder_codes.iter().skip(offset_id).step_by(remainder) {
+            if let Some((current_dt, next_dt)) = dist_table_iter.next() {
+                let current_code = (code_byte & 0x0F) as usize;
+                let next_code = (code_byte >> 4) as usize;
+                dist += current_dt[current_code] + next_dt[next_code];
+            }
+        }
+        dist
+    }
+}
+
 #[inline]
 fn get_rq_code(
     codes: &[u8],
diff --git a/rust/lance-index/src/vector/distributed/index_merger.rs b/rust/lance-index/src/vector/distributed/index_merger.rs
index 378d5bb1bb1..46d819379db 100755
--- a/rust/lance-index/src/vector/distributed/index_merger.rs
+++ b/rust/lance-index/src/vector/distributed/index_merger.rs
@@ -3,22 +3,26 @@
 
 //! Index merging mechanisms for distributed vector index building
 
+use crate::progress::IndexBuildProgress;
 use crate::vector::shared::partition_merger::{
     SupportedIvfIndexType, write_unified_ivf_and_index_metadata,
 };
 use arrow::{compute::concat_batches, datatypes::Float32Type};
 use arrow_array::cast::AsArray;
 use arrow_array::types::UInt8Type;
-use arrow_array::{Array, FixedSizeListArray, RecordBatch, UInt64Array};
+use arrow_array::{Array, FixedSizeListArray, RecordBatch};
 use futures::StreamExt as _;
 use lance_arrow::{FixedSizeListArrayExt, RecordBatchExt};
-use lance_core::utils::address::RowAddress;
 use lance_core::{Error, ROW_ID_FIELD, Result};
 use std::ops::Range;
 use std::sync::Arc;
 
 use crate::IndexMetadata as IndexMetaSchema;
 use crate::pb;
+use crate::vector::bq::storage::{
+    RABIT_CODE_COLUMN, RABIT_METADATA_KEY, RabitQuantizationMetadata, pack_codes,
+};
+use crate::vector::bq::transform::{ADD_FACTORS_FIELD, SCALE_FACTORS_FIELD};
 use crate::vector::flat::index::FlatMetadata;
 use crate::vector::ivf::storage::{IVF_METADATA_KEY, IvfModel as IvfStorageModel};
 use crate::vector::pq::storage::{PQ_METADATA_KEY, ProductQuantizationMetadata, transpose};
@@ -30,6 +34,7 @@ use crate::{INDEX_AUXILIARY_FILE_NAME, INDEX_METADATA_SCHEMA_KEY};
 use arrow_schema::{DataType, Field, Schema as ArrowSchema};
 use bytes::Bytes;
 use lance_core::datatypes::Schema as LanceSchema;
+use lance_encoding::version::LanceFileVersion;
 use lance_file::reader::{FileReader as V2Reader, FileReaderOptions as V2ReaderOptions};
 use lance_file::writer::{FileWriter as V2Writer, FileWriter, FileWriterOptions};
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
@@ -171,14 +176,16 @@ pub async fn init_writer_for_flat(
     object_store: &lance_io::object_store::ObjectStore,
     aux_out: &object_store::path::Path,
     d0: usize,
+    item_type: &DataType,
     dt: DistanceType,
+    format_version: LanceFileVersion,
 ) -> Result<FileWriter> {
     let arrow_schema = ArrowSchema::new(vec![
         (*ROW_ID_FIELD).clone(),
         Field::new(
             crate::vector::flat::storage::FLAT_COLUMN,
             DataType::FixedSizeList(
-                Arc::new(Field::new("item", DataType::Float32, true)),
+                Arc::new(Field::new("item", item_type.clone(), true)),
                 d0 as i32,
             ),
             true,
@@ -188,7 +195,10 @@ pub async fn init_writer_for_flat(
     let mut w = FileWriter::try_new(
         writer,
         LanceSchema::try_from(&arrow_schema)?,
-        FileWriterOptions::default(),
+        FileWriterOptions {
+            format_version: Some(format_version),
+            ..Default::default()
+        },
     )?;
     let meta_json = serde_json::to_string(&FlatMetadata { dim: d0 })?;
     init_writer_for_storage(&mut w, dt, &meta_json, "")?;
@@ -204,6 +214,7 @@ pub async fn init_writer_for_pq(
     aux_out: &object_store::path::Path,
     dt: DistanceType,
     pm: &ProductQuantizationMetadata,
+    format_version: LanceFileVersion,
 ) -> Result<FileWriter> {
     let num_bytes = if pm.nbits == 4 {
         pm.num_sub_vectors / 2
@@ -225,7 +236,10 @@ pub async fn init_writer_for_pq(
     let mut w = FileWriter::try_new(
         writer,
         LanceSchema::try_from(&arrow_schema)?,
-        FileWriterOptions::default(),
+        FileWriterOptions {
+            format_version: Some(format_version),
+            ..Default::default()
+        },
     )?;
     let mut pm_init = pm.clone();
     let cb = pm_init
@@ -247,6 +261,7 @@ pub async fn init_writer_for_sq(
     aux_out: &object_store::path::Path,
     dt: DistanceType,
     sq_meta: &ScalarQuantizationMetadata,
+    format_version: LanceFileVersion,
 ) -> Result<FileWriter> {
     let d0 = sq_meta.dim;
     let arrow_schema = ArrowSchema::new(vec![
@@ -264,13 +279,59 @@ pub async fn init_writer_for_sq(
     let mut w = FileWriter::try_new(
         writer,
         LanceSchema::try_from(&arrow_schema)?,
-        FileWriterOptions::default(),
+        FileWriterOptions {
+            format_version: Some(format_version),
+            ..Default::default()
+        },
     )?;
     let meta_json = serde_json::to_string(sq_meta)?;
     init_writer_for_storage(&mut w, dt, &meta_json, SQ_METADATA_KEY)?;
     Ok(w)
 }
 
+/// Create and initialize a unified writer for RQ storage.
+pub async fn init_writer_for_rq(
+    object_store: &lance_io::object_store::ObjectStore,
+    aux_out: &object_store::path::Path,
+    dt: DistanceType,
+    rq_meta: &RabitQuantizationMetadata,
+    format_version: LanceFileVersion,
+) -> Result<FileWriter> {
+    let num_bytes = (rq_meta.code_dim as usize).div_ceil(u8::BITS as usize);
+    let arrow_schema = ArrowSchema::new(vec![
+        (*ROW_ID_FIELD).clone(),
+        Field::new(
+            RABIT_CODE_COLUMN,
+            DataType::FixedSizeList(
+                Arc::new(Field::new("item", DataType::UInt8, true)),
+                num_bytes as i32,
+            ),
+            true,
+        ),
+        ADD_FACTORS_FIELD.clone(),
+        SCALE_FACTORS_FIELD.clone(),
+    ]);
+    let writer = object_store.create(aux_out).await?;
+    let mut w = FileWriter::try_new(
+        writer,
+        LanceSchema::try_from(&arrow_schema)?,
+        FileWriterOptions {
+            format_version: Some(format_version),
+            ..Default::default()
+        },
+    )?;
+
+    let mut rq_meta_init = rq_meta.clone();
+    rq_meta_init.packed = true;
+    if let Some(extra_metadata) = rq_meta_init.extra_metadata()? {
+        let pos = w.add_global_buffer(extra_metadata).await?;
+        rq_meta_init.set_buffer_index(pos);
+    }
+    let rq_meta_json = serde_json::to_string(&rq_meta_init)?;
+    init_writer_for_storage(&mut w, dt, &rq_meta_json, RABIT_METADATA_KEY)?;
+    Ok(w)
+}
+
 /// Stream and write a range of rows from reader into writer.
 ///
 /// The caller is responsible for ensuring that `range` corresponds to a
@@ -339,6 +400,41 @@ async fn write_partition_rows_pq_transposed(
     Ok(())
 }
 
+/// Pack the RQ code column for a batch and write it to the unified writer.
+///
+/// This helper assumes `batch` contains a contiguous range of rows for a single
+/// IVF partition and that the shard batch stores row-major RQ codes.
+async fn write_partition_rows_rq_packed(w: &mut FileWriter, mut batch: RecordBatch) -> Result<()> {
+    let num_rows = batch.num_rows();
+    if num_rows == 0 {
+        return Ok(());
+    }
+
+    let rq_col = batch.column_by_name(RABIT_CODE_COLUMN).ok_or_else(|| {
+        Error::index(format!(
+            "RQ column {} missing in auxiliary shard",
+            RABIT_CODE_COLUMN
+        ))
+    })?;
+    let rq_fsl = rq_col.as_fixed_size_list_opt().ok_or_else(|| {
+        Error::index(format!(
+            "RQ column {} is not a FixedSizeList in auxiliary shard, got {}",
+            RABIT_CODE_COLUMN,
+            rq_col.data_type(),
+        ))
+    })?;
+    let packed_codes = pack_codes(rq_fsl);
+    batch = batch.replace_column_by_name(RABIT_CODE_COLUMN, Arc::new(packed_codes))?;
+
+    let batch_size: usize = 10_240;
+    for offset in (0..num_rows).step_by(batch_size) {
+        let len = std::cmp::min(batch_size, num_rows - offset);
+        let slice = batch.slice(offset, len);
+        w.write_batch(&slice).await?;
+    }
+    Ok(())
+}
+
 /// Detect and return supported index type from reader and schema.
 ///
 /// This is a lightweight wrapper around SupportedIndexType::detect to keep
@@ -350,129 +446,12 @@ fn detect_supported_index_type(
     SupportedIvfIndexType::detect_from_reader_and_schema(reader, schema)
 }
 
-/// Decode the fragment id from an encoded row id.
-///
-/// Row ids are stored as a 64-bit [RowAddress] where the upper 32 bits encode
-/// the fragment id and the lower 32 bits encode the row offset.
-fn decode_fragment_id_from_row_id(row_id_u64: u64) -> u32 {
-    let addr = RowAddress::new_from_u64(row_id_u64);
-    addr.fragment_id()
-}
-
-/// Compute a content-derived shard sort key for a partial auxiliary file.
-///
-/// The key is `(min_fragment_id, min_row_id, parent_dir_name)` where:
-/// - `min_fragment_id` is the minimum fragment id observed among the first row
-///   of each non-empty IVF partition.
-/// - `min_row_id` is the minimum encoded row id (as `u64`) among the same
-///   representative rows.
-/// - `parent_dir_name` is the `partial_*` directory name extracted from
-///   `aux_path` and used only as a final lexicographic tie-breaker.
-///
-/// This helper reads exactly one row per non-empty partition (the first row in
-/// that partition) and never scans entire shards.
-async fn compute_shard_content_key(
-    sched: &std::sync::Arc<ScanScheduler>,
-    _store: &lance_io::object_store::ObjectStore,
-    aux_path: &object_store::path::Path,
-) -> Result<(u32, u64, String)> {
-    let fh = sched
-        .open_file(aux_path, &CachedFileSize::unknown())
-        .await?;
-    let reader = V2Reader::try_open(
-        fh,
-        None,
-        Arc::default(),
-        &lance_core::cache::LanceCache::no_cache(),
-        V2ReaderOptions::default(),
-    )
-    .await?;
-
-    // Locate the ROW_ID_FIELD column to decode fragment / row ids.
-    let schema_arrow: ArrowSchema = reader.schema().as_ref().into();
-    let row_id_idx = schema_arrow
-        .fields
-        .iter()
-        .position(|f| f.name() == ROW_ID_FIELD.name())
-        .ok_or_else(|| Error::index("ROW_ID_FIELD missing in auxiliary shard".to_string()))?;
-
-    // Read IVF lengths from the global buffer.
-    let ivf_idx: u32 = reader
-        .metadata()
-        .file_schema
-        .metadata
-        .get(IVF_METADATA_KEY)
-        .ok_or_else(|| Error::index("IVF meta missing".to_string()))?
-        .parse()
-        .map_err(|_| Error::index("IVF index parse error".to_string()))?;
-    let bytes = reader.read_global_buffer(ivf_idx).await?;
-    let pb_ivf: pb::Ivf = prost::Message::decode(bytes)?;
-    let lengths = pb_ivf.lengths;
-
-    let mut min_fragment_id: Option<u32> = None;
-    let mut min_row_id: Option<u64> = None;
-
-    let mut offset: usize = 0;
-    for len in &lengths {
-        let part_len = *len as usize;
-        if part_len > 0 {
-            let mut stream = reader.read_stream(
-                lance_io::ReadBatchParams::Range(offset..offset + 1),
-                u32::MAX,
-                4,
-                lance_encoding::decoder::FilterExpression::no_filter(),
-            )?;
-            if let Some(batch_res) = stream.next().await {
-                let batch = batch_res?;
-                if batch.num_rows() > 0 {
-                    let arr = batch
-                        .column(row_id_idx)
-                        .as_any()
-                        .downcast_ref::<UInt64Array>()
-                        .ok_or_else(|| {
-                            Error::index(
-                                "ROW_ID_FIELD must be a UInt64 column in auxiliary shard"
-                                    .to_string(),
-                            )
-                        })?;
-                    let row_id_val = arr.value(0);
-                    let frag_id = decode_fragment_id_from_row_id(row_id_val);
-                    min_fragment_id = Some(match min_fragment_id {
-                        Some(cur) => cur.min(frag_id),
-                        None => frag_id,
-                    });
-                    min_row_id = Some(match min_row_id {
-                        Some(cur) => cur.min(row_id_val),
-                        None => row_id_val,
-                    });
-                }
-            }
-        }
-        offset += part_len;
-    }
-
-    let min_fragment_id = min_fragment_id.unwrap_or(RowAddress::TOMBSTONE_FRAG);
-    let min_row_id = min_row_id.unwrap_or(RowAddress::TOMBSTONE_ROW);
-
-    let parent_name = {
-        let parts: Vec<_> = aux_path.parts().collect();
-        if parts.len() >= 2 {
-            parts[parts.len() - 2].as_ref().to_string()
-        } else {
-            String::new()
-        }
-    };
-
-    Ok((min_fragment_id, min_row_id, parent_name))
-}
-
 #[derive(Debug)]
 struct ShardInfo {
     reader: Arc<V2Reader>,
     lengths: Vec<u32>,
     partition_offsets: Vec<usize>,
     total_rows: usize,
-    sort_key: (u32, u64, String),
 }
 
 #[derive(Debug)]
@@ -707,64 +686,41 @@ async fn read_shard_window_partitions(
     Ok(per_partition_batches)
 }
 
-/// Merge all partial_* vector index auxiliary files under `index_dir/{uuid}/partial_*/auxiliary.idx`
-/// into `index_dir/{uuid}/auxiliary.idx`.
+/// Merge the selected segment auxiliary files into `target_dir`.
+///
+/// This is the storage merge kernel for vector segment build. Callers choose
+/// which segments belong to one built segment and pass the
+/// corresponding auxiliary files here. The merge writes one unified
+/// `auxiliary.idx` into `target_dir`.
 ///
-/// Supports IVF_FLAT, IVF_PQ, IVF_SQ, IVF_HNSW_FLAT, IVF_HNSW_PQ, IVF_HNSW_SQ storage types.
-/// For PQ and SQ, this assumes all partial indices share the same quantizer/codebook
-/// and distance type; it will reuse the first encountered metadata.
+/// Supports IVF_FLAT, IVF_PQ, IVF_SQ, IVF_HNSW_FLAT, IVF_HNSW_PQ, and
+/// IVF_HNSW_SQ storage types. For PQ and SQ, this assumes all selected source
+/// segments share the same quantizer/codebook and distance type; it reuses the
+/// first encountered metadata.
 pub async fn merge_partial_vector_auxiliary_files(
     object_store: &lance_io::object_store::ObjectStore,
-    index_dir: &object_store::path::Path,
+    aux_paths: &[object_store::path::Path],
+    target_dir: &object_store::path::Path,
+    progress: Arc<dyn IndexBuildProgress>,
 ) -> Result<()> {
-    let mut aux_paths: Vec<object_store::path::Path> = Vec::new();
-    let mut stream = object_store.list(Some(index_dir.clone()));
-    while let Some(item) = stream.next().await {
-        if let Ok(meta) = item
-            && let Some(fname) = meta.location.filename()
-            && fname == INDEX_AUXILIARY_FILE_NAME
-        {
-            // Check parent dir name starts with partial_
-            let parts: Vec<_> = meta.location.parts().collect();
-            if parts.len() >= 2 {
-                let pname = parts[parts.len() - 2].as_ref();
-                if pname.starts_with("partial_") {
-                    aux_paths.push(meta.location.clone());
-                }
-            }
-        }
-    }
-
     if aux_paths.is_empty() {
-        // If a unified auxiliary file already exists at the root, no merge is required.
-        let aux_out = index_dir.child(INDEX_AUXILIARY_FILE_NAME);
-        if object_store.exists(&aux_out).await.unwrap_or(false) {
-            log::warn!(
-                "No partial_* auxiliary files found under index dir: {}, but unified auxiliary file already exists; skipping merge",
-                index_dir
-            );
-            return Ok(());
-        }
-        // For certain index types (e.g., FLAT/HNSW-only) the merge may be a no-op in distributed setups
-        // where shards were committed directly. In such cases, proceed without error to avoid blocking
-        // index manifest merge. PQ/SQ variants still require merging artifacts and will be handled by
-        // downstream open logic if missing.
-        log::warn!(
-            "No partial_* auxiliary files found under index dir: {}; proceeding without merge for index types that do not require auxiliary shards",
-            index_dir
-        );
-        return Ok(());
+        return Err(Error::index(
+            "No partial auxiliary files were selected for merge".to_string(),
+        ));
     }
 
     // Prepare IVF model and storage metadata aggregation
     let mut distance_type: Option<DistanceType> = None;
     let mut pq_meta: Option<ProductQuantizationMetadata> = None;
     let mut sq_meta: Option<ScalarQuantizationMetadata> = None;
+    let mut rq_meta: Option<RabitQuantizationMetadata> = None;
     let mut dim: Option<usize> = None;
     let mut detected_index_type: Option<SupportedIvfIndexType> = None;
+    // Inherit file format version from the first shard (set on first iteration)
+    let mut format_version: Option<LanceFileVersion> = None;
 
     // Prepare output path; we'll create writer once when we know schema
-    let aux_out = index_dir.child(INDEX_AUXILIARY_FILE_NAME);
+    let aux_out = target_dir.child(INDEX_AUXILIARY_FILE_NAME);
 
     // We'll delay creating the V2 writer until we know the vector schema (dim and quantizer type)
     let mut v2w_opt: Option<V2Writer> = None;
@@ -775,20 +731,6 @@ pub async fn merge_partial_vector_auxiliary_files(
         SchedulerConfig::max_bandwidth(object_store),
     );
 
-    // Compute content-derived sort keys for each shard once while opening the
-    // auxiliary readers. These keys will be reused both for ordering the
-    // enumeration of shards and for per-partition writes.
-    let mut shard_keys: Vec<(object_store::path::Path, (u32, u64, String))> =
-        Vec::with_capacity(aux_paths.len());
-    for aux in aux_paths.into_iter() {
-        let key = compute_shard_content_key(&sched, object_store, &aux).await?;
-        shard_keys.push((aux, key));
-    }
-
-    // Sort shards by their content-derived keys (min_fragment_id, min_row_id,
-    // parent_dir_name) to detach from underlying listing order.
-    shard_keys.sort_by(|a, b| a.1.cmp(&b.1));
-
     // Track IVF partition count consistency and accumulate lengths per partition
     let mut nlist_opt: Option<usize> = None;
     let mut accumulated_lengths: Vec<u32> = Vec::new();
@@ -798,8 +740,16 @@ pub async fn merge_partial_vector_auxiliary_files(
     // This avoids reopening each shard file for every partition during merge.
     let mut shard_infos: Vec<ShardInfo> = Vec::new();
 
+    progress
+        .stage_start(
+            "read_shard_metadata",
+            Some(aux_paths.len() as u64),
+            "shards",
+        )
+        .await?;
+
     // Iterate over each shard auxiliary file and merge its metadata and collect lengths
-    for (aux, key) in &shard_keys {
+    for (idx, aux) in aux_paths.iter().enumerate() {
         let fh = sched.open_file(aux, &CachedFileSize::unknown()).await?;
         let reader = V2Reader::try_open(
             fh,
@@ -811,6 +761,11 @@ pub async fn merge_partial_vector_auxiliary_files(
         .await?;
         let meta = reader.metadata();
 
+        // Inherit format version from the first shard file
+        if format_version.is_none() {
+            format_version = Some(meta.version());
+        }
+
         // Read distance type
         let dt = meta
             .file_schema
@@ -866,6 +821,7 @@ pub async fn merge_partial_vector_auxiliary_files(
                         "IVF_FLAT" => SupportedIvfIndexType::IvfFlat,
                         "IVF_PQ" => SupportedIvfIndexType::IvfPq,
                         "IVF_SQ" => SupportedIvfIndexType::IvfSq,
+                        "IVF_RQ" => SupportedIvfIndexType::IvfRq,
                         "IVF_HNSW_FLAT" => SupportedIvfIndexType::IvfHnswFlat,
                         "IVF_HNSW_PQ" => SupportedIvfIndexType::IvfHnswPq,
                         "IVF_HNSW_SQ" => SupportedIvfIndexType::IvfHnswSq,
@@ -920,6 +876,10 @@ pub async fn merge_partial_vector_auxiliary_files(
         // Handle logic based on detected index type
         let idx_type = detected_index_type
             .ok_or_else(|| Error::index("Unable to detect index type".to_string()))?;
+
+        // Compute format version once; defaults to V2_0 if no shards processed yet
+        let fv = format_version.unwrap_or(LanceFileVersion::V2_0);
+
         match idx_type {
             SupportedIvfIndexType::IvfSq => {
                 // Handle Scalar Quantization (SQ) storage for IVF_SQ
@@ -973,7 +933,89 @@ pub async fn merge_partial_vector_auxiliary_files(
                     sq_meta = Some(sq_meta_parsed.clone());
                 }
                 if v2w_opt.is_none() {
-                    let w = init_writer_for_sq(object_store, &aux_out, dt, &sq_meta_parsed).await?;
+                    let w =
+                        init_writer_for_sq(object_store, &aux_out, dt, &sq_meta_parsed, fv).await?;
+                    v2w_opt = Some(w);
+                }
+            }
+            SupportedIvfIndexType::IvfRq => {
+                let rq_json = if let Some(rq_json) = reader
+                    .metadata()
+                    .file_schema
+                    .metadata
+                    .get(RABIT_METADATA_KEY)
+                {
+                    rq_json.clone()
+                } else if let Some(storage_meta_json) = reader
+                    .metadata()
+                    .file_schema
+                    .metadata
+                    .get(STORAGE_METADATA_KEY)
+                {
+                    let storage_metadata_vec: Vec<String> = serde_json::from_str(storage_meta_json)
+                        .map_err(|e| {
+                            Error::index(format!("Failed to parse storage metadata: {}", e))
+                        })?;
+                    if let Some(first_meta) = storage_metadata_vec.first() {
+                        if let Ok(_rq_meta) =
+                            serde_json::from_str::<RabitQuantizationMetadata>(first_meta)
+                        {
+                            first_meta.clone()
+                        } else {
+                            return Err(Error::index(
+                                "RQ metadata missing in storage metadata".to_string(),
+                            ));
+                        }
+                    } else {
+                        return Err(Error::index(
+                            "RQ metadata missing in storage metadata".to_string(),
+                        ));
+                    }
+                } else {
+                    return Err(Error::index("RQ metadata missing".to_string()));
+                };
+                let mut rq_meta_parsed: RabitQuantizationMetadata = serde_json::from_str(&rq_json)
+                    .map_err(|e| Error::index(format!("RQ metadata parse error: {}", e)))?;
+                if rq_meta_parsed.rotation_type == crate::vector::bq::RQRotationType::Matrix
+                    && rq_meta_parsed.rotate_mat.is_none()
+                    && let Some(buf_idx) = rq_meta_parsed.buffer_index()
+                {
+                    let rotate_mat_bytes = reader.read_global_buffer(buf_idx).await?;
+                    rq_meta_parsed.parse_buffer(rotate_mat_bytes)?;
+                }
+
+                let d0 = (rq_meta_parsed.code_dim as usize)
+                    .checked_div(rq_meta_parsed.num_bits as usize)
+                    .ok_or_else(|| {
+                        Error::index("Invalid RQ metadata: num_bits is zero".to_string())
+                    })?;
+                dim.get_or_insert(d0);
+                if let Some(dprev) = dim
+                    && dprev != d0
+                {
+                    return Err(Error::index("Dimension mismatch across shards".to_string()));
+                }
+                if let Some(existing_rq) = rq_meta.as_ref()
+                    && (existing_rq.code_dim != rq_meta_parsed.code_dim
+                        || existing_rq.num_bits != rq_meta_parsed.num_bits
+                        || existing_rq.rotation_type != rq_meta_parsed.rotation_type)
+                {
+                    return Err(Error::index(format!(
+                        "Distributed RQ merge: structural mismatch across shards; first(code_dim={}, num_bits={}, rotation_type={:?}), current(code_dim={}, num_bits={}, rotation_type={:?})",
+                        existing_rq.code_dim,
+                        existing_rq.num_bits,
+                        existing_rq.rotation_type,
+                        rq_meta_parsed.code_dim,
+                        rq_meta_parsed.num_bits,
+                        rq_meta_parsed.rotation_type
+                    )));
+                }
+                if rq_meta.is_none() {
+                    rq_meta = Some(rq_meta_parsed.clone());
+                }
+                if v2w_opt.is_none() {
+                    let w =
+                        init_writer_for_rq(object_store, &aux_out, dt, &rq_meta_parsed, fv).await?;
                     v2w_opt = Some(w);
                 }
             }
@@ -1074,7 +1116,8 @@ pub async fn merge_partial_vector_auxiliary_files(
                 if v2w_opt.is_none() {
                     let mut pm_for_unified = pm.clone();
                     pm_for_unified.transposed = true;
-                    let w = init_writer_for_pq(object_store, &aux_out, dt, &pm_for_unified).await?;
+                    let w =
+                        init_writer_for_pq(object_store, &aux_out, dt, &pm_for_unified, fv).await?;
                     v2w_opt = Some(w);
                 }
             }
@@ -1087,9 +1130,13 @@ pub async fn merge_partial_vector_auxiliary_files(
                     .iter()
                     .find(|f| f.name() == crate::vector::flat::storage::FLAT_COLUMN)
                     .ok_or_else(|| Error::index("FLAT column missing".to_string()))?;
-                let d0 = match flat_field.data_type() {
-                    DataType::FixedSizeList(_, sz) => *sz as usize,
-                    _ => 0,
+                let (d0, item_type) = match flat_field.data_type() {
+                    DataType::FixedSizeList(item, sz) => (*sz as usize, item.data_type().clone()),
+                    _ => {
+                        return Err(Error::index(
+                            "FLAT column is not a FixedSizeList in shard schema".to_string(),
+                        ));
+                    }
                 };
                 dim.get_or_insert(d0);
                 if let Some(dprev) = dim
@@ -1098,53 +1145,29 @@ pub async fn merge_partial_vector_auxiliary_files(
                     return Err(Error::index("Dimension mismatch across shards".to_string()));
                 }
                 if v2w_opt.is_none() {
-                    let w = init_writer_for_flat(object_store, &aux_out, d0, dt).await?;
+                    let w = init_writer_for_flat(object_store, &aux_out, d0, &item_type, dt, fv)
+                        .await?;
                     v2w_opt = Some(w);
                 }
             }
             SupportedIvfIndexType::IvfHnswFlat => {
-                // Treat HNSW_FLAT storage the same as FLAT: create schema with ROW_ID + flat vectors
-                // Determine dimension from shard schema (flat column) or fallback to STORAGE_METADATA_KEY
+                // Treat HNSW_FLAT storage the same as FLAT and preserve the actual flat item dtype.
                 let schema_arrow: ArrowSchema = reader.schema().as_ref().into();
-                // Try to find flat column and derive dim
-                let d0 = if let Some(flat_field) = schema_arrow
+                let Some(flat_field) = schema_arrow
                     .fields
                     .iter()
                     .find(|f| f.name() == crate::vector::flat::storage::FLAT_COLUMN)
-                {
-                    match flat_field.data_type() {
-                        DataType::FixedSizeList(_, sz) => *sz as usize,
-                        _ => 0,
-                    }
-                } else {
-                    // Fallback to STORAGE_METADATA_KEY FlatMetadata
-                    if let Some(storage_meta_json) = reader
-                        .metadata()
-                        .file_schema
-                        .metadata
-                        .get(STORAGE_METADATA_KEY)
-                    {
-                        let storage_metadata_vec: Vec<String> =
-                            serde_json::from_str(storage_meta_json).map_err(|e| {
-                                Error::index(format!("Failed to parse storage metadata: {}", e))
-                            })?;
-                        if let Some(first_meta) = storage_metadata_vec.first() {
-                            if let Ok(flat_meta) = serde_json::from_str::<FlatMetadata>(first_meta)
-                            {
-                                flat_meta.dim
-                            } else {
-                                return Err(Error::index(
-                                    "FLAT metadata missing in storage metadata".to_string(),
-                                ));
-                            }
-                        } else {
-                            return Err(Error::index(
-                                "FLAT metadata missing in storage metadata".to_string(),
-                            ));
-                        }
-                    } else {
+                else {
+                    return Err(Error::index(
+                        "FLAT column missing from IVF_HNSW_FLAT shard schema".to_string(),
+                    ));
+                };
+                let (d0, item_type) = match flat_field.data_type() {
+                    DataType::FixedSizeList(item, sz) => (*sz as usize, item.data_type().clone()),
+                    _ => {
                         return Err(Error::index(
-                            "FLAT column missing and no storage metadata".to_string(),
+                            "FLAT column is not a FixedSizeList in IVF_HNSW_FLAT shard schema"
+                                .to_string(),
                         ));
                     }
                 };
@@ -1155,7 +1178,8 @@ pub async fn merge_partial_vector_auxiliary_files(
                     return Err(Error::index("Dimension mismatch across shards".to_string()));
                 }
                 if v2w_opt.is_none() {
-                    let w = init_writer_for_flat(object_store, &aux_out, d0, dt).await?;
+                    let w = init_writer_for_flat(object_store, &aux_out, d0, &item_type, dt, fv)
+                        .await?;
                     v2w_opt = Some(w);
                 }
             }
@@ -1252,7 +1276,8 @@ pub async fn merge_partial_vector_auxiliary_files(
                 if v2w_opt.is_none() {
                     let mut pm_for_unified = pm.clone();
                     pm_for_unified.transposed = true;
-                    let w = init_writer_for_pq(object_store, &aux_out, dt, &pm_for_unified).await?;
+                    let w =
+                        init_writer_for_pq(object_store, &aux_out, dt, &pm_for_unified, fv).await?;
                     v2w_opt = Some(w);
                 }
             }
@@ -1303,7 +1328,8 @@ pub async fn merge_partial_vector_auxiliary_files(
                     sq_meta = Some(sq_meta_parsed.clone());
                 }
                 if v2w_opt.is_none() {
-                    let w = init_writer_for_sq(object_store, &aux_out, dt, &sq_meta_parsed).await?;
+                    let w =
+                        init_writer_for_sq(object_store, &aux_out, dt, &sq_meta_parsed, fv).await?;
                     v2w_opt = Some(w);
                 }
             }
@@ -1328,13 +1354,12 @@ pub async fn merge_partial_vector_auxiliary_files(
             lengths,
             partition_offsets,
             total_rows: running_offset,
-            sort_key: key.clone(),
         });
+        progress
+            .stage_progress("read_shard_metadata", idx as u64 + 1)
+            .await?;
     }
-
-    // Re-sort shard_infos using content-derived keys to decouple per-partition
-    // write ordering from discovery order.
-    shard_infos.sort_by(|a, b| a.sort_key.cmp(&b.sort_key));
+    progress.stage_complete("read_shard_metadata").await?;
 
     // Write rows grouped by partition across all shards to ensure contiguous ranges per partition
 
@@ -1347,6 +1372,15 @@ pub async fn merge_partial_vector_auxiliary_files(
     let idx_type_final = detected_index_type
         .ok_or_else(|| Error::index("Unable to detect index type".to_string()))?;
 
+    let total_rows = accumulated_lengths
+        .iter()
+        .map(|length| *length as u64)
+        .sum::<u64>();
+    progress
+        .stage_start("merge_partitions", Some(total_rows), "rows")
+        .await?;
+    let mut merged_rows = 0u64;
+
     match idx_type_final {
         SupportedIvfIndexType::IvfPq | SupportedIvfIndexType::IvfHnswPq => {
             // For PQ-backed indices, transpose PQ codes while merging partitions
@@ -1376,10 +1410,47 @@ pub async fn merge_partial_vector_auxiliary_files(
                 if let Some(w) = v2w_opt.as_mut() {
                     write_partition_rows_pq_transposed(w, partition_batch).await?;
                 }
+                merged_rows = merged_rows.saturating_add(accumulated_lengths[pid] as u64);
+                progress
+                    .stage_progress("merge_partitions", merged_rows)
+                    .await?;
+            }
+        }
+        SupportedIvfIndexType::IvfRq => {
+            let partition_window_size = *PARTITION_WINDOW_SIZE;
+            let prefetch_window_count = *PARTITION_PREFETCH_WINDOW_COUNT;
+            let mut shard_merge_reader = ShardMergeReader::new(
+                shard_infos,
+                nlist,
+                partition_window_size,
+                prefetch_window_count,
+            );
+
+            while let Some((pid, batches)) = shard_merge_reader.next_partition().await? {
+                if accumulated_lengths[pid] == 0 {
+                    continue;
+                }
+                if batches.is_empty() {
+                    return Err(Error::index(format!(
+                        "No merged batches found for non-empty partition {}",
+                        pid
+                    )));
+                }
+
+                let schema = batches[0].schema();
+                let partition_batch = concat_batches(&schema, batches.iter())?;
+                if let Some(w) = v2w_opt.as_mut() {
+                    write_partition_rows_rq_packed(w, partition_batch).await?;
+                }
+                merged_rows = merged_rows.saturating_add(accumulated_lengths[pid] as u64);
+                progress
+                    .stage_progress("merge_partitions", merged_rows)
+                    .await?;
             }
         }
         _ => {
-            for pid in 0..nlist {
+            for (pid, total_part_len) in accumulated_lengths.iter().copied().enumerate().take(nlist)
+            {
                 for shard in shard_infos.iter() {
                     let part_len = shard.lengths[pid] as usize;
                     if part_len == 0 {
@@ -1391,12 +1462,23 @@ pub async fn merge_partial_vector_auxiliary_files(
                             .await?;
                     }
                 }
+                if total_part_len == 0 {
+                    continue;
+                }
+                merged_rows = merged_rows.saturating_add(total_part_len as u64);
+                progress
+                    .stage_progress("merge_partitions", merged_rows)
+                    .await?;
             }
         }
     }
+    progress.stage_complete("merge_partitions").await?;
 
     // Write unified IVF metadata into global buffer & set schema metadata
     if let Some(w) = v2w_opt.as_mut() {
+        progress
+            .stage_start("write_auxiliary_index", Some(1), "files")
+            .await?;
         let mut ivf_model = if let Some(c) = first_centroids {
             IvfStorageModel::new(c, None)
         } else {
@@ -1408,6 +1490,8 @@ pub async fn merge_partial_vector_auxiliary_files(
         let dt2 = distance_type.ok_or_else(|| Error::index("Distance type missing".to_string()))?;
         write_unified_ivf_and_index_metadata(w, &ivf_model, dt2, idx_type_final).await?;
         w.finish().await?;
+        progress.stage_progress("write_auxiliary_index", 1).await?;
+        progress.stage_complete("write_auxiliary_index").await?;
     } else {
         return Err(Error::index(
             "Failed to initialize unified writer".to_string(),
@@ -1421,13 +1505,14 @@ pub async fn merge_partial_vector_auxiliary_files(
 mod tests {
     use super::*;
 
-    use arrow_array::{FixedSizeListArray, Float32Array, RecordBatch, UInt8Array, UInt64Array};
+    use arrow_array::{
+        FixedSizeListArray, Float32Array, Float64Array, RecordBatch, UInt8Array, UInt64Array,
+    };
     use arrow_schema::Field;
     use bytes::Bytes;
     use futures::StreamExt;
     use lance_arrow::FixedSizeListArrayExt;
     use lance_core::ROW_ID_FIELD;
-    use lance_core::utils::address::RowAddress;
     use lance_file::writer::FileWriterOptions as V2WriterOptions;
     use lance_io::object_store::ObjectStore;
     use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
@@ -1436,6 +1521,13 @@ mod tests {
     use object_store::path::Path;
     use prost::Message;
 
+    use crate::vector::bq::RQRotationType;
+    lance_testing::define_stage_event_progress!(
+        RecordingProgress,
+        IndexBuildProgress,
+        lance_core::Result<()>
+    );
+
     async fn write_flat_partial_aux(
         store: &ObjectStore,
         aux_path: &Path,
@@ -1506,33 +1598,174 @@ mod tests {
         Ok(total_rows)
     }
 
-    #[tokio::test]
-    async fn test_merge_ivf_flat_success_basic() {
-        let object_store = ObjectStore::memory();
-        let index_dir = Path::from("index/uuid");
-
-        let partial0 = index_dir.child("partial_0");
-        let partial1 = index_dir.child("partial_1");
-        let aux0 = partial0.child(INDEX_AUXILIARY_FILE_NAME);
-        let aux1 = partial1.child(INDEX_AUXILIARY_FILE_NAME);
+    async fn write_flat_partial_aux_f64(
+        store: &ObjectStore,
+        aux_path: &Path,
+        dim: i32,
+        lengths: &[u32],
+        base_row_id: u64,
+        distance_type: DistanceType,
+    ) -> Result<usize> {
+        let arrow_schema = ArrowSchema::new(vec![
+            (*ROW_ID_FIELD).clone(),
+            Field::new(
+                crate::vector::flat::storage::FLAT_COLUMN,
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float64, true)), dim),
+                true,
+            ),
+        ]);
 
-        let lengths0 = vec![2_u32, 1_u32];
-        let lengths1 = vec![1_u32, 2_u32];
-        let dim = 2_i32;
+        let writer = store.create(aux_path).await?;
+        let mut v2w = V2Writer::try_new(
+            writer,
+            lance_core::datatypes::Schema::try_from(&arrow_schema)?,
+            V2WriterOptions::default(),
+        )?;
+        v2w.add_schema_metadata(DISTANCE_TYPE_KEY, distance_type.to_string());
 
-        write_flat_partial_aux(&object_store, &aux0, dim, &lengths0, 0, DistanceType::L2)
-            .await
-            .unwrap();
-        write_flat_partial_aux(&object_store, &aux1, dim, &lengths1, 100, DistanceType::L2)
-            .await
-            .unwrap();
+        let ivf_meta = pb::Ivf {
+            centroids: Vec::new(),
+            offsets: Vec::new(),
+            lengths: lengths.to_vec(),
+            centroids_tensor: None,
+            loss: None,
+        };
+        let buf = Bytes::from(ivf_meta.encode_to_vec());
+        let pos = v2w.add_global_buffer(buf).await?;
+        v2w.add_schema_metadata(IVF_METADATA_KEY, pos.to_string());
 
-        merge_partial_vector_auxiliary_files(&object_store, &index_dir)
-            .await
-            .unwrap();
+        let total_rows: usize = lengths.iter().map(|v| *v as usize).sum();
+        let mut row_ids = Vec::with_capacity(total_rows);
+        let mut values = Vec::with_capacity(total_rows * dim as usize);
 
-        let aux_out = index_dir.child(INDEX_AUXILIARY_FILE_NAME);
-        assert!(object_store.exists(&aux_out).await.unwrap());
+        let mut current_row_id = base_row_id;
+        for (pid, len) in lengths.iter().enumerate() {
+            for _ in 0..*len {
+                row_ids.push(current_row_id);
+                current_row_id += 1;
+                for d in 0..dim {
+                    values.push(pid as f64 + d as f64 * 0.01);
+                }
+            }
+        }
+
+        let row_id_arr = UInt64Array::from(row_ids);
+        let value_arr = Float64Array::from(values);
+        let fsl = FixedSizeListArray::try_new_from_values(value_arr, dim).unwrap();
+        let batch = RecordBatch::try_new(
+            Arc::new(arrow_schema),
+            vec![Arc::new(row_id_arr), Arc::new(fsl)],
+        )
+        .unwrap();
+
+        v2w.write_batch(&batch).await?;
+        v2w.finish().await?;
+        Ok(total_rows)
+    }
+
+    #[tokio::test]
+    async fn test_merge_ivf_flat_success_basic() {
+        let object_store = ObjectStore::memory();
+        let index_dir = Path::from("index/uuid");
+
+        let partial0 = index_dir.child("partial_0");
+        let partial1 = index_dir.child("partial_1");
+        let aux0 = partial0.child(INDEX_AUXILIARY_FILE_NAME);
+        let aux1 = partial1.child(INDEX_AUXILIARY_FILE_NAME);
+
+        let lengths0 = vec![2_u32, 1_u32];
+        let lengths1 = vec![1_u32, 2_u32];
+        let dim = 2_i32;
+
+        write_flat_partial_aux(&object_store, &aux0, dim, &lengths0, 0, DistanceType::L2)
+            .await
+            .unwrap();
+        write_flat_partial_aux(&object_store, &aux1, dim, &lengths1, 100, DistanceType::L2)
+            .await
+            .unwrap();
+
+        let progress = Arc::new(RecordingProgress::default());
+        merge_partial_vector_auxiliary_files(
+            &object_store,
+            &[aux0.clone(), aux1.clone()],
+            &index_dir,
+            progress.clone(),
+        )
+        .await
+        .unwrap();
+
+        let events = progress.recorded_events();
+        let tags = events
+            .iter()
+            .map(|(kind, stage, _)| format!("{kind}:{stage}"))
+            .collect::<Vec<_>>();
+        let merge_total = events
+            .iter()
+            .find_map(|(kind, stage, value)| {
+                if kind == "start" && stage == "merge_partitions" {
+                    Some(*value)
+                } else {
+                    None
+                }
+            })
+            .expect("missing merge_partitions start total");
+        let merged_rows = events
+            .iter()
+            .filter_map(|(kind, stage, value)| {
+                if kind == "progress" && stage == "merge_partitions" {
+                    Some(*value)
+                } else {
+                    None
+                }
+            })
+            .next_back()
+            .unwrap_or_default();
+        let read_start = tags
+            .iter()
+            .position(|e| e == "start:read_shard_metadata")
+            .expect("missing read_shard_metadata start");
+        let read_complete = tags
+            .iter()
+            .position(|e| e == "complete:read_shard_metadata")
+            .expect("missing read_shard_metadata complete");
+        let merge_start = tags
+            .iter()
+            .position(|e| e == "start:merge_partitions")
+            .expect("missing merge_partitions start");
+        let merge_complete = tags
+            .iter()
+            .position(|e| e == "complete:merge_partitions")
+            .expect("missing merge_partitions complete");
+        let write_start = tags
+            .iter()
+            .position(|e| e == "start:write_auxiliary_index")
+            .expect("missing write_auxiliary_index start");
+        let write_complete = tags
+            .iter()
+            .position(|e| e == "complete:write_auxiliary_index")
+            .expect("missing write_auxiliary_index complete");
+        assert!(read_start < read_complete);
+        assert!(read_complete < merge_start);
+        assert!(merge_start < merge_complete);
+        assert!(merge_complete < write_start);
+        assert!(write_start < write_complete);
+        assert!(
+            tags.iter().any(|e| e == "progress:read_shard_metadata"),
+            "expected read_shard_metadata progress callbacks"
+        );
+        assert!(
+            tags.iter().any(|e| e == "progress:merge_partitions"),
+            "expected merge_partitions progress callbacks"
+        );
+        assert_eq!(merge_total, 6, "expected merge_partitions total rows");
+        assert_eq!(merged_rows, 6, "expected merge_partitions completed rows");
+        assert!(
+            tags.iter().any(|e| e == "progress:write_auxiliary_index"),
+            "expected write_auxiliary_index progress callbacks"
+        );
+
+        let aux_out = index_dir.child(INDEX_AUXILIARY_FILE_NAME);
+        assert!(object_store.exists(&aux_out).await.unwrap());
 
         // Use ScanScheduler to obtain a FileScheduler (required by V2Reader::try_open)
         let sched = ScanScheduler::new(
@@ -1625,7 +1858,13 @@ mod tests {
         .await
         .unwrap();
 
-        let res = merge_partial_vector_auxiliary_files(&object_store, &index_dir).await;
+        let res = merge_partial_vector_auxiliary_files(
+            &object_store,
+            &[aux0.clone(), aux1.clone()],
+            &index_dir,
+            crate::progress::noop_progress(),
+        )
+        .await;
         match res {
             Err(Error::Index { message, .. }) => {
                 assert!(
@@ -1641,6 +1880,64 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_merge_ivf_flat_preserves_float64_schema() {
+        let object_store = ObjectStore::memory();
+        let index_dir = Path::from("index/float64_uuid");
+
+        let partial0 = index_dir.child("partial_0");
+        let partial1 = index_dir.child("partial_1");
+        let aux0 = partial0.child(INDEX_AUXILIARY_FILE_NAME);
+        let aux1 = partial1.child(INDEX_AUXILIARY_FILE_NAME);
+
+        let lengths = vec![2_u32, 2_u32];
+        let dim = 3_i32;
+
+        write_flat_partial_aux_f64(&object_store, &aux0, dim, &lengths, 0, DistanceType::L2)
+            .await
+            .unwrap();
+        write_flat_partial_aux_f64(&object_store, &aux1, dim, &lengths, 100, DistanceType::L2)
+            .await
+            .unwrap();
+
+        merge_partial_vector_auxiliary_files(
+            &object_store,
+            &[aux0.clone(), aux1.clone()],
+            &index_dir,
+            Arc::new(RecordingProgress::default()),
+        )
+        .await
+        .unwrap();
+
+        let aux_out = index_dir.child(INDEX_AUXILIARY_FILE_NAME);
+        let sched = ScanScheduler::new(
+            Arc::new(object_store.clone()),
+            SchedulerConfig::max_bandwidth(&object_store),
+        );
+        let fh = sched
+            .open_file(&aux_out, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let reader = V2Reader::try_open(
+            fh,
+            None,
+            Arc::default(),
+            &lance_core::cache::LanceCache::no_cache(),
+            V2ReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+
+        let flat_field = reader
+            .schema()
+            .field(crate::vector::flat::storage::FLAT_COLUMN)
+            .unwrap();
+        let DataType::FixedSizeList(item, _) = flat_field.data_type() else {
+            panic!("flat column should be a fixed size list");
+        };
+        assert_eq!(item.data_type(), &DataType::Float64);
+    }
+
     #[allow(clippy::too_many_arguments)]
     async fn write_pq_partial_aux(
         store: &ObjectStore,
@@ -1745,6 +2042,88 @@ mod tests {
         Ok(total_rows)
     }
 
+    async fn write_rq_partial_aux(
+        store: &ObjectStore,
+        aux_path: &Path,
+        metadata: &RabitQuantizationMetadata,
+        lengths: &[u32],
+        base_row_id: u64,
+        distance_type: DistanceType,
+    ) -> Result<usize> {
+        let num_bytes = (metadata.code_dim as usize).div_ceil(u8::BITS as usize);
+        let arrow_schema = ArrowSchema::new(vec![
+            (*ROW_ID_FIELD).clone(),
+            Field::new(
+                RABIT_CODE_COLUMN,
+                DataType::FixedSizeList(
+                    Arc::new(Field::new("item", DataType::UInt8, true)),
+                    num_bytes as i32,
+                ),
+                true,
+            ),
+            ADD_FACTORS_FIELD.clone(),
+            SCALE_FACTORS_FIELD.clone(),
+        ]);
+
+        let writer = store.create(aux_path).await?;
+        let mut v2w = V2Writer::try_new(
+            writer,
+            lance_core::datatypes::Schema::try_from(&arrow_schema)?,
+            V2WriterOptions::default(),
+        )?;
+        v2w.add_schema_metadata(DISTANCE_TYPE_KEY, distance_type.to_string());
+
+        let rq_meta_json = serde_json::to_string(metadata)?;
+        v2w.add_schema_metadata(RABIT_METADATA_KEY, rq_meta_json);
+
+        let ivf_meta = pb::Ivf {
+            centroids: Vec::new(),
+            offsets: Vec::new(),
+            lengths: lengths.to_vec(),
+            centroids_tensor: None,
+            loss: None,
+        };
+        let buf = Bytes::from(ivf_meta.encode_to_vec());
+        let ivf_pos = v2w.add_global_buffer(buf).await?;
+        v2w.add_schema_metadata(IVF_METADATA_KEY, ivf_pos.to_string());
+
+        let total_rows: usize = lengths.iter().map(|v| *v as usize).sum();
+        let mut row_ids = Vec::with_capacity(total_rows);
+        let mut codes = Vec::with_capacity(total_rows * num_bytes);
+        let mut add_factors = Vec::with_capacity(total_rows);
+        let mut scale_factors = Vec::with_capacity(total_rows);
+
+        let mut current_row_id = base_row_id;
+        for (pid, len) in lengths.iter().enumerate() {
+            for row_offset in 0..*len as usize {
+                row_ids.push(current_row_id);
+                current_row_id += 1;
+                for b in 0..num_bytes {
+                    codes.push((pid + row_offset + b) as u8);
+                }
+                add_factors.push(pid as f32 + row_offset as f32 * 0.1);
+                scale_factors.push(pid as f32 + row_offset as f32 * 0.2);
+            }
+        }
+
+        let batch = RecordBatch::try_new(
+            Arc::new(arrow_schema),
+            vec![
+                Arc::new(UInt64Array::from(row_ids)),
+                Arc::new(FixedSizeListArray::try_new_from_values(
+                    UInt8Array::from(codes),
+                    num_bytes as i32,
+                )?),
+                Arc::new(Float32Array::from(add_factors)),
+                Arc::new(Float32Array::from(scale_factors)),
+            ],
+        )?;
+
+        v2w.write_batch(&batch).await?;
+        v2w.finish().await?;
+        Ok(total_rows)
+    }
+
     #[tokio::test]
     async fn test_merge_ivf_pq_success() {
         let object_store = ObjectStore::memory();
@@ -1800,9 +2179,14 @@ mod tests {
         .unwrap();
 
         // Merge PQ auxiliary files.
-        merge_partial_vector_auxiliary_files(&object_store, &index_dir)
-            .await
-            .unwrap();
+        merge_partial_vector_auxiliary_files(
+            &object_store,
+            &[aux0.clone(), aux1.clone()],
+            &index_dir,
+            crate::progress::noop_progress(),
+        )
+        .await
+        .unwrap();
 
         // 3) Unified auxiliary file exists.
         let aux_out = index_dir.child(INDEX_AUXILIARY_FILE_NAME);
@@ -1870,6 +2254,128 @@ mod tests {
         assert!(fixed_size_list_equal(&codebook, &merged_codebook));
     }
 
+    #[tokio::test]
+    async fn test_merge_ivf_rq_success() {
+        let object_store = ObjectStore::memory();
+        let index_dir = Path::from("index/uuid_rq");
+
+        let partial0 = index_dir.child("partial_0");
+        let partial1 = index_dir.child("partial_1");
+        let aux0 = partial0.child(INDEX_AUXILIARY_FILE_NAME);
+        let aux1 = partial1.child(INDEX_AUXILIARY_FILE_NAME);
+
+        let lengths0 = vec![2_u32, 1_u32];
+        let lengths1 = vec![1_u32, 2_u32];
+
+        let rq_meta = RabitQuantizationMetadata {
+            rotate_mat: None,
+            rotate_mat_position: None,
+            fast_rotation_signs: Some(vec![0xAA; 2]),
+            rotation_type: RQRotationType::Fast,
+            code_dim: 16,
+            num_bits: 1,
+            packed: false,
+        };
+
+        write_rq_partial_aux(
+            &object_store,
+            &aux0,
+            &rq_meta,
+            &lengths0,
+            0,
+            DistanceType::L2,
+        )
+        .await
+        .unwrap();
+        write_rq_partial_aux(
+            &object_store,
+            &aux1,
+            &rq_meta,
+            &lengths1,
+            1_000,
+            DistanceType::L2,
+        )
+        .await
+        .unwrap();
+
+        merge_partial_vector_auxiliary_files(
+            &object_store,
+            &[aux0.clone(), aux1.clone()],
+            &index_dir,
+            crate::progress::noop_progress(),
+        )
+        .await
+        .unwrap();
+
+        let aux_out = index_dir.child(INDEX_AUXILIARY_FILE_NAME);
+        assert!(object_store.exists(&aux_out).await.unwrap());
+
+        let sched = ScanScheduler::new(
+            Arc::new(object_store.clone()),
+            SchedulerConfig::max_bandwidth(&object_store),
+        );
+        let fh = sched
+            .open_file(&aux_out, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let reader = V2Reader::try_open(
+            fh,
+            None,
+            Arc::default(),
+            &lance_core::cache::LanceCache::no_cache(),
+            V2ReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+        let meta = reader.metadata();
+
+        let ivf_idx: u32 = meta
+            .file_schema
+            .metadata
+            .get(IVF_METADATA_KEY)
+            .unwrap()
+            .parse()
+            .unwrap();
+        let bytes = reader.read_global_buffer(ivf_idx).await.unwrap();
+        let pb_ivf: pb::Ivf = prost::Message::decode(bytes).unwrap();
+        let expected_lengths: Vec<u32> = lengths0
+            .iter()
+            .zip(lengths1.iter())
+            .map(|(a, b)| *a + *b)
+            .collect();
+        assert_eq!(pb_ivf.lengths, expected_lengths);
+
+        let idx_meta_json = meta
+            .file_schema
+            .metadata
+            .get(INDEX_METADATA_SCHEMA_KEY)
+            .unwrap();
+        let idx_meta: IndexMetaSchema = serde_json::from_str(idx_meta_json).unwrap();
+        assert_eq!(idx_meta.index_type, "IVF_RQ");
+        assert_eq!(idx_meta.distance_type, DistanceType::L2.to_string());
+
+        let rq_meta_json = meta.file_schema.metadata.get(RABIT_METADATA_KEY).unwrap();
+        let merged_rq_meta: RabitQuantizationMetadata = serde_json::from_str(rq_meta_json).unwrap();
+        assert_eq!(merged_rq_meta.code_dim, rq_meta.code_dim);
+        assert_eq!(merged_rq_meta.num_bits, rq_meta.num_bits);
+        assert!(merged_rq_meta.packed);
+
+        let mut total_rows = 0usize;
+        let mut stream = reader
+            .read_stream(
+                lance_io::ReadBatchParams::RangeFull,
+                u32::MAX,
+                4,
+                lance_encoding::decoder::FilterExpression::no_filter(),
+            )
+            .unwrap();
+        while let Some(batch) = stream.next().await {
+            total_rows += batch.unwrap().num_rows();
+        }
+        let expected_total: usize = expected_lengths.iter().map(|v| *v as usize).sum();
+        assert_eq!(total_rows, expected_total);
+    }
+
     #[tokio::test]
     async fn test_merge_ivf_pq_codebook_mismatch() {
         let object_store = ObjectStore::memory();
@@ -1928,7 +2434,13 @@ mod tests {
         .await
         .unwrap();
 
-        let res = merge_partial_vector_auxiliary_files(&object_store, &index_dir).await;
+        let res = merge_partial_vector_auxiliary_files(
+            &object_store,
+            &[aux0.clone(), aux1.clone()],
+            &index_dir,
+            crate::progress::noop_progress(),
+        )
+        .await;
         match res {
             Err(Error::Index { message, .. }) => {
                 assert!(
@@ -2003,9 +2515,14 @@ mod tests {
         .unwrap();
 
         // Merge must succeed and produce a unified auxiliary file.
-        merge_partial_vector_auxiliary_files(&object_store, &index_dir)
-            .await
-            .unwrap();
+        merge_partial_vector_auxiliary_files(
+            &object_store,
+            &[aux_a.clone(), aux_b.clone()],
+            &index_dir,
+            crate::progress::noop_progress(),
+        )
+        .await
+        .unwrap();
 
         let aux_out = index_dir.child(INDEX_AUXILIARY_FILE_NAME);
         assert!(object_store.exists(&aux_out).await.unwrap());
@@ -2059,132 +2576,4 @@ mod tests {
         let first_partition_ids = &row_ids[..4];
         assert_eq!(first_partition_ids, &[0, 1, 1_000, 1_001]);
     }
-
-    #[tokio::test]
-    async fn test_merge_content_key_order_invariance() {
-        // Two partial directories whose content-derived keys
-        // (min_fragment_id, min_row_id) are identical; ordering is determined
-        // solely by the parent directory name as a lexicographic tie-breaker.
-        let object_store = ObjectStore::memory();
-        let index_dir = Path::from("index/content_key");
-
-        let partial_a = index_dir.child("partial_content_a");
-        let partial_b = index_dir.child("partial_content_b");
-        let aux_a = partial_a.child(INDEX_AUXILIARY_FILE_NAME);
-        let aux_b = partial_b.child(INDEX_AUXILIARY_FILE_NAME);
-
-        // Equal-length shards so per-partition lengths alone cannot disambiguate
-        // ordering.
-        let lengths = vec![2_u32, 2_u32];
-
-        // PQ parameters shared by both shards.
-        let nbits = 4_u32;
-        let num_sub_vectors = 2_usize;
-        let dimension = 8_usize;
-
-        let num_centroids = 1_usize << nbits;
-        let num_codebook_vectors = num_centroids * num_sub_vectors;
-        let total_values = num_codebook_vectors * dimension;
-        let values = Float32Array::from_iter((0..total_values).map(|v| v as f32));
-        let codebook = FixedSizeListArray::try_new_from_values(values, dimension as i32).unwrap();
-
-        // Use a RowAddress-encoded base so both shards have the same
-        // (fragment_id, row_offset) for their first row, hence identical
-        // content-derived numeric keys.
-        let base_addr: u64 = RowAddress::new_from_parts(1, 5).into();
-
-        write_pq_partial_aux(
-            &object_store,
-            &aux_a,
-            nbits,
-            num_sub_vectors,
-            dimension,
-            &lengths,
-            base_addr,
-            DistanceType::L2,
-            &codebook,
-        )
-        .await
-        .unwrap();
-
-        write_pq_partial_aux(
-            &object_store,
-            &aux_b,
-            nbits,
-            num_sub_vectors,
-            dimension,
-            &lengths,
-            base_addr,
-            DistanceType::L2,
-            &codebook,
-        )
-        .await
-        .unwrap();
-
-        // Merge must succeed and produce a unified auxiliary file.
-        merge_partial_vector_auxiliary_files(&object_store, &index_dir)
-            .await
-            .unwrap();
-
-        let aux_out = index_dir.child(INDEX_AUXILIARY_FILE_NAME);
-        assert!(object_store.exists(&aux_out).await.unwrap());
-
-        // Open merged auxiliary file and inspect row id layout.
-        let sched = ScanScheduler::new(
-            Arc::new(object_store.clone()),
-            SchedulerConfig::max_bandwidth(&object_store),
-        );
-        let fh = sched
-            .open_file(&aux_out, &CachedFileSize::unknown())
-            .await
-            .unwrap();
-        let reader = V2Reader::try_open(
-            fh,
-            None,
-            Arc::default(),
-            &lance_core::cache::LanceCache::no_cache(),
-            V2ReaderOptions::default(),
-        )
-        .await
-        .unwrap();
-
-        let mut stream = reader
-            .read_stream(
-                lance_io::ReadBatchParams::RangeFull,
-                u32::MAX,
-                4,
-                lance_encoding::decoder::FilterExpression::no_filter(),
-            )
-            .unwrap();
-
-        let mut row_ids = Vec::new();
-        while let Some(batch) = stream.next().await {
-            let batch = batch.unwrap();
-            let arr = batch
-                .column(0)
-                .as_any()
-                .downcast_ref::<UInt64Array>()
-                .unwrap();
-            for i in 0..arr.len() {
-                row_ids.push(arr.value(i));
-            }
-        }
-
-        // Two shards, each contributing `sum(lengths)` rows.
-        let expected_total_rows: usize = lengths.iter().map(|v| *v as usize).sum::<usize>() * 2;
-        assert_eq!(row_ids.len(), expected_total_rows);
-
-        let first_partition_rows = lengths[0] as usize * 2;
-        let (p0, p1) = row_ids.split_at(first_partition_rows);
-
-        let base = base_addr;
-        // For partition 0 we expect rows from `partial_content_a` first, then
-        // from `partial_content_b`.
-        let expected_p0 = vec![base, base + 1, base, base + 1];
-        assert_eq!(p0, expected_p0.as_slice());
-
-        // For partition 1 the pattern continues with offsets +2, +3.
-        let expected_p1 = vec![base + 2, base + 3, base + 2, base + 3];
-        assert_eq!(p1, expected_p1.as_slice());
-    }
 }
diff --git a/rust/lance-index/src/vector/flat/index.rs b/rust/lance-index/src/vector/flat/index.rs
index 35cf9382996..bb75ca97fd7 100644
--- a/rust/lance-index/src/vector/flat/index.rs
+++ b/rust/lance-index/src/vector/flat/index.rs
@@ -343,7 +343,7 @@ impl Quantization for FlatBinQuantizer {
     }
 
     fn quantization_type() -> QuantizationType {
-        QuantizationType::Flat
+        QuantizationType::FlatBin
     }
 
     fn quantize(&self, vectors: &dyn Array) -> Result<ArrayRef> {
diff --git a/rust/lance-index/src/vector/flat/storage.rs b/rust/lance-index/src/vector/flat/storage.rs
index 8f774b6e76d..4f8efc67e62 100644
--- a/rust/lance-index/src/vector/flat/storage.rs
+++ b/rust/lance-index/src/vector/flat/storage.rs
@@ -10,18 +10,18 @@ use crate::vector::storage::{DistCalculator, VectorStore};
 use crate::vector::utils::do_prefetch;
 use arrow::array::AsArray;
 use arrow::compute::concat_batches;
-use arrow::datatypes::UInt8Type;
+use arrow::datatypes::{Float16Type, Float64Type, UInt8Type};
 use arrow_array::ArrowPrimitiveType;
 use arrow_array::{
     Array, ArrayRef, FixedSizeListArray, RecordBatch, UInt64Array,
     types::{Float32Type, UInt64Type},
 };
-use arrow_schema::SchemaRef;
+use arrow_schema::{DataType, SchemaRef};
 use deepsize::DeepSizeOf;
 use lance_core::{Error, ROW_ID, Result};
 use lance_file::previous::reader::FileReader as PreviousFileReader;
-use lance_linalg::distance::DistanceType;
 use lance_linalg::distance::hamming::hamming;
+use lance_linalg::distance::{Cosine, DistanceType, Dot, L2};
 
 pub const FLAT_COLUMN: &str = "flat";
 
@@ -126,7 +126,7 @@ impl FlatFloatStorage {
 }
 
 impl VectorStore for FlatFloatStorage {
-    type DistanceCalculator<'a> = FlatDistanceCal<'a, Float32Type>;
+    type DistanceCalculator<'a> = FlatFloatDistanceCalc<'a>;
 
     fn to_batches(&self) -> Result<impl Iterator<Item = RecordBatch>> {
         Ok([self.batch.clone()].into_iter())
@@ -136,6 +136,20 @@ impl VectorStore for FlatFloatStorage {
         // TODO: use chunked storage
         let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch].into_iter())?;
         let mut storage = self.clone();
+        storage.row_ids = Arc::new(
+            new_batch
+                .column_by_name(ROW_ID)
+                .ok_or(Error::schema(format!("column {} not found", ROW_ID)))?
+                .as_primitive::<UInt64Type>()
+                .clone(),
+        );
+        storage.vectors = Arc::new(
+            new_batch
+                .column_by_name(FLAT_COLUMN)
+                .ok_or(Error::schema("column flat not found".to_string()))?
+                .as_fixed_size_list()
+                .clone(),
+        );
         storage.batch = new_batch;
         Ok(storage)
     }
@@ -288,6 +302,20 @@ impl VectorStore for FlatBinStorage {
         // TODO: use chunked storage
         let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch].into_iter())?;
         let mut storage = self.clone();
+        storage.row_ids = Arc::new(
+            new_batch
+                .column_by_name(ROW_ID)
+                .ok_or(Error::schema(format!("column {} not found", ROW_ID)))?
+                .as_primitive::<UInt64Type>()
+                .clone(),
+        );
+        storage.vectors = Arc::new(
+            new_batch
+                .column_by_name(FLAT_COLUMN)
+                .ok_or(Error::schema("column flat not found".to_string()))?
+                .as_fixed_size_list()
+                .clone(),
+        );
         storage.batch = new_batch;
         Ok(storage)
     }
@@ -317,11 +345,11 @@ impl VectorStore for FlatBinStorage {
     }
 
     fn dist_calculator(&self, query: ArrayRef, _dist_q_c: f32) -> Self::DistanceCalculator<'_> {
-        Self::DistanceCalculator::new(self.vectors.as_ref(), query, self.distance_type)
+        Self::DistanceCalculator::new_binary(self.vectors.as_ref(), query, self.distance_type)
     }
 
     fn dist_calculator_from_id(&self, id: u32) -> Self::DistanceCalculator<'_> {
-        Self::DistanceCalculator::new(
+        Self::DistanceCalculator::new_binary(
             self.vectors.as_ref(),
             self.vectors.value(id as usize),
             self.distance_type,
@@ -337,15 +365,18 @@ pub struct FlatDistanceCal<'a, T: ArrowPrimitiveType> {
     distance_fn: fn(&[T::Native], &[T::Native]) -> f32,
 }
 
-impl<'a> FlatDistanceCal<'a, Float32Type> {
+impl<'a, T> FlatDistanceCal<'a, T>
+where
+    T: ArrowPrimitiveType,
+    T::Native: L2 + Cosine + Dot,
+{
     fn new(vectors: &'a FixedSizeListArray, query: ArrayRef, distance_type: DistanceType) -> Self {
         // Gained significant performance improvement by using strong typed primitive slice.
-        // TODO: to support other data types other than `f32`, make FlatDistanceCal a generic struct.
-        let flat_array = vectors.values().as_primitive::<Float32Type>();
+        let flat_array = vectors.values().as_primitive::<T>();
         let dimension = vectors.value_length() as usize;
         Self {
             vectors: flat_array.values(),
-            query: query.as_primitive::<Float32Type>().values().to_vec(),
+            query: query.as_primitive::<T>().values().to_vec(),
             dimension,
             distance_fn: distance_type.func(),
         }
@@ -353,7 +384,11 @@ impl<'a> FlatDistanceCal<'a, Float32Type> {
 }
 
 impl<'a> FlatDistanceCal<'a, UInt8Type> {
-    fn new(vectors: &'a FixedSizeListArray, query: ArrayRef, _distance_type: DistanceType) -> Self {
+    fn new_binary(
+        vectors: &'a FixedSizeListArray,
+        query: ArrayRef,
+        _distance_type: DistanceType,
+    ) -> Self {
         // Gained significant performance improvement by using strong typed primitive slice.
         // TODO: to support other data types other than `f32`, make FlatDistanceCal a generic struct.
         let flat_array = vectors.values().as_primitive::<UInt8Type>();
@@ -395,3 +430,113 @@ impl<T: ArrowPrimitiveType> DistCalculator for FlatDistanceCal<'_, T> {
         do_prefetch(vector.as_ptr_range())
     }
 }
+
+pub enum FlatFloatDistanceCalc<'a> {
+    Float16(FlatDistanceCal<'a, Float16Type>),
+    Float32(FlatDistanceCal<'a, Float32Type>),
+    Float64(FlatDistanceCal<'a, Float64Type>),
+}
+
+impl<'a> FlatFloatDistanceCalc<'a> {
+    fn new(vectors: &'a FixedSizeListArray, query: ArrayRef, distance_type: DistanceType) -> Self {
+        match vectors.value_type() {
+            DataType::Float16 => Self::Float16(FlatDistanceCal::<Float16Type>::new(
+                vectors,
+                query,
+                distance_type,
+            )),
+            DataType::Float32 => Self::Float32(FlatDistanceCal::<Float32Type>::new(
+                vectors,
+                query,
+                distance_type,
+            )),
+            DataType::Float64 => Self::Float64(FlatDistanceCal::<Float64Type>::new(
+                vectors,
+                query,
+                distance_type,
+            )),
+            dt => panic!("flat float storage does not support data type {dt}"),
+        }
+    }
+}
+
+impl DistCalculator for FlatFloatDistanceCalc<'_> {
+    fn distance(&self, id: u32) -> f32 {
+        match self {
+            Self::Float16(calc) => calc.distance(id),
+            Self::Float32(calc) => calc.distance(id),
+            Self::Float64(calc) => calc.distance(id),
+        }
+    }
+
+    fn distance_all(&self, k_hint: usize) -> Vec<f32> {
+        match self {
+            Self::Float16(calc) => calc.distance_all(k_hint),
+            Self::Float32(calc) => calc.distance_all(k_hint),
+            Self::Float64(calc) => calc.distance_all(k_hint),
+        }
+    }
+
+    fn prefetch(&self, id: u32) {
+        match self {
+            Self::Float16(calc) => calc.prefetch(id),
+            Self::Float32(calc) => calc.prefetch(id),
+            Self::Float64(calc) => calc.prefetch(id),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use arrow_array::{Float16Array, Float64Array};
+    use half::f16;
+    use lance_arrow::FixedSizeListArrayExt;
+
+    fn make_f16_storage() -> FlatFloatStorage {
+        let values = Float16Array::from(vec![
+            f16::from_f32(1.0),
+            f16::from_f32(2.0),
+            f16::from_f32(4.0),
+            f16::from_f32(6.0),
+        ]);
+        let vectors = FixedSizeListArray::try_new_from_values(values, 2).unwrap();
+        FlatFloatStorage::new(vectors, DistanceType::L2)
+    }
+
+    fn make_f64_storage() -> FlatFloatStorage {
+        let values = Float64Array::from(vec![1.0, 2.0, 4.0, 6.0]);
+        let vectors = FixedSizeListArray::try_new_from_values(values, 2).unwrap();
+        FlatFloatStorage::new(vectors, DistanceType::L2)
+    }
+
+    #[test]
+    fn test_flat_float_storage_distance_f16() {
+        let storage = make_f16_storage();
+        let query: ArrayRef = Arc::new(Float16Array::from(vec![
+            f16::from_f32(1.0),
+            f16::from_f32(2.0),
+        ]));
+
+        let calc = storage.dist_calculator(query, 0.0);
+        let distances = calc.distance_all(2);
+
+        assert_eq!(distances.len(), 2);
+        assert_eq!(distances[0], 0.0);
+        assert!((distances[1] - 25.0).abs() < 1e-4);
+    }
+
+    #[test]
+    fn test_flat_float_storage_distance_f64() {
+        let storage = make_f64_storage();
+        let query: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0]));
+
+        let calc = storage.dist_calculator(query, 0.0);
+        let distances = calc.distance_all(2);
+
+        assert_eq!(distances.len(), 2);
+        assert_eq!(distances[0], 0.0);
+        assert!((distances[1] - 25.0).abs() < 1e-6);
+    }
+}
diff --git a/rust/lance-index/src/vector/graph.rs b/rust/lance-index/src/vector/graph.rs
index 2dac8a23f7e..b79ce4e9947 100644
--- a/rust/lance-index/src/vector/graph.rs
+++ b/rust/lance-index/src/vector/graph.rs
@@ -9,7 +9,6 @@ use std::collections::BinaryHeap;
 use std::sync::Arc;
 
 use arrow_schema::{DataType, Field};
-use bitvec::vec::BitVec;
 use deepsize::DeepSizeOf;
 
 use crate::vector::hnsw::builder::HnswQueryParams;
@@ -161,40 +160,64 @@ pub trait Graph {
     fn neighbors(&self, key: u32) -> Arc<Vec<u32>>;
 }
 
-/// Array-based visited list (faster than HashSet)
+pub trait BorrowingGraph {
+    /// Get the number of nodes in the graph.
+    fn len(&self) -> usize;
+
+    /// Returns true if the graph is empty.
+    fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Borrow the neighbors of a graph node, identified by the index.
+    fn neighbors(&self, key: u32) -> &[u32];
+}
+
+const WORD_BITS: usize = usize::BITS as usize;
+
+/// Compact visited list for graph traversals.
 pub struct Visited<'a> {
-    visited: &'a mut BitVec,
-    recently_visited: Vec<u32>,
+    visited: &'a mut Vec<usize>,
+    recently_visited: &'a mut Vec<u32>,
 }
 
 impl Visited<'_> {
     pub fn insert(&mut self, node_id: u32) {
         let node_id_usize = node_id as usize;
-        if !self.visited[node_id_usize] {
-            self.visited.set(node_id_usize, true);
+        let word_index = node_id_usize / WORD_BITS;
+        let mask = 1usize << (node_id_usize % WORD_BITS);
+        if self.visited[word_index] & mask == 0 {
+            self.visited[word_index] |= mask;
             self.recently_visited.push(node_id);
         }
     }
 
     pub fn contains(&self, node_id: u32) -> bool {
         let node_id_usize = node_id as usize;
-        self.visited[node_id_usize]
+        let word_index = node_id_usize / WORD_BITS;
+        let mask = 1usize << (node_id_usize % WORD_BITS);
+        self.visited[word_index] & mask != 0
     }
 
     #[inline(always)]
     pub fn iter_ones(&self) -> impl Iterator<Item = usize> + '_ {
-        self.visited.iter_ones()
+        self.recently_visited
+            .iter()
+            .map(|node_id| *node_id as usize)
     }
 
     pub fn count_ones(&self) -> usize {
-        self.visited.count_ones()
+        self.recently_visited.len()
     }
 }
 
 impl Drop for Visited<'_> {
     fn drop(&mut self) {
-        for node_id in self.recently_visited.iter() {
-            self.visited.set(*node_id as usize, false);
+        for node_id in self.recently_visited.iter().copied() {
+            let node_id_usize = node_id as usize;
+            let word_index = node_id_usize / WORD_BITS;
+            let mask = 1usize << (node_id_usize % WORD_BITS);
+            self.visited[word_index] &= !mask;
         }
         self.recently_visited.clear();
     }
@@ -202,14 +225,16 @@ impl Drop for Visited<'_> {
 
 #[derive(Debug, Clone)]
 pub struct VisitedGenerator {
-    visited: BitVec,
+    visited: Vec<usize>,
+    recently_visited: Vec<u32>,
     capacity: usize,
 }
 
 impl VisitedGenerator {
     pub fn new(capacity: usize) -> Self {
         Self {
-            visited: BitVec::repeat(false, capacity),
+            visited: vec![0; capacity.div_ceil(WORD_BITS)],
+            recently_visited: Vec::new(),
             capacity,
         }
     }
@@ -217,12 +242,12 @@ impl VisitedGenerator {
     pub fn generate(&mut self, node_count: usize) -> Visited<'_> {
         if node_count > self.capacity {
             let new_capacity = self.capacity.max(node_count).next_power_of_two();
-            self.visited.resize(new_capacity, false);
+            self.visited.resize(new_capacity.div_ceil(WORD_BITS), 0);
             self.capacity = new_capacity;
         }
         Visited {
             visited: &mut self.visited,
-            recently_visited: Vec::new(),
+            recently_visited: &mut self.recently_visited,
         }
     }
 }
@@ -253,6 +278,89 @@ fn process_neighbors_with_look_ahead<F>(
     }
 }
 
+#[inline]
+fn furthest_distance(results: &BinaryHeap<OrderedNode>) -> OrderedFloat {
+    results
+        .peek()
+        .map(|node| node.dist)
+        .unwrap_or(OrderedFloat(f32::INFINITY))
+}
+
+#[inline]
+fn push_result(results: &mut BinaryHeap<OrderedNode>, candidate: OrderedNode, k: usize) {
+    if results.len() < k {
+        results.push(candidate);
+    } else if candidate.dist < results.peek().unwrap().dist {
+        results.pop();
+        results.push(candidate);
+    }
+}
+
+macro_rules! beam_search_loop {
+    (
+        $candidates:ident,
+        $results:ident,
+        $visited:ident,
+        $k:expr,
+        $dist_calc:expr,
+        $prefetch_distance:expr,
+        $accepts_result:expr,
+        |$current:ident, $process_neighbor:ident| $visit_neighbors:block
+    ) => {{
+        while !$candidates.is_empty() {
+            let $current = $candidates.pop().expect("candidates is empty").0;
+            let furthest = furthest_distance(&$results);
+
+            if $current.dist > furthest && $results.len() == $k {
+                break;
+            }
+
+            let $process_neighbor = |neighbor: u32| {
+                if $visited.contains(neighbor) {
+                    return;
+                }
+                $visited.insert(neighbor);
+                let dist: OrderedFloat = $dist_calc.distance(neighbor).into();
+                if dist <= furthest || $results.len() < $k {
+                    if $accepts_result(neighbor, dist) {
+                        push_result(&mut $results, (dist, neighbor).into(), $k);
+                    }
+                    $candidates.push(Reverse((dist, neighbor).into()));
+                }
+            };
+            $visit_neighbors
+        }
+    }};
+}
+
+macro_rules! greedy_search_loop {
+    (
+        $current:ident,
+        $closest_dist:ident,
+        $dist_calc:expr,
+        $prefetch_distance:expr,
+        |$process_neighbor:ident| $visit_neighbors:block
+    ) => {{
+        loop {
+            let mut next = None;
+            let $process_neighbor = |neighbor: u32| {
+                let dist = $dist_calc.distance(neighbor);
+                if dist < $closest_dist {
+                    $closest_dist = dist;
+                    next = Some(neighbor);
+                }
+            };
+            $visit_neighbors
+
+            if let Some(next) = next {
+                $current = next;
+            } else {
+                break;
+            }
+        }
+    }};
+}
+
 /// Beam search over a graph
 ///
 /// This is the same as ``search-layer`` in HNSW.
@@ -291,12 +399,38 @@ pub fn beam_search(
     visited.insert(ep.id);
     candidates.push(Reverse(ep.clone()));
 
+    let mut results = BinaryHeap::with_capacity(k);
+    let no_filter =
+        bitset.is_none() && params.lower_bound.is_none() && params.upper_bound.is_none();
+
+    if no_filter {
+        results.push(ep.clone());
+        let accepts_result = |_: u32, _: OrderedFloat| true;
+        beam_search_loop!(
+            candidates,
+            results,
+            visited,
+            k,
+            dist_calc,
+            prefetch_distance,
+            accepts_result,
+            |current, process_neighbor| {
+                let neighbors = graph.neighbors(current.id);
+                process_neighbors_with_look_ahead(
+                    &neighbors,
+                    process_neighbor,
+                    prefetch_distance,
+                    dist_calc,
+                );
+            }
+        );
+        return results.into_sorted_vec();
+    }
+
     // add range search support
     let lower_bound: OrderedFloat = params.lower_bound.unwrap_or(f32::MIN).into();
     let upper_bound: OrderedFloat = params.upper_bound.unwrap_or(f32::MAX).into();
 
-    let mut results = BinaryHeap::with_capacity(k);
-
     if bitset.map(|bitset| bitset.contains(ep.id)).unwrap_or(true)
         && ep.dist >= lower_bound
         && ep.dist < upper_bound
@@ -304,54 +438,111 @@ pub fn beam_search(
         results.push(ep.clone());
     }
 
-    while !candidates.is_empty() {
-        let current = candidates.pop().expect("candidates is empty").0;
-        let furthest = results
-            .peek()
-            .map(|node| node.dist)
-            .unwrap_or(OrderedFloat(f32::INFINITY));
-
-        // TODO: add an option to ignore the second condition for better performance.
-        if current.dist > furthest && results.len() == k {
-            break;
+    let accepts_result = |node_id: u32, dist: OrderedFloat| {
+        bitset
+            .map(|bitset| bitset.contains(node_id))
+            .unwrap_or(true)
+            && dist >= lower_bound
+            && dist < upper_bound
+    };
+    beam_search_loop!(
+        candidates,
+        results,
+        visited,
+        k,
+        dist_calc,
+        prefetch_distance,
+        accepts_result,
+        |current, process_neighbor| {
+            let neighbors = graph.neighbors(current.id);
+            process_neighbors_with_look_ahead(
+                &neighbors,
+                process_neighbor,
+                prefetch_distance,
+                dist_calc,
+            );
         }
-        let furthest = results
-            .peek()
-            .map(|node| node.dist)
-            .unwrap_or(OrderedFloat(f32::INFINITY));
-
-        let process_neighbor = |neighbor: u32| {
-            if visited.contains(neighbor) {
-                return;
-            }
-            visited.insert(neighbor);
-            let dist: OrderedFloat = dist_calc.distance(neighbor).into();
-            if dist <= furthest || results.len() < k {
-                if bitset
-                    .map(|bitset| bitset.contains(neighbor))
-                    .unwrap_or(true)
-                    && dist >= lower_bound
-                    && dist < upper_bound
-                {
-                    if results.len() < k {
-                        results.push((dist, neighbor).into());
-                    } else if results.len() == k && dist < results.peek().unwrap().dist {
-                        results.pop();
-                        results.push((dist, neighbor).into());
-                    }
-                }
-                candidates.push(Reverse((dist, neighbor).into()));
-            }
-        };
-        let neighbors = graph.neighbors(current.id);
-        process_neighbors_with_look_ahead(
-            &neighbors,
-            process_neighbor,
-            prefetch_distance,
+    );
+    results.into_sorted_vec()
+}
+
+pub fn beam_search_borrowed(
+    graph: &impl BorrowingGraph,
+    ep: &OrderedNode,
+    params: &HnswQueryParams,
+    dist_calc: &impl DistCalculator,
+    bitset: Option<&Visited>,
+    prefetch_distance: Option<usize>,
+    visited: &mut Visited,
+) -> Vec<OrderedNode> {
+    let k = params.ef;
+    let mut candidates = BinaryHeap::with_capacity(k);
+    visited.insert(ep.id);
+    candidates.push(Reverse(ep.clone()));
+
+    let mut results = BinaryHeap::with_capacity(k);
+    let no_filter =
+        bitset.is_none() && params.lower_bound.is_none() && params.upper_bound.is_none();
+
+    if no_filter {
+        results.push(ep.clone());
+        let accepts_result = |_: u32, _: OrderedFloat| true;
+        beam_search_loop!(
+            candidates,
+            results,
+            visited,
+            k,
             dist_calc,
+            prefetch_distance,
+            accepts_result,
+            |current, process_neighbor| {
+                let neighbors = graph.neighbors(current.id);
+                process_neighbors_with_look_ahead(
+                    neighbors,
+                    process_neighbor,
+                    prefetch_distance,
+                    dist_calc,
+                );
+            }
         );
+        return results.into_sorted_vec();
+    }
+
+    let lower_bound: OrderedFloat = params.lower_bound.unwrap_or(f32::MIN).into();
+    let upper_bound: OrderedFloat = params.upper_bound.unwrap_or(f32::MAX).into();
+
+    if bitset.map(|bitset| bitset.contains(ep.id)).unwrap_or(true)
+        && ep.dist >= lower_bound
+        && ep.dist < upper_bound
+    {
+        results.push(ep.clone());
     }
 
+    let accepts_result = |node_id: u32, dist: OrderedFloat| {
+        bitset
+            .map(|bitset| bitset.contains(node_id))
+            .unwrap_or(true)
+            && dist >= lower_bound
+            && dist < upper_bound
+    };
+    beam_search_loop!(
+        candidates,
+        results,
+        visited,
+        k,
+        dist_calc,
+        prefetch_distance,
+        accepts_result,
+        |current, process_neighbor| {
+            let neighbors = graph.neighbors(current.id);
+            process_neighbors_with_look_ahead(
+                neighbors,
+                process_neighbor,
+                prefetch_distance,
+                dist_calc,
+            );
+        }
+    );
     results.into_sorted_vec()
 }
 
@@ -381,31 +572,47 @@ pub fn greedy_search(
 ) -> OrderedNode {
     let mut current = start.id;
     let mut closest_dist = start.dist.0;
-    loop {
-        let neighbors = graph.neighbors(current);
-        let mut next = None;
-
-        let process_neighbor = |neighbor: u32| {
-            let dist = dist_calc.distance(neighbor);
-            if dist < closest_dist {
-                closest_dist = dist;
-                next = Some(neighbor);
-            }
-        };
-        process_neighbors_with_look_ahead(
-            &neighbors,
-            process_neighbor,
-            prefetch_distance,
-            dist_calc,
-        );
-
-        if let Some(next) = next {
-            current = next;
-        } else {
-            break;
+    greedy_search_loop!(
+        current,
+        closest_dist,
+        dist_calc,
+        prefetch_distance,
+        |process_neighbor| {
+            let neighbors = graph.neighbors(current);
+            process_neighbors_with_look_ahead(
+                &neighbors,
+                process_neighbor,
+                prefetch_distance,
+                dist_calc,
+            );
         }
-    }
+    );
+    OrderedNode::new(current, closest_dist.into())
+}
 
+pub fn greedy_search_borrowed(
+    graph: &impl BorrowingGraph,
+    start: OrderedNode,
+    dist_calc: &impl DistCalculator,
+    prefetch_distance: Option<usize>,
+) -> OrderedNode {
+    let mut current = start.id;
+    let mut closest_dist = start.dist.0;
+    greedy_search_loop!(
+        current,
+        closest_dist,
+        dist_calc,
+        prefetch_distance,
+        |process_neighbor| {
+            let neighbors = graph.neighbors(current);
+            process_neighbors_with_look_ahead(
+                neighbors,
+                process_neighbor,
+                prefetch_distance,
+                dist_calc,
+            );
+        }
+    );
     OrderedNode::new(current, closest_dist.into())
 }
 
diff --git a/rust/lance-index/src/vector/hnsw.rs b/rust/lance-index/src/vector/hnsw.rs
index 73d111cf56c..28960bf6ba3 100644
--- a/rust/lance-index/src/vector/hnsw.rs
+++ b/rust/lance-index/src/vector/hnsw.rs
@@ -12,7 +12,7 @@ use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 
 use self::builder::HnswBuildParams;
-use super::graph::{OrderedFloat, OrderedNode};
+use super::graph::OrderedNode;
 use super::storage::VectorStore;
 
 pub mod builder;
@@ -76,11 +76,7 @@ fn select_neighbors_heuristic(
             break;
         }
 
-        if results.is_empty()
-            || results
-                .iter()
-                .all(|v| u.dist < OrderedFloat(storage.dist_between(u.id, v.id)))
-        {
+        if results.is_empty() || storage.prefers_candidate(u, &results) {
             results.push(u.clone());
         }
     }
diff --git a/rust/lance-index/src/vector/hnsw/builder.rs b/rust/lance-index/src/vector/hnsw/builder.rs
index 06b533b2301..6de41fbc162 100644
--- a/rust/lance-index/src/vector/hnsw/builder.rs
+++ b/rust/lance-index/src/vector/hnsw/builder.rs
@@ -5,7 +5,7 @@
 
 use arrow::array::{AsArray, ListBuilder, UInt32Builder};
 use arrow::compute::concat_batches;
-use arrow::datatypes::{Float32Type, UInt32Type};
+use arrow::datatypes::{DataType, Float32Type, UInt32Type};
 use arrow_array::{ArrayRef, Float32Array, RecordBatch, UInt64Array};
 use crossbeam_queue::ArrayQueue;
 use deepsize::DeepSizeOf;
@@ -31,12 +31,13 @@ use super::super::graph::beam_search;
 use super::{HNSW_TYPE, HnswMetadata, VECTOR_ID_COL, VECTOR_ID_FIELD, select_neighbors_heuristic};
 use crate::metrics::MetricsCollector;
 use crate::prefilter::PreFilter;
-use crate::vector::flat::storage::FlatFloatStorage;
+use crate::vector::flat::storage::{FlatBinStorage, FlatFloatStorage};
 use crate::vector::graph::builder::GraphBuilderNode;
 use crate::vector::graph::{
-    DISTS_FIELD, Graph, NEIGHBORS_COL, NEIGHBORS_FIELD, OrderedFloat, OrderedNode, VisitedGenerator,
+    BorrowingGraph, DISTS_FIELD, Graph, NEIGHBORS_COL, NEIGHBORS_FIELD, OrderedFloat, OrderedNode,
+    VisitedGenerator,
 };
-use crate::vector::graph::{Visited, greedy_search};
+use crate::vector::graph::{Visited, beam_search_borrowed, greedy_search, greedy_search_borrowed};
 use crate::vector::storage::{DistCalculator, VectorStore};
 use crate::vector::v3::subindex::IvfSubIndex;
 use crate::vector::{DIST_COL, Query, VECTOR_RESULT_SCHEMA};
@@ -100,11 +101,25 @@ impl HnswBuildParams {
     /// - `data`: A FixedSizeList to build the HNSW.
     /// - `distance_type`: The distance type to use.
     pub async fn build(self, data: ArrayRef, distance_type: DistanceType) -> Result<HNSW> {
-        let vec_store = Arc::new(FlatFloatStorage::new(
-            data.as_fixed_size_list().clone(),
-            distance_type,
-        ));
-        HNSW::index_vectors(vec_store.as_ref(), self)
+        let vectors = data.as_fixed_size_list().clone();
+        match (vectors.value_type(), distance_type) {
+            (DataType::UInt8, DistanceType::Hamming) => {
+                let vec_store = Arc::new(FlatBinStorage::new(vectors, distance_type));
+                HNSW::index_vectors(vec_store.as_ref(), self)
+            }
+            (DataType::UInt8, _) => Err(Error::invalid_input(format!(
+                "HNSW only supports hamming distance for UInt8 vectors, got {}",
+                distance_type
+            ))),
+            (_, DistanceType::Hamming) => Err(Error::invalid_input(format!(
+                "HNSW hamming distance only supports UInt8 vectors, got {}",
+                vectors.value_type()
+            ))),
+            _ => {
+                let vec_store = Arc::new(FlatFloatStorage::new(vectors, distance_type));
+                HNSW::index_vectors(vec_store.as_ref(), self)
+            }
+        }
     }
 }
 
@@ -117,7 +132,38 @@ impl HnswBuildParams {
 /// Each node in the graph has a global ID which is the index on the base layer.
 #[derive(Clone, DeepSizeOf)]
 pub struct HNSW {
-    inner: Arc<HnswBuilder>,
+    inner: Arc<HnswCore>,
+}
+
+struct HnswCore {
+    params: HnswBuildParams,
+    nodes: Arc<Vec<GraphBuilderNode>>,
+    level_count: Vec<usize>,
+    entry_point: u32,
+    visited_generator_queue: Arc<ArrayQueue<VisitedGenerator>>,
+}
+
+impl DeepSizeOf for HnswCore {
+    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+        self.params.deep_size_of_children(context)
+            + self.nodes.deep_size_of_children(context)
+            + self.level_count.deep_size_of_children(context)
+        // Skipping the visited_generator_queue
+    }
+}
+
+impl HnswCore {
+    fn max_level(&self) -> u16 {
+        self.params.max_level
+    }
+
+    fn num_nodes(&self, level: usize) -> usize {
+        self.level_count[level]
+    }
+
+    fn nodes(&self) -> Arc<Vec<GraphBuilderNode>> {
+        self.nodes.clone()
+    }
 }
 
 impl Debug for HNSW {
@@ -129,7 +175,7 @@ impl Debug for HNSW {
 impl HNSW {
     pub fn empty() -> Self {
         Self {
-            inner: Arc::new(HnswBuilder {
+            inner: Arc::new(HnswCore {
                 params: HnswBuildParams::default(),
                 nodes: Arc::new(Vec::new()),
                 level_count: Vec::new(),
@@ -155,7 +201,7 @@ impl HNSW {
         self.inner.num_nodes(level)
     }
 
-    pub fn nodes(&self) -> Arc<Vec<RwLock<GraphBuilderNode>>> {
+    pub fn nodes(&self) -> Arc<Vec<GraphBuilderNode>> {
         self.inner.nodes()
     }
 
@@ -172,10 +218,10 @@ impl HNSW {
     ) -> Result<Vec<OrderedNode>> {
         let dist_calc = storage.dist_calculator(query, params.dist_q_c);
         let mut ep = OrderedNode::new(0, dist_calc.distance(0).into());
-        let nodes = &self.nodes();
+        let nodes = self.inner.nodes.as_ref();
         for level in (0..self.max_level()).rev() {
-            let cur_level = HnswLevelView::new(level, nodes);
-            ep = greedy_search(
+            let cur_level = ImmutableHnswLevelView::new(level, nodes);
+            ep = greedy_search_borrowed(
                 &cur_level,
                 ep,
                 &dist_calc,
@@ -183,9 +229,9 @@ impl HNSW {
             );
         }
 
-        let bottom_level = HnswBottomView::new(nodes);
+        let bottom_level = ImmutableHnswBottomView::new(nodes);
         let mut visited = visited_generator.generate(storage.len());
-        Ok(beam_search(
+        Ok(beam_search_borrowed(
             &bottom_level,
             &ep,
             params,
@@ -306,10 +352,10 @@ impl HNSW {
             .inner
             .level_count
             .iter()
-            .chain(iter::once(&AtomicUsize::new(0)))
+            .chain(iter::once(&0))
             .scan(0, |state, x| {
                 let start = *state;
-                *state += x.load(Ordering::Relaxed);
+                *state += *x;
                 Some(start)
             })
             .collect();
@@ -343,16 +389,33 @@ impl DeepSizeOf for HnswBuilder {
 }
 
 impl HnswBuilder {
-    fn max_level(&self) -> u16 {
-        self.params.max_level
-    }
+    fn finish(self) -> HNSW {
+        let nodes = match Arc::try_unwrap(self.nodes) {
+            Ok(nodes) => nodes
+                .into_iter()
+                .map(|node| node.into_inner().expect("builder lock poisoned"))
+                .collect(),
+            Err(nodes) => nodes
+                .iter()
+                .map(|node| node.read().expect("builder lock poisoned").clone())
+                .collect(),
+        };
 
-    fn num_nodes(&self, level: usize) -> usize {
-        self.level_count[level].load(Ordering::Relaxed)
-    }
+        let level_count = self
+            .level_count
+            .into_iter()
+            .map(|count| count.load(Ordering::Relaxed))
+            .collect();
 
-    fn nodes(&self) -> Arc<Vec<RwLock<GraphBuilderNode>>> {
-        self.nodes.clone()
+        HNSW {
+            inner: Arc::new(HnswCore {
+                params: self.params,
+                nodes: Arc::new(nodes),
+                level_count,
+                entry_point: self.entry_point,
+                visited_generator_queue: self.visited_generator_queue,
+            }),
+        }
     }
 
     /// Create a new [`HNSWBuilder`] with prepared params and in memory vector storage.
@@ -387,10 +450,11 @@ impl HnswBuilder {
             if len > 0 {
                 nodes.push(RwLock::new(GraphBuilderNode::new(0, max_level as usize)));
             }
+            let mut level_rng = rng();
             for i in 1..len {
                 nodes.push(RwLock::new(GraphBuilderNode::new(
                     i as u32,
-                    builder.random_level() as usize + 1,
+                    builder.random_level(&mut level_rng) as usize + 1,
                 )));
             }
         }
@@ -402,8 +466,7 @@ impl HnswBuilder {
     /// New node's level
     ///
     /// See paper `Algorithm 1`
-    fn random_level(&self) -> u16 {
-        let mut rng = rng();
+    fn random_level<R: Rng + ?Sized>(&self, rng: &mut R) -> u16 {
         let ml = 1.0 / (self.params.m as f32).ln();
         min(
             (-rng.random::<f32>().ln() * ml) as u16,
@@ -486,7 +549,7 @@ impl HnswBuilder {
         ep: &OrderedNode,
         level: u16,
         dist_calc: &impl DistCalculator,
-        nodes: &Vec<RwLock<GraphBuilderNode>>,
+        nodes: &[RwLock<GraphBuilderNode>],
         visited_generator: &mut VisitedGenerator,
     ) -> Vec<OrderedNode> {
         let cur_level = HnswLevelView::new(level, nodes);
@@ -518,7 +581,6 @@ impl HnswBuilder {
         if level_neighbors.len() <= m_max {
             builder_node.update_from_ranked_neighbors(level);
             return;
-            //return level_neighbors;
         }
 
         *neighbors_ranked = select_neighbors_heuristic(storage, &level_neighbors, m_max);
@@ -530,11 +592,11 @@ impl HnswBuilder {
 // This is used to iterate over neighbors in a specific level.
 pub(crate) struct HnswLevelView<'a> {
     level: u16,
-    nodes: &'a Vec<RwLock<GraphBuilderNode>>,
+    nodes: &'a [RwLock<GraphBuilderNode>],
 }
 
 impl<'a> HnswLevelView<'a> {
-    pub fn new(level: u16, nodes: &'a Vec<RwLock<GraphBuilderNode>>) -> Self {
+    pub fn new(level: u16, nodes: &'a [RwLock<GraphBuilderNode>]) -> Self {
         Self { level, nodes }
     }
 }
@@ -550,24 +612,64 @@ impl Graph for HnswLevelView<'_> {
     }
 }
 
-pub(crate) struct HnswBottomView<'a> {
-    nodes: &'a Vec<RwLock<GraphBuilderNode>>,
+pub(crate) struct ImmutableHnswLevelView<'a> {
+    level: u16,
+    nodes: &'a [GraphBuilderNode],
+}
+
+impl<'a> ImmutableHnswLevelView<'a> {
+    pub fn new(level: u16, nodes: &'a [GraphBuilderNode]) -> Self {
+        Self { level, nodes }
+    }
+}
+
+impl Graph for ImmutableHnswLevelView<'_> {
+    fn len(&self) -> usize {
+        self.nodes.len()
+    }
+
+    fn neighbors(&self, key: u32) -> Arc<Vec<u32>> {
+        self.nodes[key as usize].level_neighbors[self.level as usize].clone()
+    }
+}
+
+impl BorrowingGraph for ImmutableHnswLevelView<'_> {
+    fn len(&self) -> usize {
+        self.nodes.len()
+    }
+
+    fn neighbors(&self, key: u32) -> &[u32] {
+        self.nodes[key as usize].level_neighbors[self.level as usize].as_slice()
+    }
+}
+
+pub(crate) struct ImmutableHnswBottomView<'a> {
+    nodes: &'a [GraphBuilderNode],
 }
 
-impl<'a> HnswBottomView<'a> {
-    pub fn new(nodes: &'a Vec<RwLock<GraphBuilderNode>>) -> Self {
+impl<'a> ImmutableHnswBottomView<'a> {
+    pub fn new(nodes: &'a [GraphBuilderNode]) -> Self {
         Self { nodes }
     }
 }
 
-impl Graph for HnswBottomView<'_> {
+impl Graph for ImmutableHnswBottomView<'_> {
     fn len(&self) -> usize {
         self.nodes.len()
     }
 
     fn neighbors(&self, key: u32) -> Arc<Vec<u32>> {
-        let node = &self.nodes[key as usize];
-        node.read().unwrap().bottom_neighbors.clone()
+        self.nodes[key as usize].bottom_neighbors.clone()
+    }
+}
+
+impl BorrowingGraph for ImmutableHnswBottomView<'_> {
+    fn len(&self) -> usize {
+        self.nodes.len()
+    }
+
+    fn neighbors(&self, key: u32) -> &[u32] {
+        self.nodes[key as usize].bottom_neighbors.as_slice()
     }
 }
 
@@ -657,10 +759,10 @@ impl IvfSubIndex for HNSW {
                 .push(VisitedGenerator::new(0))
                 .unwrap();
         }
-        let inner = HnswBuilder {
+        let inner = HnswCore {
             params: hnsw_metadata.params,
-            nodes: Arc::new(nodes.into_iter().map(RwLock::new).collect()),
-            level_count: level_count.into_iter().map(AtomicUsize::new).collect(),
+            nodes: Arc::new(nodes),
+            level_count,
             entry_point: hnsw_metadata.entry_point,
             visited_generator_queue,
         };
@@ -756,35 +858,32 @@ impl IvfSubIndex for HNSW {
     where
         Self: Sized,
     {
-        let inner = HnswBuilder::with_params(params, storage);
-        let hnsw = Self {
-            inner: Arc::new(inner),
-        };
+        let builder = HnswBuilder::with_params(params, storage);
 
         log::debug!(
             "Building HNSW graph: num={}, max_levels={}, m={}, ef_construction={}, distance_type:{}",
             storage.len(),
-            hnsw.inner.params.max_level,
-            hnsw.inner.params.m,
-            hnsw.inner.params.ef_construction,
+            builder.params.max_level,
+            builder.params.m,
+            builder.params.ef_construction,
             storage.distance_type(),
         );
 
         if storage.is_empty() {
-            return Ok(hnsw);
+            return Ok(builder.finish());
         }
 
         let len = storage.len();
-        hnsw.inner.level_count[0].fetch_add(1, Ordering::Relaxed);
+        builder.level_count[0].fetch_add(1, Ordering::Relaxed);
         (1..len).into_par_iter().for_each_init(
             || VisitedGenerator::new(len),
             |visited_generator, node| {
-                hnsw.inner.insert(node as u32, visited_generator, storage);
+                builder.insert(node as u32, visited_generator, storage);
             },
         );
 
-        assert_eq!(hnsw.inner.level_count[0].load(Ordering::Relaxed), len);
-        Ok(hnsw)
+        assert_eq!(builder.level_count[0].load(Ordering::Relaxed), len);
+        Ok(builder.finish())
     }
 
     fn remap(
@@ -807,7 +906,6 @@ impl IvfSubIndex for HNSW {
         for level in 0..self.max_level() {
             let level = level as usize;
             for (id, node) in self.inner.nodes.iter().enumerate() {
-                let node = node.read().unwrap();
                 if level >= node.level_neighbors.len() {
                     continue;
                 }
@@ -850,7 +948,7 @@ impl IvfSubIndex for HNSW {
 mod tests {
     use std::sync::Arc;
 
-    use arrow_array::FixedSizeListArray;
+    use arrow_array::{FixedSizeListArray, UInt8Array};
     use arrow_schema::Schema;
     use lance_arrow::FixedSizeListArrayExt;
     use lance_file::previous::{
@@ -869,7 +967,7 @@ mod tests {
     use crate::scalar::IndexWriter;
     use crate::vector::v3::subindex::IvfSubIndex;
     use crate::vector::{
-        flat::storage::FlatFloatStorage,
+        flat::storage::{FlatBinStorage, FlatFloatStorage},
         graph::{DISTS_FIELD, NEIGHBORS_FIELD},
         hnsw::{
             HNSW, VECTOR_ID_FIELD,
@@ -938,4 +1036,65 @@ mod tests {
             .unwrap();
         assert_eq!(builder_results, loaded_results);
     }
+
+    #[tokio::test]
+    async fn test_builder_write_load_binary_hamming() {
+        const DIM: usize = 8;
+        const TOTAL: usize = 256;
+        const NUM_EDGES: usize = 20;
+        let data = UInt8Array::from_iter_values((0..TOTAL * DIM).map(|v| (v % 16) as u8));
+        let fsl = FixedSizeListArray::try_new_from_values(data, DIM as i32).unwrap();
+        let store = Arc::new(FlatBinStorage::new(fsl.clone(), DistanceType::Hamming));
+        let builder = HnswBuildParams::default()
+            .num_edges(NUM_EDGES)
+            .ef_construction(50)
+            .build(Arc::new(fsl.clone()), DistanceType::Hamming)
+            .await
+            .unwrap();
+
+        let object_store = ObjectStore::memory();
+        let path = Path::from("test_builder_write_load_binary_hamming");
+        let writer = object_store.create(&path).await.unwrap();
+        let schema = Schema::new(vec![
+            VECTOR_ID_FIELD.clone(),
+            NEIGHBORS_FIELD.clone(),
+            DISTS_FIELD.clone(),
+        ]);
+        let schema = lance_core::datatypes::Schema::try_from(&schema).unwrap();
+        let mut writer = PreviousFileWriter::<ManifestDescribing>::with_object_writer(
+            writer,
+            schema,
+            &PreviousFileWriterOptions::default(),
+        )
+        .unwrap();
+        let batch = builder.to_batch().unwrap();
+        let metadata = batch.schema_ref().metadata().clone();
+        writer.write_record_batch(batch).await.unwrap();
+        writer.finish_with_metadata(&metadata).await.unwrap();
+
+        let reader = PreviousFileReader::try_new_self_described(&object_store, &path, None)
+            .await
+            .unwrap();
+        let batch = reader
+            .read_range(0..reader.len(), reader.schema())
+            .await
+            .unwrap();
+        let loaded_hnsw = HNSW::load(batch).unwrap();
+
+        let query = fsl.value(0);
+        let k = 10;
+        let params = HnswQueryParams {
+            ef: 50,
+            lower_bound: None,
+            upper_bound: None,
+            dist_q_c: 0.0,
+        };
+        let builder_results = builder
+            .search_basic(query.clone(), k, &params, None, store.as_ref())
+            .unwrap();
+        let loaded_results = loaded_hnsw
+            .search_basic(query, k, &params, None, store.as_ref())
+            .unwrap();
+        assert_eq!(builder_results, loaded_results);
+    }
 }
diff --git a/rust/lance-index/src/vector/ivf/shuffler.rs b/rust/lance-index/src/vector/ivf/shuffler.rs
index 24928a147d8..f4e03c8f036 100644
--- a/rust/lance-index/src/vector/ivf/shuffler.rs
+++ b/rust/lance-index/src/vector/ivf/shuffler.rs
@@ -31,6 +31,7 @@ use lance_core::cache::LanceCache;
 use lance_core::utils::tokio::get_num_compute_intensive_cpus;
 use lance_core::{Error, ROW_ID, Result, datatypes::Schema};
 use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
+use lance_encoding::version::LanceFileVersion;
 use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_file::previous::writer::FileWriter as PreviousFileWriter;
 use lance_file::reader::{FileReader as Lancev2FileReader, FileReaderOptions};
@@ -388,6 +389,8 @@ pub struct IvfShuffler {
     is_legacy: bool,
 
     shuffle_output_root_filename: String,
+
+    format_version: LanceFileVersion,
 }
 
 /// Represents a range of batches in a file that should be shuffled
@@ -423,9 +426,15 @@ impl IvfShuffler {
             unsorted_buffers: vec![],
             is_legacy,
             shuffle_output_root_filename,
+            format_version: LanceFileVersion::V2_0,
         })
     }
 
+    pub fn with_format_version(mut self, format_version: LanceFileVersion) -> Self {
+        self.format_version = format_version;
+        self
+    }
+
     /// Set the unsorted buffers to be shuffled.
     ///
     /// # Safety
@@ -778,7 +787,10 @@ impl IvfShuffler {
                     let mut file_writer = lance_file::writer::FileWriter::try_new(
                         writer,
                         lance_schema,
-                        FileWriterOptions::default(),
+                        FileWriterOptions {
+                            format_version: Some(this.format_version),
+                            ..Default::default()
+                        },
                     )?;
 
                     for partition_and_idx in shuffled.into_iter().enumerate() {
diff --git a/rust/lance-index/src/vector/pq.rs b/rust/lance-index/src/vector/pq.rs
index e522d9d15d2..3d11fc4a99e 100644
--- a/rust/lance-index/src/vector/pq.rs
+++ b/rust/lance-index/src/vector/pq.rs
@@ -14,7 +14,7 @@ use deepsize::DeepSizeOf;
 use distance::build_distance_table_dot;
 use lance_arrow::*;
 use lance_core::{Error, Result, assume_eq};
-use lance_linalg::distance::{DistanceType, Dot, L2};
+use lance_linalg::distance::{DistanceType, Dot, L2, l2::L2Prepared};
 use lance_table::utils::LanceIteratorExtension;
 use num_traits::Float;
 use prost::Message;
@@ -27,7 +27,9 @@ pub mod storage;
 pub mod transform;
 pub(crate) mod utils;
 
-use self::distance::{build_distance_table_l2, compute_pq_distance};
+use self::distance::{
+    build_distance_table_l2, build_distance_table_l2_prepared, compute_pq_distance,
+};
 pub use self::utils::num_centroids;
 use super::quantizer::{
     Quantization, QuantizationMetadata, QuantizationType, Quantizer, QuantizerBuildParams,
@@ -44,6 +46,10 @@ pub struct ProductQuantizer {
     pub dimension: usize,
     pub codebook: FixedSizeListArray,
     pub distance_type: DistanceType,
+    /// Pre-transposed L2 targets per sub-vector for fast f32 L2 batch computation.
+    /// Only populated when codebook is f32 and distance_type is L2
+    /// (Cosine is converted to L2 before construction, so it benefits too).
+    l2_targets: Option<Vec<L2Prepared>>,
 }
 
 impl DeepSizeOf for ProductQuantizer {
@@ -53,10 +59,43 @@ impl DeepSizeOf for ProductQuantizer {
             + self.num_bits.deep_size_of_children(_context)
             + self.dimension.deep_size_of_children(_context)
             + self.distance_type.deep_size_of_children(_context)
+            + self
+                .l2_targets
+                .as_ref()
+                .map_or(0, |v| v.iter().map(|t| t.size_bytes()).sum())
     }
 }
 
 impl ProductQuantizer {
+    /// Build per-sub-vector L2Prepared from the codebook if applicable (f32 + L2).
+    fn build_l2_targets(
+        codebook: &FixedSizeListArray,
+        distance_type: DistanceType,
+        num_sub_vectors: usize,
+        num_bits: u32,
+        dimension: usize,
+    ) -> Option<Vec<L2Prepared>> {
+        if codebook.value_type() != DataType::Float32 || distance_type != DistanceType::L2 {
+            return None;
+        }
+        let values = codebook
+            .values()
+            .as_primitive::<datatypes::Float32Type>()
+            .values();
+        let sub_dim = dimension / num_sub_vectors;
+        let num_centroids = 2_usize.pow(num_bits);
+        let block_size = sub_dim * num_centroids;
+
+        let targets = (0..num_sub_vectors)
+            .map(|sub_idx| {
+                let block_start = sub_idx * block_size;
+                let block = &values[block_start..block_start + block_size];
+                L2Prepared::new(block, sub_dim)
+            })
+            .collect();
+        Some(targets)
+    }
+
     pub fn new(
         num_sub_vectors: usize,
         num_bits: u32,
@@ -64,12 +103,20 @@ impl ProductQuantizer {
         codebook: FixedSizeListArray,
         distance_type: DistanceType,
     ) -> Self {
+        let l2_targets = Self::build_l2_targets(
+            &codebook,
+            distance_type,
+            num_sub_vectors,
+            num_bits,
+            dimension,
+        );
         Self {
             num_bits,
             num_sub_vectors,
             dimension,
             codebook,
             distance_type,
+            l2_targets,
         }
     }
 
@@ -85,13 +132,13 @@ impl ProductQuantizer {
                 proto.dimension as i32,
             )?,
         };
-        Ok(Self {
-            num_bits: proto.num_bits,
-            num_sub_vectors: proto.num_sub_vectors as usize,
-            dimension: proto.dimension as usize,
+        Ok(Self::new(
+            proto.num_sub_vectors as usize,
+            proto.num_bits,
+            proto.dimension as usize,
             codebook,
             distance_type,
-        })
+        ))
     }
 
     #[instrument(name = "ProductQuantizer::transform", level = "debug", skip_all)]
@@ -136,38 +183,74 @@ impl ProductQuantizer {
 
         let flatten_data = fsl.values().as_primitive::<T>();
         let sub_dim = dim / num_sub_vectors;
+        let num_centroids = 2_usize.pow(NUM_BITS);
         let total_code_length = fsl.len() * num_sub_vectors / (8 / NUM_BITS as usize);
-        let values = flatten_data
-            .values()
-            .chunks_exact(dim)
-            .flat_map(|vector| {
-                let sub_vec_code = vector
-                    .chunks_exact(sub_dim)
-                    .enumerate()
-                    .map(|(sub_idx, sub_vector)| {
-                        let centroids = get_sub_vector_centroids::<NUM_BITS, _>(
-                            codebook.values(),
-                            dim,
-                            num_sub_vectors,
-                            sub_idx,
-                        );
-                        // SAFETY: The must be 2^NUM_BITS centroids, it's safe to unwrap_or(0),
-                        // this could happen if all distances are INFs in the case of vectors are large.
-                        assume_eq!(centroids.len(), 2_usize.pow(NUM_BITS) * sub_dim);
-                        compute_partition(centroids, sub_vector, distance_type).unwrap_or(0) as u8
-                    })
-                    .collect::<Vec<_>>();
+
+        let values = if let Some(targets) = &self.l2_targets {
+            // Fast path for f32 + L2: use pre-transposed codebook.
+            // SAFETY: l2_targets is only populated when T::Native is f32.
+            let flat_f32: &[f32] = unsafe {
+                std::slice::from_raw_parts(
+                    flatten_data.values().as_ptr() as *const f32,
+                    flatten_data.values().len(),
+                )
+            };
+            let mut values = vec![0u8; total_code_length];
+            let bytes_per_vector = num_sub_vectors / (8 / NUM_BITS as usize);
+            let mut dist_buf = vec![0.0f32; num_centroids];
+            for (vec_idx, vector) in flat_f32.chunks_exact(dim).enumerate() {
+                let out = &mut values[vec_idx * bytes_per_vector..][..bytes_per_vector];
                 if NUM_BITS == 4 {
-                    sub_vec_code
-                        .chunks_exact(2)
-                        .map(|v| (v[1] << 4) | v[0])
-                        .collect::<Vec<_>>()
+                    for (pair_idx, pair) in vector.chunks_exact(sub_dim * 2).enumerate() {
+                        let lo = targets[pair_idx * 2]
+                            .nearest_into(&pair[..sub_dim], &mut dist_buf)
+                            .unwrap_or(0) as u8;
+                        let hi = targets[pair_idx * 2 + 1]
+                            .nearest_into(&pair[sub_dim..], &mut dist_buf)
+                            .unwrap_or(0) as u8;
+                        out[pair_idx] = (hi << 4) | lo;
+                    }
                 } else {
-                    sub_vec_code
+                    for (sub_idx, sv) in vector.chunks_exact(sub_dim).enumerate() {
+                        out[sub_idx] = targets[sub_idx]
+                            .nearest_into(sv, &mut dist_buf)
+                            .unwrap_or(0) as u8;
+                    }
                 }
-            })
-            .exact_size(total_code_length)
-            .collect::<Vec<_>>();
+            }
+            values
+        } else {
+            flatten_data
+                .values()
+                .chunks_exact(dim)
+                .flat_map(|vector| {
+                    let sub_vec_code: Vec<u8> = vector
+                        .chunks_exact(sub_dim)
+                        .enumerate()
+                        .map(|(sub_idx, sub_vector)| {
+                            let centroids = get_sub_vector_centroids::<NUM_BITS, _>(
+                                codebook.values(),
+                                dim,
+                                num_sub_vectors,
+                                sub_idx,
+                            );
+                            assume_eq!(centroids.len(), num_centroids * sub_dim);
+                            compute_partition(centroids, sub_vector, distance_type).unwrap_or(0)
+                                as u8
+                        })
+                        .collect();
+                    if NUM_BITS == 4 {
+                        sub_vec_code
+                            .chunks_exact(2)
+                            .map(|v| (v[1] << 4) | v[0])
+                            .collect::<Vec<_>>()
+                    } else {
+                        sub_vec_code
+                    }
+                })
+                .exact_size(total_code_length)
+                .collect::<Vec<_>>()
+        };
 
         let num_sub_vectors_in_byte = if NUM_BITS == 4 {
             num_sub_vectors / 2
@@ -216,15 +299,7 @@ impl ProductQuantizer {
     /// It returns the squared L2 distance.
     fn l2_distances(&self, key: &dyn Array, code: &UInt8Array) -> Result<Float32Array> {
         let distance_table = self.build_l2_distance_table(key)?;
-
-        #[cfg(target_feature = "avx512f")]
-        {
-            Ok(self.compute_l2_distance(&distance_table, code.values()))
-        }
-        #[cfg(not(target_feature = "avx512f"))]
-        {
-            Ok(self.compute_l2_distance(&distance_table, code.values()))
-        }
+        Ok(self.compute_l2_distance(&distance_table, code.values()))
     }
 
     /// Parameters
@@ -284,7 +359,13 @@ impl ProductQuantizer {
                 Ok(self.build_l2_distance_table_impl::<datatypes::Float16Type>(key.as_primitive()))
             }
             DataType::Float32 => {
-                Ok(self.build_l2_distance_table_impl::<datatypes::Float32Type>(key.as_primitive()))
+                if let Some(targets) = &self.l2_targets {
+                    let query = key.as_primitive::<datatypes::Float32Type>().values();
+                    Ok(build_distance_table_l2_prepared(targets, query))
+                } else {
+                    Ok(self
+                        .build_l2_distance_table_impl::<datatypes::Float32Type>(key.as_primitive()))
+                }
             }
             DataType::Float64 => {
                 Ok(self.build_l2_distance_table_impl::<datatypes::Float64Type>(key.as_primitive()))
diff --git a/rust/lance-index/src/vector/pq/distance.rs b/rust/lance-index/src/vector/pq/distance.rs
index dd76c170e59..b341ba98af7 100644
--- a/rust/lance-index/src/vector/pq/distance.rs
+++ b/rust/lance-index/src/vector/pq/distance.rs
@@ -4,9 +4,9 @@
 use core::panic;
 use std::cmp::{max, min};
 
-use super::{num_centroids, utils::get_sub_vector_centroids};
+use super::utils::get_sub_vector_centroids;
 use lance_core::assume_eq;
-use lance_linalg::distance::{Dot, L2, dot_distance_batch, l2_distance_batch};
+use lance_linalg::distance::{Dot, L2, dot_distance_batch, l2::L2Prepared, l2_distance_batch};
 use lance_linalg::simd::u8::u8x16;
 use lance_linalg::simd::{SIMD, Shuffle};
 
@@ -54,6 +54,21 @@ pub fn build_distance_table_l2_impl<const NUM_BITS: u32, T: L2>(
     result
 }
 
+/// Build an L2 distance table using pre-prepared [L2Prepared] per sub-vector.
+///
+/// This avoids the per-call AoS→SoA transpose by reusing targets that were
+/// transposed once at `ProductQuantizer` construction time.
+pub fn build_distance_table_l2_prepared(l2_targets: &[L2Prepared], query: &[f32]) -> Vec<f32> {
+    let sub_dim = query.len() / l2_targets.len();
+    let num_targets = l2_targets[0].num_targets();
+
+    let mut result = vec![0.0f32; l2_targets.len() * num_targets];
+    for (i, sub_vec) in query.chunks_exact(sub_dim).enumerate() {
+        l2_targets[i].distances_into(sub_vec, &mut result[i * num_targets..][..num_targets]);
+    }
+    result
+}
+
 /// Build a Distance Table from the query to each PQ centroid
 /// using Dot distance.
 pub fn build_distance_table_dot<T: Dot>(
@@ -291,14 +306,14 @@ fn quantize_distance_table(distance_table: &[f32], qmax: f32) -> (f32, Vec<u8>)
 /// - C: the tile size of code-book to run at once.
 /// - V: the tile size of PQ code to run at once.
 ///
-#[allow(dead_code)]
+#[cfg(test)]
 fn compute_l2_distance_without_transposing<const C: usize, const V: usize>(
     distance_table: &[f32],
     num_bits: u32,
     num_sub_vectors: usize,
     code: &[u8],
 ) -> Vec<f32> {
-    let num_centroids = num_centroids(num_bits);
+    let num_centroids = super::num_centroids(num_bits);
     let iter = code.chunks_exact(num_sub_vectors * V);
     let distances = iter.clone().flat_map(|c| {
         let mut sums = [0.0_f32; V];
diff --git a/rust/lance-index/src/vector/pq/storage.rs b/rust/lance-index/src/vector/pq/storage.rs
index 3495ba8c8c1..14233abc0c0 100644
--- a/rust/lance-index/src/vector/pq/storage.rs
+++ b/rust/lance-index/src/vector/pq/storage.rs
@@ -34,6 +34,7 @@ use serde::{Deserialize, Serialize};
 use super::ProductQuantizer;
 use super::distance::{build_distance_table_dot, build_distance_table_l2, compute_pq_distance};
 use crate::frag_reuse::FragReuseIndex;
+use crate::vector::graph::{OrderedFloat, OrderedNode};
 use crate::{
     INDEX_METADATA_SCHEMA_KEY, IndexMetadata, pb,
     vector::{
@@ -106,11 +107,19 @@ impl QuantizerMetadata for ProductQuantizationMetadata {
     }
 
     fn extra_metadata(&self) -> Result<Option<Bytes>> {
-        debug_assert!(self.codebook.is_some());
-        let codebook_tensor: pb::Tensor = pb::Tensor::try_from(self.codebook.as_ref().unwrap())?;
-        let mut bytes = BytesMut::new();
-        codebook_tensor.encode(&mut bytes)?;
-        Ok(Some(bytes.freeze()))
+        if let Some(codebook) = &self.codebook {
+            let codebook_tensor: pb::Tensor = pb::Tensor::try_from(codebook)?;
+            let mut bytes = BytesMut::new();
+            codebook_tensor.encode(&mut bytes)?;
+            Ok(Some(bytes.freeze()))
+        } else if !self.codebook_tensor.is_empty() {
+            // Legacy format: codebook is stored inline in the metadata JSON.
+            // Return it as-is; it's already a protobuf-encoded Tensor that
+            // parse_buffer() can handle.
+            Ok(Some(Bytes::from(self.codebook_tensor.clone())))
+        } else {
+            Ok(None)
+        }
     }
 
     async fn load(reader: &PreviousFileReader) -> Result<Self> {
@@ -824,6 +833,12 @@ impl VectorStore for ProductQuantizationStorage {
             _ => unimplemented!("Unsupported data type: {:?}", codebook.value_type()),
         }
     }
+
+    fn prefers_candidate(&self, candidate: &OrderedNode, selected: &[OrderedNode]) -> bool {
+        selected
+            .iter()
+            .all(|other| candidate.dist < OrderedFloat(self.dist_between(candidate.id, other.id)))
+    }
 }
 
 /// Distance calculator backed by PQ code.
diff --git a/rust/lance-index/src/vector/quantizer.rs b/rust/lance-index/src/vector/quantizer.rs
index 887269b19c9..8e2dd333606 100644
--- a/rust/lance-index/src/vector/quantizer.rs
+++ b/rust/lance-index/src/vector/quantizer.rs
@@ -65,6 +65,7 @@ pub trait Quantization:
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum QuantizationType {
     Flat,
+    FlatBin,
     Product,
     Scalar,
     Rabit,
@@ -76,6 +77,7 @@ impl FromStr for QuantizationType {
     fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
         match s {
             "FLAT" => Ok(Self::Flat),
+            "FLATBIN" => Ok(Self::FlatBin),
             "PQ" => Ok(Self::Product),
             "SQ" => Ok(Self::Scalar),
             "RABIT" => Ok(Self::Rabit),
@@ -88,6 +90,7 @@ impl std::fmt::Display for QuantizationType {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Self::Flat => write!(f, "FLAT"),
+            Self::FlatBin => write!(f, "FLATBIN"),
             Self::Product => write!(f, "PQ"),
             Self::Scalar => write!(f, "SQ"),
             Self::Rabit => write!(f, "RQ"),
@@ -156,7 +159,7 @@ impl Quantizer {
     pub fn quantization_type(&self) -> QuantizationType {
         match self {
             Self::Flat(_) => QuantizationType::Flat,
-            Self::FlatBin(_) => QuantizationType::Flat,
+            Self::FlatBin(_) => QuantizationType::FlatBin,
             Self::Product(_) => QuantizationType::Product,
             Self::Scalar(_) => QuantizationType::Scalar,
             Self::Rabit(_) => QuantizationType::Rabit,
@@ -313,7 +316,6 @@ impl<Q: Quantization> Clone for IvfQuantizationStorage<Q> {
     }
 }
 
-#[allow(dead_code)]
 impl<Q: Quantization> IvfQuantizationStorage<Q> {
     /// Open a Loader.
     ///
diff --git a/rust/lance-index/src/vector/shared/partition_merger.rs b/rust/lance-index/src/vector/shared/partition_merger.rs
index 9ca9469551c..a94c462e46b 100644
--- a/rust/lance-index/src/vector/shared/partition_merger.rs
+++ b/rust/lance-index/src/vector/shared/partition_merger.rs
@@ -17,6 +17,7 @@ use lance_linalg::distance::DistanceType;
 use prost::Message;
 
 use crate::pb;
+use crate::vector::bq::storage::RABIT_METADATA_KEY;
 use crate::vector::ivf::storage::{IVF_METADATA_KEY, IvfModel};
 use crate::vector::pq::storage::PQ_METADATA_KEY;
 use crate::vector::sq::storage::SQ_METADATA_KEY;
@@ -34,6 +35,7 @@ pub enum SupportedIvfIndexType {
     IvfFlat,
     IvfPq,
     IvfSq,
+    IvfRq,
     IvfHnswFlat,
     IvfHnswPq,
     IvfHnswSq,
@@ -46,6 +48,7 @@ impl SupportedIvfIndexType {
             Self::IvfFlat => "IVF_FLAT",
             Self::IvfPq => "IVF_PQ",
             Self::IvfSq => "IVF_SQ",
+            Self::IvfRq => "IVF_RQ",
             Self::IvfHnswFlat => "IVF_HNSW_FLAT",
             Self::IvfHnswPq => "IVF_HNSW_PQ",
             Self::IvfHnswSq => "IVF_HNSW_SQ",
@@ -60,6 +63,7 @@ impl SupportedIvfIndexType {
             "IVF_FLAT" => Some(Self::IvfFlat),
             "IVF_PQ" => Some(Self::IvfPq),
             "IVF_SQ" => Some(Self::IvfSq),
+            "IVF_RQ" => Some(Self::IvfRq),
             "IVF_HNSW_FLAT" => Some(Self::IvfHnswFlat),
             "IVF_HNSW_PQ" => Some(Self::IvfHnswPq),
             "IVF_HNSW_SQ" => Some(Self::IvfHnswSq),
@@ -74,6 +78,10 @@ impl SupportedIvfIndexType {
     pub fn detect_from_reader_and_schema(reader: &V2Reader, schema: &ArrowSchema) -> Result<Self> {
         let has_pq_code_col = schema.fields.iter().any(|f| f.name() == PQ_CODE_COLUMN);
         let has_sq_code_col = schema.fields.iter().any(|f| f.name() == SQ_CODE_COLUMN);
+        let has_rq_code_col = schema
+            .fields
+            .iter()
+            .any(|f| f.name() == crate::vector::bq::storage::RABIT_CODE_COLUMN);
 
         let is_pq = reader
             .metadata()
@@ -87,19 +95,26 @@ impl SupportedIvfIndexType {
             .metadata
             .contains_key(SQ_METADATA_KEY)
             || has_sq_code_col;
+        let is_rq = reader
+            .metadata()
+            .file_schema
+            .metadata
+            .contains_key(RABIT_METADATA_KEY)
+            || has_rq_code_col;
 
         // Detect HNSW-related columns
         let has_hnsw_vector_id_col = schema.fields.iter().any(|f| f.name() == "__vector_id");
         let has_hnsw_pointer_col = schema.fields.iter().any(|f| f.name() == "__pointer");
         let has_hnsw = has_hnsw_vector_id_col || has_hnsw_pointer_col;
 
-        let index_type = match (has_hnsw, is_pq, is_sq) {
-            (false, false, false) => Self::IvfFlat,
-            (false, true, false) => Self::IvfPq,
-            (false, false, true) => Self::IvfSq,
-            (true, false, false) => Self::IvfHnswFlat,
-            (true, true, false) => Self::IvfHnswPq,
-            (true, false, true) => Self::IvfHnswSq,
+        let index_type = match (has_hnsw, is_pq, is_sq, is_rq) {
+            (false, false, false, false) => Self::IvfFlat,
+            (false, true, false, false) => Self::IvfPq,
+            (false, false, true, false) => Self::IvfSq,
+            (false, false, false, true) => Self::IvfRq,
+            (true, false, false, false) => Self::IvfHnswFlat,
+            (true, true, false, false) => Self::IvfHnswPq,
+            (true, false, true, false) => Self::IvfHnswSq,
             _ => {
                 return Err(Error::not_supported_source(
                     "Unsupported index type combination detected".into(),
diff --git a/rust/lance-index/src/vector/sq/storage.rs b/rust/lance-index/src/vector/sq/storage.rs
index 8f6bcea6c1a..8311c20acaa 100644
--- a/rust/lance-index/src/vector/sq/storage.rs
+++ b/rust/lance-index/src/vector/sq/storage.rs
@@ -36,7 +36,7 @@ use crate::{
 
 pub const SQ_METADATA_KEY: &str = "lance:sq";
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct ScalarQuantizationMetadata {
     pub dim: usize,
     pub num_bits: u16,
diff --git a/rust/lance-index/src/vector/storage.rs b/rust/lance-index/src/vector/storage.rs
index 598e8f95ca9..5ad20f0ccd7 100644
--- a/rust/lance-index/src/vector/storage.rs
+++ b/rust/lance-index/src/vector/storage.rs
@@ -28,6 +28,8 @@ use crate::{
 };
 
 use super::DISTANCE_TYPE_KEY;
+use super::graph::OrderedFloat;
+use super::graph::OrderedNode;
 use super::quantizer::{Quantizer, QuantizerMetadata};
 
 /// <section class="warning">
@@ -102,6 +104,13 @@ pub trait VectorStore: Send + Sync + Sized + Clone {
         let dist_cal_u = self.dist_calculator_from_id(u);
         dist_cal_u.distance(v)
     }
+
+    fn prefers_candidate(&self, candidate: &OrderedNode, selected: &[OrderedNode]) -> bool {
+        let dist_cal_candidate = self.dist_calculator_from_id(candidate.id);
+        selected
+            .iter()
+            .all(|other| candidate.dist < OrderedFloat(dist_cal_candidate.distance(other.id)))
+    }
 }
 
 pub struct StorageBuilder<Q: Quantization> {
@@ -230,6 +239,32 @@ impl<Q: Quantization> IvfQuantizationStorage<Q> {
         })
     }
 
+    /// Construct from pre-parsed metadata, skipping global buffer reads.
+    /// Used when reconstructing from a disk cache.
+    pub fn from_cached(
+        reader: FileReader,
+        ivf: IvfModel,
+        metadata: Q::Metadata,
+        distance_type: DistanceType,
+        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+    ) -> Self {
+        Self {
+            reader,
+            distance_type,
+            metadata,
+            ivf,
+            frag_reuse_index,
+        }
+    }
+
+    pub fn reader(&self) -> &FileReader {
+        &self.reader
+    }
+
+    pub fn ivf(&self) -> &IvfModel {
+        &self.ivf
+    }
+
     pub fn num_rows(&self) -> u64 {
         self.reader.num_rows()
     }
diff --git a/rust/lance-index/src/vector/utils.rs b/rust/lance-index/src/vector/utils.rs
index 0ec4eba4f2b..1e56370613e 100644
--- a/rust/lance-index/src/vector/utils.rs
+++ b/rust/lance-index/src/vector/utils.rs
@@ -1,11 +1,9 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use arrow::{
-    array::AsArray,
-    datatypes::{Float16Type, Float32Type, Float64Type},
-};
-use arrow_array::{Array, ArrayRef, BooleanArray, FixedSizeListArray};
+use arrow::compute::cast;
+use arrow_array::types::{Float16Type, Float32Type, Float64Type};
+use arrow_array::{Array, ArrayRef, BooleanArray, FixedSizeListArray, cast::AsArray};
 use arrow_schema::{DataType, Field};
 use lance_arrow::FixedSizeListArrayExt;
 use lance_core::{Error, Result};
@@ -17,6 +15,7 @@ use std::{ops::Range, sync::Arc};
 
 use super::pb;
 use crate::pb::Tensor;
+use crate::vector::flat::storage::FlatBinStorage;
 use crate::vector::flat::storage::FlatFloatStorage;
 use crate::vector::hnsw::HNSW;
 use crate::vector::hnsw::builder::{HnswBuildParams, HnswQueryParams};
@@ -44,22 +43,34 @@ static USE_HNSW_SPEEDUP_INDEXING: LazyLock<SimpleIndexStatus> = LazyLock::new(||
 
 #[derive(Debug)]
 pub struct SimpleIndex {
-    store: FlatFloatStorage,
+    store: SimpleStore,
     index: HNSW,
 }
 
+#[derive(Debug)]
+enum SimpleStore {
+    Float(FlatFloatStorage),
+    Binary(FlatBinStorage),
+}
+
 impl SimpleIndex {
-    pub fn try_new(store: FlatFloatStorage) -> Result<Self> {
-        let hnsw = HNSW::index_vectors(
-            &store,
-            HnswBuildParams::default().ef_construction(15).num_edges(12),
-        )?;
+    fn try_new(store: SimpleStore) -> Result<Self> {
+        let hnsw = match &store {
+            SimpleStore::Float(store) => HNSW::index_vectors(
+                store,
+                HnswBuildParams::default().ef_construction(15).num_edges(12),
+            )?,
+            SimpleStore::Binary(store) => HNSW::index_vectors(
+                store,
+                HnswBuildParams::default().ef_construction(15).num_edges(12),
+            )?,
+        };
         Ok(Self { store, index: hnsw })
     }
 
     // train HNSW over the centroids to speed up finding the nearest clusters,
     // only train if all conditions are met:
-    //  - the centroids are float32s or uint8s
+    //  - the centroids are float16/float32 or uint8 with hamming distance
     //  - `num_centroids * dimension >= 1_000_000`
     //      we benchmarked that it's 2x faster in the case of 1024 centroids and 1024 dimensions,
     //      so set the threshold to 1_000_000.
@@ -78,65 +89,42 @@ impl SimpleIndex {
             _ => {}
         }
 
-        match centroids.data_type() {
-            DataType::Float32 => {
-                let fsl =
-                    FixedSizeListArray::try_new_from_values(centroids.clone(), dimension as i32)?;
-                let store = FlatFloatStorage::new(fsl, distance_type);
-                Self::try_new(store).map(Some)
+        let store = match (centroids.data_type(), distance_type) {
+            (DataType::Float16 | DataType::Float32 | DataType::Float64, _) => {
+                let fsl = FixedSizeListArray::try_new_from_values(centroids, dimension as i32)?;
+                SimpleStore::Float(FlatFloatStorage::new(fsl, distance_type))
             }
-            _ => Ok(None),
-        }
+            (DataType::UInt8, DistanceType::Hamming) => {
+                let fsl = FixedSizeListArray::try_new_from_values(centroids, dimension as i32)?;
+                SimpleStore::Binary(FlatBinStorage::new(fsl, distance_type))
+            }
+            _ => return Ok(None),
+        };
+        Self::try_new(store).map(Some)
     }
 
     pub(crate) fn search(&self, query: ArrayRef) -> Result<(u32, f32)> {
-        let res = self.index.search_basic(
-            query,
-            1,
-            &HnswQueryParams {
-                ef: 15,
-                lower_bound: None,
-                upper_bound: None,
-                dist_q_c: 0.0,
-            },
-            None,
-            &self.store,
-        )?;
+        let params = HnswQueryParams {
+            ef: 15,
+            lower_bound: None,
+            upper_bound: None,
+            dist_q_c: 0.0,
+        };
+        let res = match &self.store {
+            SimpleStore::Float(store) => self.index.search_basic(query, 1, &params, None, store)?,
+            SimpleStore::Binary(store) => {
+                let query = if query.data_type() == &DataType::UInt8 {
+                    query
+                } else {
+                    cast(&query, &DataType::UInt8).map_err(|e| Error::index(e.to_string()))?
+                };
+                self.index.search_basic(query, 1, &params, None, store)?
+            }
+        };
         Ok((res[0].id, res[0].dist.0))
     }
 }
 
-#[inline]
-#[allow(dead_code)]
-pub(crate) fn prefetch_arrow_array(array: &dyn Array) -> Result<()> {
-    match array.data_type() {
-        DataType::FixedSizeList(_, _) => {
-            let array = array.as_fixed_size_list();
-            return prefetch_arrow_array(array.values());
-        }
-        DataType::Float16 => {
-            let array = array.as_primitive::<Float16Type>();
-            do_prefetch(array.values().as_ptr_range())
-        }
-        DataType::Float32 => {
-            let array = array.as_primitive::<Float32Type>();
-            do_prefetch(array.values().as_ptr_range())
-        }
-        DataType::Float64 => {
-            let array = array.as_primitive::<Float64Type>();
-            do_prefetch(array.values().as_ptr_range())
-        }
-        _ => {
-            return Err(Error::invalid_input(format!(
-                "Unsupported data type for prefetch: {}",
-                array.data_type()
-            )));
-        }
-    }
-
-    Ok(())
-}
-
 #[inline]
 pub(crate) fn do_prefetch<T>(ptrs: Range<*const T>) {
     // TODO use rust intrinsics instead of x86 intrinsics
@@ -205,7 +193,7 @@ impl TryFrom<&FixedSizeListArray> for pb::Tensor {
     fn try_from(array: &FixedSizeListArray) -> Result<Self> {
         let mut tensor = Self::default();
         tensor.data_type = pb::tensor::DataType::try_from(array.value_type())? as i32;
-        tensor.shape = vec![array.len() as u32, array.value_length() as u32];
+        tensor.shape = vec![Array::len(array) as u32, array.value_length() as u32];
         let flat_array = array.values();
         tensor.data = flat_array.into_data().buffers()[0].to_vec();
         Ok(tensor)
@@ -263,17 +251,17 @@ pub fn is_finite(fsl: &FixedSizeListArray) -> BooleanArray {
             Some(v) => match v.data_type() {
                 DataType::Float16 => {
                     let v = v.as_primitive::<Float16Type>();
-                    v.null_count() == 0 && v.values().iter().all(|v| v.is_finite())
+                    Array::null_count(v) == 0 && v.values().iter().all(|v| v.is_finite())
                 }
                 DataType::Float32 => {
                     let v = v.as_primitive::<Float32Type>();
-                    v.null_count() == 0 && v.values().iter().all(|v| v.is_finite())
+                    Array::null_count(v) == 0 && v.values().iter().all(|v| v.is_finite())
                 }
                 DataType::Float64 => {
                     let v = v.as_primitive::<Float64Type>();
-                    v.null_count() == 0 && v.values().iter().all(|v| v.is_finite())
+                    Array::null_count(v) == 0 && v.values().iter().all(|v| v.is_finite())
                 }
-                _ => v.null_count() == 0,
+                _ => Array::null_count(&v) == 0,
             },
             None => false,
         })
@@ -285,11 +273,74 @@ pub fn is_finite(fsl: &FixedSizeListArray) -> BooleanArray {
 mod tests {
     use super::*;
 
-    use arrow_array::{Float16Array, Float32Array, Float64Array};
+    use arrow_array::{Float16Array, Float32Array, Float64Array, UInt8Array};
     use half::f16;
     use lance_arrow::FixedSizeListArrayExt;
     use num_traits::identities::Zero;
 
+    use arrow::compute::cast;
+    use rstest::rstest;
+
+    fn build_index(centroids: ArrayRef, dim: usize) -> SimpleIndex {
+        let f32_centroids = cast(&centroids, &DataType::Float32).unwrap();
+        let fsl = FixedSizeListArray::try_new_from_values(f32_centroids, dim as i32).unwrap();
+        let store = SimpleStore::Float(FlatFloatStorage::new(fsl, DistanceType::L2));
+        SimpleIndex::try_new(store).unwrap()
+    }
+
+    fn build_binary_index(centroids: ArrayRef, dim: usize) -> SimpleIndex {
+        let u8_centroids = if centroids.data_type() == &DataType::UInt8 {
+            centroids
+        } else {
+            cast(&centroids, &DataType::UInt8).unwrap()
+        };
+        let fsl = FixedSizeListArray::try_new_from_values(u8_centroids, dim as i32).unwrap();
+        let store = SimpleStore::Binary(FlatBinStorage::new(fsl, DistanceType::Hamming));
+        SimpleIndex::try_new(store).unwrap()
+    }
+
+    #[rstest]
+    #[case::f16(Arc::new(Float16Array::from(
+        (0..100).flat_map(|i| std::iter::repeat_n(f16::from_f32(i as f32), 16)).collect::<Vec<_>>(),
+    )) as ArrayRef)]
+    #[case::f32(Arc::new(Float32Array::from(
+        (0..100).flat_map(|i| std::iter::repeat_n(i as f32, 16)).collect::<Vec<_>>(),
+    )) as ArrayRef)]
+    fn test_simple_index_nearest_centroid(#[case] centroids: ArrayRef) {
+        let index = build_index(centroids, 16);
+        let query: ArrayRef = Arc::new(Float32Array::from(vec![42.1f32; 16]));
+        let (id, _) = index.search(query).unwrap();
+        assert_eq!(id, 42);
+    }
+
+    #[test]
+    fn test_simple_index_nearest_centroid_binary() {
+        let centroids: ArrayRef = Arc::new(UInt8Array::from(
+            (0..100)
+                .flat_map(|i| std::iter::repeat_n(i as u8, 16))
+                .collect::<Vec<_>>(),
+        ));
+        let index = build_binary_index(centroids, 16);
+        let query: ArrayRef = Arc::new(UInt8Array::from(vec![42u8; 16]));
+        let (id, dist) = index.search(query).unwrap();
+        assert_eq!(id, 42);
+        assert_eq!(dist, 0.0);
+    }
+
+    #[test]
+    fn test_simple_index_rejects_f64() {
+        let centroids: ArrayRef = Arc::new(Float64Array::from(vec![0.0; 1600]));
+        let result = SimpleIndex::may_train_index(centroids, 16, DistanceType::L2).unwrap();
+        assert!(result.is_none());
+    }
+
+    #[test]
+    fn test_simple_index_rejects_uint8_non_hamming() {
+        let centroids: ArrayRef = Arc::new(UInt8Array::from(vec![0u8; 1600]));
+        let result = SimpleIndex::may_train_index(centroids, 16, DistanceType::L2).unwrap();
+        assert!(result.is_none());
+    }
+
     #[test]
     fn test_fsl_to_tensor() {
         let fsl =
diff --git a/rust/lance-index/src/vector/v3/shuffler.rs b/rust/lance-index/src/vector/v3/shuffler.rs
index 9e37ec1ad59..0bf714df237 100644
--- a/rust/lance-index/src/vector/v3/shuffler.rs
+++ b/rust/lance-index/src/vector/v3/shuffler.rs
@@ -4,12 +4,17 @@
 //! Shuffler is a component that takes a stream of record batches and shuffles them into
 //! the corresponding IVF partitions.
 
-use std::sync::Arc;
+use std::ops::Range;
+use std::sync::atomic::AtomicU64;
+use std::sync::{Arc, Mutex};
 
+use arrow::compute::concat_batches;
+use arrow::datatypes::UInt64Type;
 use arrow::{array::AsArray, compute::sort_to_indices};
-use arrow_array::{RecordBatch, UInt32Array};
-use arrow_schema::Schema;
+use arrow_array::{RecordBatch, UInt32Array, UInt64Array};
+use arrow_schema::{DataType, Field, Schema};
 use futures::{future::try_join_all, prelude::*};
+use lance_arrow::stream::rechunk_stream_by_size;
 use lance_arrow::{RecordBatchExt, SchemaExt};
 use lance_core::{
     Error, Result,
@@ -17,9 +22,11 @@ use lance_core::{
     utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu},
 };
 use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
+use lance_encoding::version::LanceFileVersion;
 use lance_file::reader::{FileReader, FileReaderOptions};
-use lance_file::writer::FileWriter;
+use lance_file::writer::{FileWriter, FileWriterOptions};
 use lance_io::{
+    ReadBatchParams,
     object_store::ObjectStore,
     scheduler::{ScanScheduler, SchedulerConfig},
     stream::{RecordBatchStream, RecordBatchStreamAdapter},
@@ -66,9 +73,8 @@ pub struct IvfShuffler {
     object_store: Arc<ObjectStore>,
     output_dir: Path,
     num_partitions: usize,
+    format_version: LanceFileVersion,
 
-    // options
-    precomputed_shuffle_buffers: Option<Vec<String>>,
     progress: Arc<dyn crate::progress::IndexBuildProgress>,
 }
 
@@ -78,21 +84,18 @@ impl IvfShuffler {
             object_store: Arc::new(ObjectStore::local()),
             output_dir,
             num_partitions,
-            precomputed_shuffle_buffers: None,
+            format_version: LanceFileVersion::V2_0,
             progress: crate::progress::noop_progress(),
         }
     }
 
-    pub fn with_progress(mut self, progress: Arc<dyn crate::progress::IndexBuildProgress>) -> Self {
-        self.progress = progress;
+    pub fn with_format_version(mut self, format_version: LanceFileVersion) -> Self {
+        self.format_version = format_version;
         self
     }
 
-    pub fn with_precomputed_shuffle_buffers(
-        mut self,
-        precomputed_shuffle_buffers: Option<Vec<String>>,
-    ) -> Self {
-        self.precomputed_shuffle_buffers = precomputed_shuffle_buffers;
+    pub fn with_progress(mut self, progress: Arc<dyn crate::progress::IndexBuildProgress>) -> Self {
+        self.progress = progress;
         self
     }
 }
@@ -112,12 +115,16 @@ impl Shuffler for IvfShuffler {
                 let spill_path = self.output_dir.child(format!("ivf_{}.spill", partition_id));
                 let object_store = self.object_store.clone();
                 let schema = schema.clone();
+                let format_version = self.format_version;
                 async move {
                     let writer = object_store.create(&part_path).await?;
                     let file_writer = FileWriter::try_new(
                         writer,
                         lance_core::datatypes::Schema::try_from(&schema)?,
-                        Default::default(),
+                        FileWriterOptions {
+                            format_version: Some(format_version),
+                            ..Default::default()
+                        },
                     )?
                     .with_page_metadata_spill(object_store.clone(), spill_path);
                     Result::Ok(file_writer)
@@ -166,7 +173,7 @@ impl Shuffler for IvfShuffler {
             .buffered(get_num_compute_intensive_cpus());
 
         let mut total_loss = 0.0;
-        let mut counter: u64 = 0;
+        let mut num_rows = 0u64;
         while let Some(shuffled) = parallel_sort_stream.next().await {
             let (shuffled, loss) = shuffled?;
             total_loss += loss;
@@ -175,14 +182,15 @@ impl Shuffler for IvfShuffler {
             for (part_id, (writer, batches)) in writers.iter_mut().zip(shuffled.iter()).enumerate()
             {
                 if !batches.is_empty() {
-                    partition_sizes[part_id] += batches.iter().map(|b| b.num_rows()).sum::<usize>();
+                    let rows = batches.iter().map(|b| b.num_rows()).sum::<usize>();
+                    partition_sizes[part_id] += rows;
+                    num_rows += rows as u64;
                     futs.push(writer.write_batches(batches.iter()));
                 }
             }
             try_join_all(futs).await?;
 
-            counter += 1;
-            self.progress.stage_progress("shuffle", counter).await?;
+            self.progress.stage_progress("shuffle", num_rows).await?;
         }
 
         // finish all writers
@@ -286,3 +294,625 @@ impl ShuffleReader for EmptyReader {
         None
     }
 }
+
+/// Create an IVF shuffler. Uses [`TwoFileShuffler`] by default, which writes
+/// all data to just two files (data + offsets) instead of one file per partition.
+/// Set `LANCE_LEGACY_SHUFFLER=1` to fall back to [`IvfShuffler`], which opens
+/// one file per partition.
+///
+/// An optional `progress` callback can be provided to receive shuffle progress
+/// updates.
+pub fn create_ivf_shuffler(
+    output_dir: Path,
+    num_partitions: usize,
+    format_version: LanceFileVersion,
+    progress: Option<Arc<dyn crate::progress::IndexBuildProgress>>,
+) -> Box<dyn Shuffler> {
+    let use_legacy = std::env::var("LANCE_LEGACY_SHUFFLER")
+        .map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
+        .unwrap_or(false);
+    if use_legacy {
+        let mut shuffler =
+            IvfShuffler::new(output_dir, num_partitions).with_format_version(format_version);
+        if let Some(progress) = progress {
+            shuffler = shuffler.with_progress(progress);
+        }
+        Box::new(shuffler)
+    } else {
+        let mut shuffler = TwoFileShuffler::new(output_dir, num_partitions);
+        if let Some(progress) = progress {
+            shuffler = shuffler.with_progress(progress);
+        }
+        Box::new(shuffler)
+    }
+}
+
+const DEFAULT_SHUFFLE_BATCH_BYTES: usize = 128 * 1024 * 1024;
+
+/// Limit of how much transformed data we accumulate before spilling to disk.
+///
+/// A larger value will use more RAM but require less random access during the
+/// read phase.
+///
+/// This default is likely to be fine for most use cases.
+fn shuffle_batch_bytes() -> usize {
+    let batch_size = std::env::var("LANCE_SHUFFLE_BATCH_BYTES")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(DEFAULT_SHUFFLE_BATCH_BYTES);
+    if batch_size == 0 {
+        log::warn!(
+            "LANCE_SHUFFLE_BATCH_BYTES is 0, using default of {}",
+            DEFAULT_SHUFFLE_BATCH_BYTES
+        );
+        DEFAULT_SHUFFLE_BATCH_BYTES
+    } else {
+        batch_size
+    }
+}
+
+/// A shuffler that writes all data to just two files (data + offsets) instead
+/// of one file per partition. This avoids hitting OS file descriptor limits
+/// when there are many partitions.
+///
+/// First we accumulate data in memory until we reach the batch size limit.
+/// Then we sort the data by partition ID and compute an offset per partition.
+/// Then we write the data to a data file and the offsets to an offsets file.
+///
+/// To read the data back, we read every Nth value from the offsets file to get
+/// the start and end of each partition.
+///
+/// Then we read those ranges from the data file.
+pub struct TwoFileShuffler {
+    object_store: Arc<ObjectStore>,
+    output_dir: Path,
+    num_partitions: usize,
+    batch_size_bytes: usize,
+
+    progress: Arc<dyn crate::progress::IndexBuildProgress>,
+}
+
+impl TwoFileShuffler {
+    pub fn new(output_dir: Path, num_partitions: usize) -> Self {
+        Self {
+            object_store: Arc::new(ObjectStore::local()),
+            output_dir,
+            num_partitions,
+            batch_size_bytes: shuffle_batch_bytes(),
+            progress: crate::progress::noop_progress(),
+        }
+    }
+
+    pub fn with_progress(mut self, progress: Arc<dyn crate::progress::IndexBuildProgress>) -> Self {
+        self.progress = progress;
+        self
+    }
+
+    #[cfg(test)]
+    fn with_batch_size_bytes(mut self, batch_size_bytes: usize) -> Self {
+        self.batch_size_bytes = batch_size_bytes;
+        self
+    }
+}
+
+#[async_trait::async_trait]
+impl Shuffler for TwoFileShuffler {
+    async fn shuffle(
+        &self,
+        data: Box<dyn RecordBatchStream + Unpin + 'static>,
+    ) -> Result<Box<dyn ShuffleReader>> {
+        let num_partitions = self.num_partitions;
+        let full_schema = Arc::new(data.schema().as_ref().clone());
+        // No need to write partition ids since we can infer this
+        let schema = data.schema().without_column(PART_ID_COLUMN);
+        let offsets_schema = Arc::new(Schema::new(vec![Field::new(
+            "offset",
+            DataType::UInt64,
+            false,
+        )]));
+        let batch_size_bytes = self.batch_size_bytes;
+
+        // Extract loss from batch metadata before rechunking (concat_batches drops metadata)
+        let total_loss = Arc::new(Mutex::new(0.0f64));
+        let loss_ref = total_loss.clone();
+        let loss_stream = data.map(move |result| {
+            result.inspect(|batch| {
+                let loss = batch
+                    .metadata()
+                    .get(LOSS_METADATA_KEY)
+                    .and_then(|s| s.parse::<f64>().ok())
+                    .unwrap_or(0.0);
+                *loss_ref.lock().unwrap() += loss;
+            })
+        });
+
+        // Rechunk to target batch size
+        let rechunked = rechunk_stream_by_size(
+            loss_stream,
+            full_schema,
+            batch_size_bytes,
+            batch_size_bytes * 2,
+        );
+
+        // Create data file writer
+        let data_path = self.output_dir.child("shuffle_data.lance");
+        let spill_path = self.output_dir.child("shuffle_data.spill");
+        let writer = self.object_store.create(&data_path).await?;
+        let mut file_writer = FileWriter::try_new(
+            writer,
+            lance_core::datatypes::Schema::try_from(&schema)?,
+            Default::default(),
+        )?
+        .with_page_metadata_spill(self.object_store.clone(), spill_path);
+
+        // Create offsets file writer
+        let offsets_path = self.output_dir.child("shuffle_offsets.lance");
+        let spill_path = self.output_dir.child("shuffle_offsets.spill");
+        let writer = self.object_store.create(&offsets_path).await?;
+        let mut offsets_writer = FileWriter::try_new(
+            writer,
+            lance_core::datatypes::Schema::try_from(offsets_schema.as_ref())?,
+            Default::default(),
+        )?
+        .with_page_metadata_spill(self.object_store.clone(), spill_path);
+
+        let num_batches = Arc::new(AtomicU64::new(0));
+        let num_batches_ref = num_batches.clone();
+        let mut partition_counts: Vec<u64> = vec![0; num_partitions];
+        let mut global_row_count: u64 = 0;
+        let mut rows_processed: u64 = 0;
+
+        let mut rechunked = std::pin::pin!(rechunked);
+        while let Some(batch) = rechunked.next().await {
+            num_batches_ref.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+            let batch = batch?;
+            let np = num_partitions;
+            let num_rows = batch.num_rows() as u64;
+
+            // Sort by partition ID and compute offsets on CPU
+            let (sorted_batch, batch_offsets) = spawn_cpu(move || {
+                let part_ids: &UInt32Array = batch[PART_ID_COLUMN].as_primitive();
+                let indices = sort_to_indices(part_ids, None, None)?;
+                let batch = batch.take(&indices)?;
+
+                let part_ids: &UInt32Array = batch[PART_ID_COLUMN].as_primitive();
+                let batch = batch.drop_column(PART_ID_COLUMN)?;
+
+                // Count rows per partition by scanning sorted part IDs
+                let mut partition_counts = vec![0u64; np];
+                for i in 0..part_ids.len() {
+                    let pid = part_ids.value(i) as usize;
+                    if pid < np {
+                        partition_counts[pid] += 1;
+                    } else {
+                        log::warn!("Partition ID {} is out of range [0, {})", pid, np);
+                    }
+                }
+
+                // Build cumulative offsets (end positions) for this batch
+                let mut batch_offsets = Vec::with_capacity(np);
+                let mut running = 0u64;
+                for count in &partition_counts {
+                    running += count;
+                    batch_offsets.push(running);
+                }
+
+                Ok::<(RecordBatch, Vec<u64>), Error>((batch, batch_offsets))
+            })
+            .await?;
+
+            // Write sorted batch to data file
+            file_writer.write_batch(&sorted_batch).await?;
+
+            // Record offsets adjusted by global row count
+            let mut adjusted_offsets = Vec::with_capacity(batch_offsets.len());
+            let mut last_offset = 0;
+            for (idx, offset) in batch_offsets.iter().enumerate() {
+                adjusted_offsets.push(global_row_count + offset);
+                partition_counts[idx] += offset - last_offset;
+                last_offset = *offset;
+            }
+            global_row_count += sorted_batch.num_rows() as u64;
+
+            // Write offsets to offsets file
+            let offsets_batch = RecordBatch::try_new(
+                offsets_schema.clone(),
+                vec![Arc::new(UInt64Array::from(adjusted_offsets))],
+            )?;
+            offsets_writer.write_batch(&offsets_batch).await?;
+
+            rows_processed += num_rows;
+            self.progress
+                .stage_progress("shuffle", rows_processed)
+                .await?;
+        }
+
+        // Finish files
+        file_writer.finish().await?;
+        offsets_writer.finish().await?;
+
+        let num_batches = num_batches.load(std::sync::atomic::Ordering::Relaxed);
+
+        let total_loss_val = *total_loss.lock().unwrap();
+
+        TwoFileShuffleReader::try_new(
+            self.object_store.clone(),
+            self.output_dir.clone(),
+            num_partitions,
+            num_batches,
+            partition_counts,
+            total_loss_val,
+        )
+        .await
+    }
+}
+
+pub struct TwoFileShuffleReader {
+    _scheduler: Arc<ScanScheduler>,
+    file_reader: FileReader,
+    offsets_reader: FileReader,
+    num_partitions: usize,
+    num_batches: u64,
+    partition_counts: Vec<u64>,
+    total_loss: f64,
+}
+
+impl TwoFileShuffleReader {
+    async fn try_new(
+        object_store: Arc<ObjectStore>,
+        output_dir: Path,
+        num_partitions: usize,
+        num_batches: u64,
+        partition_counts: Vec<u64>,
+        total_loss: f64,
+    ) -> Result<Box<dyn ShuffleReader>> {
+        if num_batches == 0 {
+            return Ok(Box::new(EmptyReader));
+        }
+
+        let scheduler_config = SchedulerConfig::max_bandwidth(&object_store);
+        let scheduler = ScanScheduler::new(object_store, scheduler_config);
+
+        let data_path = output_dir.child("shuffle_data.lance");
+        let file_reader = FileReader::try_open(
+            scheduler
+                .open_file(&data_path, &CachedFileSize::unknown())
+                .await?,
+            None,
+            Arc::<DecoderPlugins>::default(),
+            &LanceCache::no_cache(),
+            FileReaderOptions::default(),
+        )
+        .await?;
+
+        let offsets_path = output_dir.child("shuffle_offsets.lance");
+        let offsets_reader = FileReader::try_open(
+            scheduler
+                .open_file(&offsets_path, &CachedFileSize::unknown())
+                .await?,
+            None,
+            Arc::<DecoderPlugins>::default(),
+            &LanceCache::no_cache(),
+            FileReaderOptions::default(),
+        )
+        .await?;
+
+        Ok(Box::new(Self {
+            _scheduler: scheduler,
+            file_reader,
+            offsets_reader,
+            num_partitions,
+            num_batches,
+            partition_counts,
+            total_loss,
+        }))
+    }
+
+    async fn partition_ranges(&self, partition_id: usize) -> Result<Vec<Range<u64>>> {
+        let mut positions = Vec::with_capacity(self.num_batches as usize * 2);
+        for batch_idx in 0..self.num_batches {
+            let end_pos = u32::try_from(batch_idx as usize * self.num_partitions + partition_id)
+                .map_err(|_| Error::invalid_input("There are more than 2^32 partition offsets in the spill file.  Need to support 64-bit take"))?;
+            if end_pos != 0 {
+                positions.push(end_pos - 1);
+            }
+            positions.push(end_pos);
+        }
+        let positions = UInt32Array::from(positions);
+        let num_positions = positions.len() as u32;
+        let offsets_stream = self.offsets_reader.read_stream(
+            ReadBatchParams::Indices(positions),
+            num_positions,
+            1,
+            FilterExpression::no_filter(),
+        )?;
+        let schema = offsets_stream.schema().clone();
+        let offsets = offsets_stream.try_collect::<Vec<_>>().await?;
+        let offsets = if offsets.is_empty() {
+            // We should not hit this path if there is no batches
+            unreachable!()
+        } else if offsets.len() == 1 {
+            offsets.into_iter().next().unwrap()
+        } else {
+            concat_batches(&schema, &offsets)?
+        };
+
+        let offsets = offsets.column(0).as_primitive::<UInt64Type>();
+        let mut offsets_iter = offsets.values().iter().copied();
+
+        let mut ranges = Vec::with_capacity(self.num_batches as usize);
+        for batch_idx in 0..self.num_batches {
+            if batch_idx == 0 && partition_id == 0 {
+                // Implicit 0 for start-of-file
+                ranges.push(0..offsets_iter.next().unwrap());
+            } else {
+                ranges.push(offsets_iter.next().unwrap()..offsets_iter.next().unwrap());
+            }
+        }
+        Ok(ranges)
+    }
+}
+
+#[async_trait::async_trait]
+impl ShuffleReader for TwoFileShuffleReader {
+    async fn read_partition(
+        &self,
+        partition_id: usize,
+    ) -> Result<Option<Box<dyn RecordBatchStream + Unpin + 'static>>> {
+        if partition_id >= self.num_partitions {
+            return Ok(None);
+        }
+        if self.partition_counts[partition_id] == 0 {
+            return Ok(None);
+        }
+
+        let ranges = self.partition_ranges(partition_id).await?;
+        if ranges.is_empty() {
+            return Ok(None);
+        }
+
+        let schema: Schema = self.file_reader.schema().as_ref().into();
+        Ok(Some(Box::new(RecordBatchStreamAdapter::new(
+            Arc::new(schema),
+            self.file_reader.read_stream(
+                ReadBatchParams::Ranges(ranges.into()),
+                u32::MAX,
+                16,
+                FilterExpression::no_filter(),
+            )?,
+        ))))
+    }
+
+    fn partition_size(&self, partition_id: usize) -> Result<usize> {
+        Ok(self
+            .partition_counts
+            .get(partition_id)
+            .copied()
+            .unwrap_or(0) as usize)
+    }
+
+    fn total_loss(&self) -> Option<f64> {
+        Some(self.total_loss)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use arrow_array::{Int32Array, RecordBatch, UInt32Array};
+    use arrow_schema::{DataType, Field, Schema as ArrowSchema};
+    use futures::stream;
+    use lance_arrow::RecordBatchExt;
+    use lance_core::utils::tempfile::TempStrDir;
+    use lance_io::stream::RecordBatchStreamAdapter;
+
+    use crate::vector::{LOSS_METADATA_KEY, PART_ID_COLUMN};
+
+    /// Create a test batch with partition IDs, an int column, and optional loss metadata.
+    fn make_batch(part_ids: &[u32], values: &[i32], loss: Option<f64>) -> RecordBatch {
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new(PART_ID_COLUMN, DataType::UInt32, false),
+            Field::new("val", DataType::Int32, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(UInt32Array::from(part_ids.to_vec())),
+                Arc::new(Int32Array::from(values.to_vec())),
+            ],
+        )
+        .unwrap();
+        if let Some(loss_val) = loss {
+            batch
+                .add_metadata(LOSS_METADATA_KEY.to_owned(), loss_val.to_string())
+                .unwrap()
+        } else {
+            batch
+        }
+    }
+
+    fn batches_to_stream(
+        batches: Vec<RecordBatch>,
+    ) -> Box<dyn RecordBatchStream + Unpin + 'static> {
+        let schema = batches[0].schema();
+        let stream = stream::iter(batches.into_iter().map(Ok));
+        Box::new(RecordBatchStreamAdapter::new(schema, stream))
+    }
+
+    /// Collect all rows from a partition into a single RecordBatch.
+    async fn collect_partition(
+        reader: &dyn ShuffleReader,
+        partition_id: usize,
+    ) -> Option<RecordBatch> {
+        let stream = reader.read_partition(partition_id).await.unwrap()?;
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+        if batches.is_empty() {
+            return None;
+        }
+        Some(arrow::compute::concat_batches(&batches[0].schema(), &batches).unwrap())
+    }
+
+    #[tokio::test]
+    async fn test_two_file_shuffler_round_trip() {
+        let dir = TempStrDir::default();
+        let output_dir = Path::from(dir.as_ref());
+        let num_partitions = 3;
+
+        // Partition 0: rows with values 10, 40
+        // Partition 1: rows with values 20, 50
+        // Partition 2: rows with values 30
+        let batch = make_batch(&[0, 1, 2, 0, 1], &[10, 20, 30, 40, 50], None);
+
+        let shuffler = TwoFileShuffler::new(output_dir, num_partitions);
+        let stream = batches_to_stream(vec![batch]);
+        let reader = shuffler.shuffle(stream).await.unwrap();
+
+        // Verify partition sizes
+        assert_eq!(reader.partition_size(0).unwrap(), 2);
+        assert_eq!(reader.partition_size(1).unwrap(), 2);
+        assert_eq!(reader.partition_size(2).unwrap(), 1);
+
+        // Verify partition 0 data
+        let p0 = collect_partition(reader.as_ref(), 0).await.unwrap();
+        let vals: &Int32Array = p0.column_by_name("val").unwrap().as_primitive();
+        let mut v: Vec<i32> = vals.iter().map(|x| x.unwrap()).collect();
+        v.sort();
+        assert_eq!(v, vec![10, 40]);
+
+        // Verify partition 1 data
+        let p1 = collect_partition(reader.as_ref(), 1).await.unwrap();
+        let vals: &Int32Array = p1.column_by_name("val").unwrap().as_primitive();
+        let mut v: Vec<i32> = vals.iter().map(|x| x.unwrap()).collect();
+        v.sort();
+        assert_eq!(v, vec![20, 50]);
+
+        // Verify partition 2 data
+        let p2 = collect_partition(reader.as_ref(), 2).await.unwrap();
+        let vals: &Int32Array = p2.column_by_name("val").unwrap().as_primitive();
+        let v: Vec<i32> = vals.iter().map(|x| x.unwrap()).collect();
+        assert_eq!(v, vec![30]);
+
+        // Out of range partition returns None
+        assert!(reader.read_partition(3).await.unwrap().is_none());
+    }
+
+    #[tokio::test]
+    async fn test_two_file_shuffler_empty_partitions() {
+        let dir = TempStrDir::default();
+        let output_dir = Path::from(dir.as_ref());
+        let num_partitions = 5;
+
+        // Only use partitions 0 and 3, leaving 1, 2, 4 empty
+        let batch = make_batch(&[0, 3, 0, 3], &[10, 20, 30, 40], None);
+
+        let shuffler = TwoFileShuffler::new(output_dir, num_partitions);
+        let stream = batches_to_stream(vec![batch]);
+        let reader = shuffler.shuffle(stream).await.unwrap();
+
+        assert_eq!(reader.partition_size(0).unwrap(), 2);
+        assert_eq!(reader.partition_size(1).unwrap(), 0);
+        assert_eq!(reader.partition_size(2).unwrap(), 0);
+        assert_eq!(reader.partition_size(3).unwrap(), 2);
+        assert_eq!(reader.partition_size(4).unwrap(), 0);
+
+        assert!(reader.read_partition(1).await.unwrap().is_none());
+        assert!(reader.read_partition(2).await.unwrap().is_none());
+        assert!(reader.read_partition(4).await.unwrap().is_none());
+
+        let p0 = collect_partition(reader.as_ref(), 0).await.unwrap();
+        assert_eq!(p0.num_rows(), 2);
+        let p3 = collect_partition(reader.as_ref(), 3).await.unwrap();
+        assert_eq!(p3.num_rows(), 2);
+    }
+
+    #[tokio::test]
+    async fn test_two_file_shuffler_loss_tracking() {
+        let dir = TempStrDir::default();
+        let output_dir = Path::from(dir.as_ref());
+        let num_partitions = 2;
+
+        let batch1 = make_batch(&[0, 1], &[10, 20], Some(1.5));
+        let batch2 = make_batch(&[0, 1], &[30, 40], Some(2.5));
+        let batch3 = make_batch(&[0], &[50], Some(0.25));
+
+        let shuffler = TwoFileShuffler::new(output_dir, num_partitions);
+        let stream = batches_to_stream(vec![batch1, batch2, batch3]);
+        let reader = shuffler.shuffle(stream).await.unwrap();
+
+        let loss = reader.total_loss().unwrap();
+        assert!((loss - 4.25).abs() < 1e-10, "expected 4.25, got {}", loss);
+    }
+
+    #[tokio::test]
+    async fn test_two_file_shuffler_single_batch() {
+        let dir = TempStrDir::default();
+        let output_dir = Path::from(dir.as_ref());
+        let num_partitions = 2;
+
+        let batch = make_batch(&[1, 0], &[100, 200], Some(3.0));
+
+        let shuffler = TwoFileShuffler::new(output_dir, num_partitions);
+        let stream = batches_to_stream(vec![batch]);
+        let reader = shuffler.shuffle(stream).await.unwrap();
+
+        assert_eq!(reader.partition_size(0).unwrap(), 1);
+        assert_eq!(reader.partition_size(1).unwrap(), 1);
+
+        let p0 = collect_partition(reader.as_ref(), 0).await.unwrap();
+        let vals: &Int32Array = p0.column_by_name("val").unwrap().as_primitive();
+        assert_eq!(vals.value(0), 200);
+
+        let p1 = collect_partition(reader.as_ref(), 1).await.unwrap();
+        let vals: &Int32Array = p1.column_by_name("val").unwrap().as_primitive();
+        assert_eq!(vals.value(0), 100);
+
+        assert!((reader.total_loss().unwrap() - 3.0).abs() < 1e-10);
+    }
+
+    #[tokio::test]
+    async fn test_two_file_shuffler_multiple_batches() {
+        let dir = TempStrDir::default();
+        let output_dir = Path::from(dir.as_ref());
+        let num_partitions = 3;
+
+        // Use a very small batch size to force multiple write batches
+        // Each i32 is 4 bytes, each u32 is 4 bytes, so ~8 bytes/row.
+        // With a small batch_size_bytes, we get multiple rechunked batches.
+        let batch1 = make_batch(&[0, 1, 2], &[10, 20, 30], Some(1.0));
+        let batch2 = make_batch(&[2, 0, 1], &[40, 50, 60], Some(2.0));
+        let batch3 = make_batch(&[1, 2, 0], &[70, 80, 90], Some(3.0));
+
+        let shuffler = TwoFileShuffler::new(output_dir, num_partitions)
+            // Set very small batch size to force multiple batches
+            .with_batch_size_bytes(16);
+        let stream = batches_to_stream(vec![batch1, batch2, batch3]);
+        let reader = shuffler.shuffle(stream).await.unwrap();
+
+        // Partition 0 should have values: 10, 50, 90
+        assert_eq!(reader.partition_size(0).unwrap(), 3);
+        let p0 = collect_partition(reader.as_ref(), 0).await.unwrap();
+        let vals: &Int32Array = p0.column_by_name("val").unwrap().as_primitive();
+        let mut v: Vec<i32> = vals.iter().map(|x| x.unwrap()).collect();
+        v.sort();
+        assert_eq!(v, vec![10, 50, 90]);
+
+        // Partition 1 should have values: 20, 60, 70
+        assert_eq!(reader.partition_size(1).unwrap(), 3);
+        let p1 = collect_partition(reader.as_ref(), 1).await.unwrap();
+        let vals: &Int32Array = p1.column_by_name("val").unwrap().as_primitive();
+        let mut v: Vec<i32> = vals.iter().map(|x| x.unwrap()).collect();
+        v.sort();
+        assert_eq!(v, vec![20, 60, 70]);
+
+        // Partition 2 should have values: 30, 40, 80
+        assert_eq!(reader.partition_size(2).unwrap(), 3);
+        let p2 = collect_partition(reader.as_ref(), 2).await.unwrap();
+        let vals: &Int32Array = p2.column_by_name("val").unwrap().as_primitive();
+        let mut v: Vec<i32> = vals.iter().map(|x| x.unwrap()).collect();
+        v.sort();
+        assert_eq!(v, vec![30, 40, 80]);
+
+        assert!((reader.total_loss().unwrap() - 6.0).abs() < 1e-10);
+    }
+}
diff --git a/rust/lance-index/src/vector/v3/subindex.rs b/rust/lance-index/src/vector/v3/subindex.rs
index af0bb337352..dd5d2b078a9 100644
--- a/rust/lance-index/src/vector/v3/subindex.rs
+++ b/rust/lance-index/src/vector/v3/subindex.rs
@@ -59,6 +59,7 @@ pub trait IvfSubIndex: Send + Sync + Debug + DeepSizeOf {
     fn to_batch(&self) -> Result<RecordBatch>;
 }
 
+#[derive(Debug, Clone, Copy)]
 pub enum SubIndexType {
     Flat,
     Hnsw,
diff --git a/rust/lance-io/Cargo.toml b/rust/lance-io/Cargo.toml
index 0cd8f75747a..e1cbfd70f2b 100644
--- a/rust/lance-io/Cargo.toml
+++ b/rust/lance-io/Cargo.toml
@@ -38,6 +38,7 @@ deepsize.workspace = true
 futures.workspace = true
 http.workspace = true
 log.workspace = true
+moka.workspace = true
 pin-project.workspace = true
 prost.workspace = true
 serde.workspace = true
@@ -49,6 +50,10 @@ path_abs.workspace = true
 rand.workspace = true
 tempfile.workspace = true
 
+[target.'cfg(target_os = "linux")'.dependencies]
+libc = { workspace = true }
+io-uring = { workspace = true }
+
 [dev-dependencies]
 criterion.workspace = true
 test-log.workspace = true
@@ -69,7 +74,7 @@ default = ["aws", "azure", "gcp"]
 gcs-test = []
 gcp = ["object_store/gcp", "dep:opendal", "opendal/services-gcs", "dep:object_store_opendal"]
 aws = ["object_store/aws", "dep:aws-config", "dep:aws-credential-types", "dep:opendal", "opendal/services-s3", "dep:object_store_opendal"]
-azure = ["object_store/azure", "dep:opendal", "opendal/services-azblob", "dep:object_store_opendal"]
+azure = ["object_store/azure", "dep:opendal", "opendal/services-azblob", "opendal/services-azdls", "dep:object_store_opendal"]
 oss = ["dep:opendal", "opendal/services-oss", "dep:object_store_opendal"]
 tencent = ["dep:opendal", "opendal/services-cos", "dep:object_store_opendal"]
 huggingface = ["dep:opendal", "opendal/services-huggingface", "dep:object_store_opendal"]
diff --git a/rust/lance-io/benches/scheduler.rs b/rust/lance-io/benches/scheduler.rs
index 82687867e7a..46d8ac830e3 100644
--- a/rust/lance-io/benches/scheduler.rs
+++ b/rust/lance-io/benches/scheduler.rs
@@ -22,14 +22,14 @@ use pprof::criterion::{Output, PProfProfiler};
 struct FullReadParams {
     io_parallelism: u32,
     page_size: u64,
-    use_lite_scheduler: bool,
+    use_lite_scheduler: Option<bool>,
 }
 
 impl Display for FullReadParams {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
-            "full_read,parallel={},read_size={},use_lite_scheduler={}",
+            "full_read,parallel={},read_size={},use_lite_scheduler={:?}",
             self.io_parallelism, self.page_size, self.use_lite_scheduler
         )
     }
@@ -74,7 +74,7 @@ fn bench_full_read(c: &mut Criterion) {
     let runtime = Runtime::new().unwrap();
     let (obj_store, tmp_file) = runtime.block_on(create_data(DATA_SIZE));
 
-    for use_lite_scheduler in [false, true] {
+    for use_lite_scheduler in [Some(false), Some(true)] {
         for io_parallelism in [1, 16] {
             for page_size in [4096, 1024 * 1024] {
                 let params = FullReadParams {
@@ -101,10 +101,10 @@ fn bench_full_read(c: &mut Criterion) {
                             unsafe {
                                 std::env::set_var("IO_THREADS", io_parallelism.to_string());
                             }
-                            let mut config = SchedulerConfig::default_for_testing();
-                            if use_lite_scheduler {
-                                config = config.with_lite_scheduler();
-                            }
+                            let config = SchedulerConfig {
+                                use_lite_scheduler,
+                                ..SchedulerConfig::default_for_testing()
+                            };
                             runtime.block_on(async {
                                 let scheduler = ScanScheduler::new(obj_store, config);
                                 let file_scheduler = scheduler
@@ -142,7 +142,7 @@ struct RandomReadParams {
     io_parallelism: u32,
     item_size: u32,
     indices: Arc<Vec<u32>>,
-    use_lite_scheduler: bool,
+    use_lite_scheduler: Option<bool>,
     noisy_runtime: bool,
 }
 
@@ -150,7 +150,7 @@ impl Display for RandomReadParams {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
-            "random_read,parallel={},item_size={},use_lite_scheduler={},noisy={}",
+            "random_read,parallel={},item_size={},use_lite_scheduler={:?},noisy={}",
             self.io_parallelism, self.item_size, self.use_lite_scheduler, self.noisy_runtime
         )
     }
@@ -187,7 +187,7 @@ fn bench_random_read(c: &mut Criterion) {
     ));
 
     for noisy_runtime in [false, true] {
-        for use_lite_scheduler in [false, true] {
+        for use_lite_scheduler in [Some(false), Some(true)] {
             for io_parallelism in [1, 16] {
                 for item_size in [4096, 32 * 1024] {
                     let runtime = Runtime::new().unwrap();
@@ -240,10 +240,10 @@ fn bench_random_read(c: &mut Criterion) {
                                         }
                                     }
 
-                                    let mut config = SchedulerConfig::default_for_testing();
-                                    if use_lite_scheduler {
-                                        config = config.with_lite_scheduler();
-                                    }
+                                    let config = SchedulerConfig {
+                                        use_lite_scheduler,
+                                        ..SchedulerConfig::default_for_testing()
+                                    };
                                     let scheduler = ScanScheduler::new(obj_store, config);
                                     let file_scheduler = scheduler
                                         .open_file(&tmp_file, &CachedFileSize::unknown())
diff --git a/rust/lance-io/src/lib.rs b/rust/lance-io/src/lib.rs
index e1729db73be..f3c772e9b01 100644
--- a/rust/lance-io/src/lib.rs
+++ b/rust/lance-io/src/lib.rs
@@ -21,6 +21,8 @@ pub mod stream;
 #[cfg(test)]
 pub mod testing;
 pub mod traits;
+#[cfg(target_os = "linux")]
+pub mod uring;
 pub mod utils;
 
 pub use scheduler::{bytes_read_counter, iops_counter};
diff --git a/rust/lance-io/src/local.rs b/rust/lance-io/src/local.rs
index ab4a21a8874..12f846bcc52 100644
--- a/rust/lance-io/src/local.rs
+++ b/rust/lance-io/src/local.rs
@@ -24,9 +24,10 @@ use tokio::io::AsyncSeekExt;
 use tokio::sync::OnceCell;
 use tracing::instrument;
 
+use crate::object_reader::stream_local_range;
 use crate::object_store::DEFAULT_LOCAL_IO_PARALLELISM;
 use crate::object_writer::WriteResult;
-use crate::traits::{Reader, Writer};
+use crate::traits::{ByteStream, Reader, Writer};
 use crate::utils::tracking_store::IOTracker;
 
 /// Convert an [`object_store::path::Path`] to a [`std::path::Path`].
@@ -240,10 +241,42 @@ impl Reader for LocalObjectReader {
             result
         })
     }
+
+    fn get_stream(&self) -> BoxFuture<'_, object_store::Result<ByteStream>> {
+        Box::pin(async move {
+            let size = self.size().await?;
+            Ok(stream_local_range(
+                self.file.clone(),
+                self.path.clone(),
+                self.io_tracker.clone(),
+                0..size,
+                self.block_size.max(8 * 1024),
+            ))
+        })
+    }
+
+    fn get_range_stream(
+        &self,
+        range: Range<usize>,
+    ) -> BoxFuture<'_, object_store::Result<ByteStream>> {
+        let file = self.file.clone();
+        let path = self.path.clone();
+        let io_tracker = self.io_tracker.clone();
+        let chunk_size = self.block_size.max(8 * 1024);
+        Box::pin(async move {
+            Ok(stream_local_range(
+                file, path, io_tracker, range, chunk_size,
+            ))
+        })
+    }
 }
 
 #[cfg(windows)]
-fn read_exact_at(file: Arc<File>, mut buf: &mut [u8], mut offset: u64) -> std::io::Result<()> {
+pub(crate) fn read_exact_at(
+    file: Arc<File>,
+    mut buf: &mut [u8],
+    mut offset: u64,
+) -> std::io::Result<()> {
     let expected_len = buf.len();
     while !buf.is_empty() {
         match file.seek_read(buf, offset) {
diff --git a/rust/lance-io/src/object_reader.rs b/rust/lance-io/src/object_reader.rs
index d6b34671aed..d7ad909079a 100644
--- a/rust/lance-io/src/object_reader.rs
+++ b/rust/lance-io/src/object_reader.rs
@@ -1,21 +1,31 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use std::fs::File;
 use std::ops::Range;
 use std::sync::Arc;
 
+#[cfg(windows)]
+use crate::local::read_exact_at;
+#[cfg(unix)]
+use std::os::unix::fs::FileExt;
+
 use bytes::Bytes;
 use deepsize::DeepSizeOf;
 use futures::{
     FutureExt,
     future::{BoxFuture, Shared},
+    stream::{self, StreamExt},
 };
 use lance_core::{Error, Result, error::CloneableError};
 use object_store::{GetOptions, GetResult, ObjectStore, Result as OSResult, path::Path};
 use tokio::sync::OnceCell;
 use tracing::instrument;
 
-use crate::{object_store::DEFAULT_CLOUD_IO_PARALLELISM, traits::Reader};
+use crate::{
+    object_store::DEFAULT_CLOUD_IO_PARALLELISM,
+    traits::{ByteStream, Reader},
+};
 
 trait StaticGetRange {
     fn path(&self) -> &Path;
@@ -211,6 +221,41 @@ impl Reader for CloudObjectReader {
             .await
         })
     }
+
+    fn get_stream(&self) -> BoxFuture<'_, OSResult<ByteStream>> {
+        let get_request = Arc::new(GetRequest {
+            object_store: self.object_store.clone(),
+            path: self.path.clone(),
+            options: GetOptions::default(),
+        });
+        Box::pin(async move {
+            let get_request_clone = get_request.clone();
+            let get_result = do_with_retry(move || get_request_clone.get_range()).await?;
+            Ok(get_result.into_stream())
+        })
+    }
+
+    fn get_range_stream(&self, range: Range<usize>) -> BoxFuture<'_, OSResult<ByteStream>> {
+        let get_request = Arc::new(GetRequest {
+            object_store: self.object_store.clone(),
+            path: self.path.clone(),
+            options: GetOptions {
+                range: Some(
+                    Range {
+                        start: range.start as u64,
+                        end: range.end as u64,
+                    }
+                    .into(),
+                ),
+                ..Default::default()
+            },
+        });
+        Box::pin(async move {
+            let get_request_clone = get_request.clone();
+            let get_result = do_with_retry(move || get_request_clone.get_range()).await?;
+            Ok(get_result.into_stream())
+        })
+    }
 }
 
 #[derive(Debug)]
@@ -350,6 +395,53 @@ impl Reader for SmallReader {
     }
 }
 
+pub(crate) fn stream_local_range(
+    file: Arc<File>,
+    path: Path,
+    io_tracker: Arc<crate::utils::tracking_store::IOTracker>,
+    range: Range<usize>,
+    chunk_size: usize,
+) -> ByteStream {
+    stream::try_unfold(
+        (file, path, io_tracker, range.start, range.end),
+        move |state| async move {
+            let (file, path, io_tracker, start, end) = state;
+            if start >= end {
+                return Ok(None);
+            }
+
+            let next = (start + chunk_size).min(end);
+            let file_clone = file.clone();
+            let path_clone = path.clone();
+            let bytes = tokio::task::spawn_blocking(move || {
+                let mut buf = bytes::BytesMut::with_capacity(next - start);
+                // Safety: buffer capacity matches the exact number of bytes we read below.
+                unsafe { buf.set_len(next - start) };
+                #[cfg(unix)]
+                file_clone.read_exact_at(buf.as_mut(), start as u64)?;
+                #[cfg(windows)]
+                read_exact_at(file_clone, buf.as_mut(), start as u64)?;
+                Ok::<_, std::io::Error>(buf.freeze())
+            })
+            .await?
+            .map_err(|err: std::io::Error| object_store::Error::Generic {
+                store: "LocalFileSystem",
+                source: err.into(),
+            })?;
+
+            io_tracker.record_read(
+                "get_range_stream",
+                path_clone,
+                (next - start) as u64,
+                Some(start as u64..next as u64),
+            );
+
+            Ok(Some((bytes, (file, path, io_tracker, next, end))))
+        },
+    )
+    .boxed()
+}
+
 impl DeepSizeOf for SmallReader {
     fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
         let mut size = self.inner.path.as_ref().deep_size_of_children(context);
diff --git a/rust/lance-io/src/object_store.rs b/rust/lance-io/src/object_store.rs
index 4327671b99b..e565cf031cd 100644
--- a/rust/lance-io/src/object_store.rs
+++ b/rust/lance-io/src/object_store.rs
@@ -32,9 +32,12 @@ use tokio::io::AsyncWriteExt;
 use url::Url;
 
 use super::local::LocalObjectReader;
+#[cfg(target_os = "linux")]
+use crate::uring::{UringCurrentThreadReader, UringReader};
 mod list_retry;
 pub mod providers;
 pub mod storage_options;
+pub mod throttle;
 mod tracing;
 use crate::object_reader::SmallReader;
 use crate::object_writer::{LocalWriter, WriteResult};
@@ -531,11 +534,19 @@ impl ObjectStore {
 
     /// Returns true if the object store pointed to a local file system.
     pub fn is_local(&self) -> bool {
-        self.scheme == "file"
+        self.scheme == "file" || self.scheme == "file+uring"
     }
 
     pub fn is_cloud(&self) -> bool {
-        self.scheme != "file" && self.scheme != "memory"
+        !self.is_local() && self.scheme != "memory"
+    }
+
+    /// Whether this object store prefers the lite scheduler.
+    ///
+    /// The lite scheduler is designed for backends like io_uring where
+    /// tasks should only be polled when the consumer polls them.
+    pub fn prefers_lite_scheduler(&self) -> bool {
+        self.scheme == "file+uring"
     }
 
     pub fn scheme(&self) -> &str {
@@ -596,6 +607,31 @@ impl ObjectStore {
                 )
                 .await
             }
+            #[cfg(target_os = "linux")]
+            "file+uring" => {
+                // Check if current-thread mode enabled
+                let use_current_thread = std::env::var("LANCE_URING_CURRENT_THREAD")
+                    .map(|v| str_is_truthy(&v))
+                    .unwrap_or(false);
+
+                if use_current_thread {
+                    UringCurrentThreadReader::open(
+                        path,
+                        self.block_size,
+                        None,
+                        Arc::new(self.io_tracker.clone()),
+                    )
+                    .await
+                } else {
+                    UringReader::open(
+                        path,
+                        self.block_size,
+                        None,
+                        Arc::new(self.io_tracker.clone()),
+                    )
+                    .await
+                }
+            }
             _ => Ok(Box::new(CloudObjectReader::new(
                 self.inner.clone(),
                 path.clone(),
@@ -633,6 +669,31 @@ impl ObjectStore {
                 )
                 .await
             }
+            #[cfg(target_os = "linux")]
+            "file+uring" => {
+                // Check if current-thread mode enabled
+                let use_current_thread = std::env::var("LANCE_URING_CURRENT_THREAD")
+                    .map(|v| str_is_truthy(&v))
+                    .unwrap_or(false);
+
+                if use_current_thread {
+                    UringCurrentThreadReader::open(
+                        path,
+                        self.block_size,
+                        Some(known_size),
+                        Arc::new(self.io_tracker.clone()),
+                    )
+                    .await
+                } else {
+                    UringReader::open(
+                        path,
+                        self.block_size,
+                        Some(known_size),
+                        Arc::new(self.io_tracker.clone()),
+                    )
+                    .await
+                }
+            }
             _ => Ok(Box::new(CloudObjectReader::new(
                 self.inner.clone(),
                 path.clone(),
@@ -873,7 +934,7 @@ impl StorageOptions {
             .iter()
             .find(|(key, _)| key.eq_ignore_ascii_case("client_max_retries"))
             .and_then(|(_, value)| value.parse::<usize>().ok())
-            .unwrap_or(10)
+            .unwrap_or(3)
     }
 
     /// Seconds of timeout to set in RetryConfig for object store client
@@ -1065,6 +1126,13 @@ mod tests {
             .unwrap();
         assert_eq!(store.scheme, "gs");
         assert_eq!(path.to_string(), "foo.lance");
+
+        let (store, path) =
+            ObjectStore::from_uri("abfss://filesystem@account.dfs.core.windows.net/foo.lance")
+                .await
+                .unwrap();
+        assert_eq!(store.scheme, "abfss");
+        assert_eq!(path.to_string(), "foo.lance");
     }
 
     async fn test_block_size_used_test_helper(
@@ -1107,6 +1175,11 @@ mod tests {
             (String::from("account_name"), String::from("account")),
             (String::from("container_name"), String::from("container"))
            ])))]
+    #[case("abfss://filesystem@account.dfs.core.windows.net/foo.lance",
+      Some(HashMap::from([
+            (String::from("account_name"), String::from("account")),
+            (String::from("container_name"), String::from("filesystem"))
+           ])))]
     #[tokio::test]
     async fn test_block_size_used_cloud(
         #[case] uri: &str,
diff --git a/rust/lance-io/src/object_store/providers.rs b/rust/lance-io/src/object_store/providers.rs
index 90964ba2586..03efa281e01 100644
--- a/rust/lance-io/src/object_store/providers.rs
+++ b/rust/lance-io/src/object_store/providers.rs
@@ -89,6 +89,7 @@ pub struct ObjectStoreRegistryStats {
 /// - `file`: A local file object store, with optimized code paths.
 /// - `file-object-store`: A local file object store that uses the ObjectStore API,
 ///   for all operations. Used for testing with ObjectStore wrappers.
+/// - `file+uring`: A local file object store using io_uring (Linux only).
 /// - `s3`: An S3 object store.
 /// - `s3+ddb`: An S3 object store with DynamoDB for metadata.
 /// - `az`: An Azure Blob Storage object store.
@@ -301,6 +302,8 @@ impl Default for ObjectStoreRegistry {
             "file-object-store".into(),
             Arc::new(local::FileStoreProvider),
         );
+        #[cfg(target_os = "linux")]
+        providers.insert("file+uring".into(), Arc::new(local::FileStoreProvider));
 
         #[cfg(feature = "aws")]
         {
@@ -309,7 +312,11 @@ impl Default for ObjectStoreRegistry {
             providers.insert("s3+ddb".into(), aws);
         }
         #[cfg(feature = "azure")]
-        providers.insert("az".into(), Arc::new(azure::AzureBlobStoreProvider));
+        {
+            let azure = Arc::new(azure::AzureBlobStoreProvider);
+            providers.insert("az".into(), azure.clone());
+            providers.insert("abfss".into(), azure);
+        }
         #[cfg(feature = "gcp")]
         providers.insert("gs".into(), Arc::new(gcp::GcsStoreProvider));
         #[cfg(feature = "oss")]
diff --git a/rust/lance-io/src/object_store/providers/aws.rs b/rust/lance-io/src/object_store/providers/aws.rs
index 18165c11598..f7ba1f6ad66 100644
--- a/rust/lance-io/src/object_store/providers/aws.rs
+++ b/rust/lance-io/src/object_store/providers/aws.rs
@@ -30,6 +30,7 @@ use crate::object_store::{
     DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore,
     ObjectStoreParams, ObjectStoreProvider, StorageOptions, StorageOptionsAccessor,
     StorageOptionsProvider,
+    throttle::{AimdThrottleConfig, AimdThrottledStore},
 };
 use lance_core::error::{Error, Result};
 
@@ -44,12 +45,12 @@ impl AwsStoreProvider {
         storage_options: &StorageOptions,
         is_s3_express: bool,
     ) -> Result<Arc<dyn OSObjectStore>> {
-        let max_retries = storage_options.client_max_retries();
-        let retry_timeout = storage_options.client_retry_timeout();
+        // Use a low retry count since the AIMD throttle layer handles
+        // throttle recovery with its own retry loop.
         let retry_config = RetryConfig {
             backoff: Default::default(),
-            max_retries,
-            retry_timeout: Duration::from_secs(retry_timeout),
+            max_retries: storage_options.client_max_retries(),
+            retry_timeout: Duration::from_secs(storage_options.client_retry_timeout()),
         };
 
         let mut s3_storage_options = storage_options.as_s3_options();
@@ -131,7 +132,7 @@ impl ObjectStoreProvider for AwsStoreProvider {
     ) -> Result<ObjectStore> {
         let block_size = params.block_size.unwrap_or(DEFAULT_CLOUD_BLOCK_SIZE);
         let mut storage_options =
-            StorageOptions(params.storage_options().cloned().unwrap_or_default());
+            StorageOptions::new(params.storage_options().cloned().unwrap_or_default());
         storage_options.with_env_s3();
         let download_retry_count = storage_options.download_retry_count();
 
@@ -159,6 +160,19 @@ impl ObjectStoreProvider for AwsStoreProvider {
             self.build_amazon_s3_store(&mut base_path, params, &storage_options, is_s3_express)
                 .await?
         };
+        let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?;
+        let inner = if throttle_config.is_disabled() {
+            inner
+        } else if storage_options.client_max_retries() == 0 {
+            log::warn!(
+                "AIMD throttle disabled: the current implementation relies on the object store \
+                 client surfacing retry errors, which requires client_max_retries > 0. \
+                 No throttle or retry layer will be applied."
+            );
+            inner
+        } else {
+            Arc::new(AimdThrottledStore::new(inner, throttle_config)?) as Arc<dyn OSObjectStore>
+        };
 
         Ok(ObjectStore {
             inner,
@@ -260,7 +274,9 @@ pub async fn build_aws_credential(
 
     let storage_options_credentials = storage_options.and_then(extract_static_s3_credentials);
 
-    // If accessor has a provider, use DynamicStorageOptionsCredentialProvider
+    // If accessor has a provider, check whether it vends credentials.
+    // If it does, use DynamicStorageOptionsCredentialProvider for ongoing
+    // refresh. If not, fall through to the default credentials chain.
     if let Some(accessor) = storage_options_accessor
         && accessor.has_provider()
     {
@@ -268,11 +284,21 @@ pub async fn build_aws_credential(
         if let Some(creds) = credentials {
             return Ok((creds, region));
         }
-        // Use accessor for dynamic credential refresh
-        return Ok((
-            Arc::new(DynamicStorageOptionsCredentialProvider::new(accessor)),
-            region,
-        ));
+
+        // Check if the accessor's storage options contain credentials
+        let opts = accessor.get_storage_options().await?;
+        let s3_options = opts.as_s3_options();
+        if extract_static_s3_credentials(&s3_options).is_some() {
+            return Ok((
+                Arc::new(DynamicStorageOptionsCredentialProvider::new(accessor)),
+                region,
+            ));
+        }
+
+        log::debug!(
+            "Storage options from provider do not contain explicit AWS credentials, \
+             falling back to default AWS credentials chain."
+        );
     }
 
     // Fall back to existing logic for static credentials
diff --git a/rust/lance-io/src/object_store/providers/azure.rs b/rust/lance-io/src/object_store/providers/azure.rs
index 2bd0bf2878d..ae7fc1d19ff 100644
--- a/rust/lance-io/src/object_store/providers/azure.rs
+++ b/rust/lance-io/src/object_store/providers/azure.rs
@@ -10,7 +10,7 @@ use std::{
 
 use object_store::ObjectStore as OSObjectStore;
 use object_store_opendal::OpendalStore;
-use opendal::{Operator, services::Azblob};
+use opendal::{Operator, services::Azblob, services::Azdls};
 
 use object_store::{
     RetryConfig,
@@ -21,6 +21,7 @@ use url::Url;
 use crate::object_store::{
     DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore,
     ObjectStoreParams, ObjectStoreProvider, StorageOptions,
+    throttle::{AimdThrottleConfig, AimdThrottledStore},
 };
 use lance_core::error::{Error, Result};
 
@@ -28,36 +29,84 @@ use lance_core::error::{Error, Result};
 pub struct AzureBlobStoreProvider;
 
 impl AzureBlobStoreProvider {
-    async fn build_opendal_azure_store(
-        &self,
+    fn build_opendal_operator(
         base_path: &Url,
         storage_options: &StorageOptions,
-    ) -> Result<Arc<dyn OSObjectStore>> {
-        let container = base_path
-            .host_str()
-            .ok_or_else(|| Error::invalid_input("Azure URL must contain container name"))?
-            .to_string();
-
-        let prefix = base_path.path().trim_start_matches('/').to_string();
-
+    ) -> Result<Operator> {
         // Start with all storage options as the config map
         // OpenDAL will handle environment variables through its default credentials chain
         let mut config_map: HashMap<String, String> = storage_options.0.clone();
 
-        // Set required OpenDAL configuration
-        config_map.insert("container".to_string(), container);
+        match base_path.scheme() {
+            "az" => {
+                let container = base_path
+                    .host_str()
+                    .ok_or_else(|| Error::invalid_input("Azure URL must contain container name"))?
+                    .to_string();
 
-        if !prefix.is_empty() {
-            config_map.insert("root".to_string(), format!("/{}", prefix));
-        }
+                config_map.insert("container".to_string(), container);
 
-        let operator = Operator::from_iter::<Azblob>(config_map)
-            .map_err(|e| {
-                Error::invalid_input(format!("Failed to create Azure Blob operator: {:?}", e))
-            })?
-            .finish();
+                let prefix = base_path.path().trim_start_matches('/');
+                if !prefix.is_empty() {
+                    config_map.insert("root".to_string(), format!("/{}", prefix));
+                }
 
-        Ok(Arc::new(OpendalStore::new(operator)) as Arc<dyn OSObjectStore>)
+                Operator::from_iter::<Azblob>(config_map)
+                    .map_err(|e| {
+                        Error::invalid_input(format!(
+                            "Failed to create Azure Blob operator: {:?}",
+                            e
+                        ))
+                    })
+                    .map(|b| b.finish())
+            }
+            "abfss" => {
+                let filesystem = base_path.username();
+                if filesystem.is_empty() {
+                    return Err(Error::invalid_input(
+                        "abfss:// URL must include account: abfss://<filesystem>@<account>.dfs.core.windows.net/path",
+                    ));
+                }
+                let host = base_path.host_str().ok_or_else(|| {
+                    Error::invalid_input(
+                        "abfss:// URL must include account: abfss://<filesystem>@<account>.dfs.core.windows.net/path"
+                    )
+                })?;
+
+                config_map.insert("filesystem".to_string(), filesystem.to_string());
+                config_map.insert("endpoint".to_string(), format!("https://{}", host));
+                config_map
+                    .entry("account_name".to_string())
+                    .or_insert_with(|| host.split('.').next().unwrap_or(host).to_string());
+
+                let root_path = base_path.path().trim_start_matches('/');
+                if !root_path.is_empty() {
+                    config_map.insert("root".to_string(), format!("/{}", root_path));
+                }
+
+                Operator::from_iter::<Azdls>(config_map)
+                    .map_err(|e| {
+                        Error::invalid_input(format!(
+                            "Failed to create Azure DFS (ADLS Gen2) operator: {:?}",
+                            e
+                        ))
+                    })
+                    .map(|b| b.finish())
+            }
+            _ => Err(Error::invalid_input(format!(
+                "Unsupported Azure scheme: {}",
+                base_path.scheme()
+            ))),
+        }
+    }
+
+    async fn build_opendal_azure_store(
+        &self,
+        base_path: &Url,
+        storage_options: &StorageOptions,
+    ) -> Result<Arc<dyn OSObjectStore>> {
+        let operator = Self::build_opendal_operator(base_path, storage_options)?;
+        Ok(Arc::new(OpendalStore::new(operator)))
     }
 
     async fn build_microsoft_azure_store(
@@ -65,12 +114,12 @@ impl AzureBlobStoreProvider {
         base_path: &Url,
         storage_options: &StorageOptions,
     ) -> Result<Arc<dyn OSObjectStore>> {
-        let max_retries = storage_options.client_max_retries();
-        let retry_timeout = storage_options.client_retry_timeout();
+        // Use a low retry count since the AIMD throttle layer handles
+        // throttle recovery with its own retry loop.
         let retry_config = RetryConfig {
             backoff: Default::default(),
-            max_retries,
-            retry_timeout: Duration::from_secs(retry_timeout),
+            max_retries: storage_options.client_max_retries(),
+            retry_timeout: Duration::from_secs(storage_options.client_retry_timeout()),
         };
 
         let mut builder = MicrosoftAzureBuilder::new()
@@ -88,9 +137,17 @@ impl AzureBlobStoreProvider {
 #[async_trait::async_trait]
 impl ObjectStoreProvider for AzureBlobStoreProvider {
     async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
+        let scheme = base_path.scheme().to_string();
+        if scheme != "az" && scheme != "abfss" {
+            return Err(Error::invalid_input(format!(
+                "Unsupported Azure scheme '{}', expected 'az' or 'abfss'",
+                scheme
+            )));
+        }
+
         let block_size = params.block_size.unwrap_or(DEFAULT_CLOUD_BLOCK_SIZE);
         let mut storage_options =
-            StorageOptions(params.storage_options().cloned().unwrap_or_default());
+            StorageOptions::new(params.storage_options().cloned().unwrap_or_default());
         storage_options.with_env_azure();
         let download_retry_count = storage_options.download_retry_count();
 
@@ -100,17 +157,30 @@ impl ObjectStoreProvider for AzureBlobStoreProvider {
             .map(|v| v.as_str() == "true")
             .unwrap_or(false);
 
-        let inner = if use_opendal {
+        let inner: Arc<dyn OSObjectStore> = if use_opendal {
             self.build_opendal_azure_store(&base_path, &storage_options)
                 .await?
         } else {
             self.build_microsoft_azure_store(&base_path, &storage_options)
                 .await?
         };
+        let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?;
+        let inner = if throttle_config.is_disabled() {
+            inner
+        } else if storage_options.client_max_retries() == 0 {
+            log::warn!(
+                "AIMD throttle disabled: the current implementation relies on the object store \
+                 client surfacing retry errors, which requires client_max_retries > 0. \
+                 No throttle or retry layer will be applied."
+            );
+            inner
+        } else {
+            Arc::new(AimdThrottledStore::new(inner, throttle_config)?) as Arc<dyn OSObjectStore>
+        };
 
         Ok(ObjectStore {
             inner,
-            scheme: String::from("az"),
+            scheme,
             block_size,
             max_iop_size: *DEFAULT_MAX_IOP_SIZE,
             use_constant_size_upload_parts: false,
@@ -131,8 +201,10 @@ impl ObjectStoreProvider for AzureBlobStoreProvider {
         let authority = url.authority();
         let (container, account) = match authority.find("@") {
             Some(at_index) => {
-                // The URI looks like 'az://container@account.dfs.core.windows.net/path-part/file',
-                // or possibly 'az://container@account/path-part/file'.
+                // The URI has an:
+                // - az:// schema type and is similar to 'az://container@account.dfs.core.windows.net/path-part/file
+                //         or possibly 'az://container@account/path-part/file' (the short version).
+                // - abfss:// schema type and is similar to 'abfss://filesystem@account.dfs.core.windows.net/path-part/file'.
                 let container = &authority[..at_index];
                 let account = &authority[at_index + 1..];
                 (
@@ -250,6 +322,12 @@ mod tests {
             .await
             .unwrap();
         assert_eq!(store.scheme, "az");
+        let inner_desc = store.inner.to_string();
+        assert!(
+            inner_desc.contains("Opendal") && inner_desc.contains("azblob"),
+            "az:// with use_opendal=true should use OpenDAL Azblob, got: {}",
+            inner_desc
+        );
     }
 
     #[test]
@@ -299,6 +377,21 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_calculate_object_store_prefix_from_url_short_account() {
+        let provider = AzureBlobStoreProvider;
+        let options = HashMap::from_iter([("account_name".to_string(), "bob".to_string())]);
+        assert_eq!(
+            "az$container@account",
+            provider
+                .calculate_object_store_prefix(
+                    &Url::parse("az://container@account/path").unwrap(),
+                    Some(&options)
+                )
+                .unwrap()
+        );
+    }
+
     #[test]
     fn test_fail_to_calculate_object_store_prefix_from_url() {
         let provider = AzureBlobStoreProvider;
@@ -313,4 +406,157 @@ mod tests {
             .to_string();
         assert_eq!(expected, &result[..expected.len()]);
     }
+
+    // --- abfss:// tests ---
+
+    #[test]
+    fn test_abfss_extract_path() {
+        let provider = AzureBlobStoreProvider;
+        let url = Url::parse("abfss://myfs@myaccount.dfs.core.windows.net/path/to/dataset.lance")
+            .unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        assert_eq!(
+            path,
+            object_store::path::Path::from("path/to/dataset.lance")
+        );
+    }
+
+    #[test]
+    fn test_calculate_abfss_prefix() {
+        let provider = AzureBlobStoreProvider;
+        let url = Url::parse("abfss://myfs@myaccount.dfs.core.windows.net/path/to/data").unwrap();
+        let prefix = provider.calculate_object_store_prefix(&url, None).unwrap();
+        assert_eq!(prefix, "abfss$myfs@myaccount");
+    }
+
+    #[test]
+    fn test_calculate_abfss_prefix_ignores_storage_options() {
+        let provider = AzureBlobStoreProvider;
+        let options =
+            HashMap::from_iter([("account_name".to_string(), "other_account".to_string())]);
+        let url = Url::parse("abfss://myfs@myaccount.dfs.core.windows.net/path").unwrap();
+        let prefix = provider
+            .calculate_object_store_prefix(&url, Some(&options))
+            .unwrap();
+        assert_eq!(prefix, "abfss$myfs@myaccount");
+    }
+
+    #[tokio::test]
+    async fn test_abfss_default_uses_microsoft_builder() {
+        use crate::object_store::StorageOptionsAccessor;
+        let provider = AzureBlobStoreProvider;
+        let url = Url::parse("abfss://testfs@testaccount.dfs.core.windows.net/data").unwrap();
+        let params = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("account_name".to_string(), "testaccount".to_string()),
+                    ("account_key".to_string(), "dGVzdA==".to_string()),
+                ]),
+            ))),
+            ..Default::default()
+        };
+
+        let store = provider.new_store(url, &params).await.unwrap();
+        assert_eq!(store.scheme, "abfss");
+        assert!(!store.is_local());
+        assert!(store.is_cloud());
+        let inner_desc = store.inner.to_string();
+        assert!(
+            inner_desc.contains("MicrosoftAzure"),
+            "abfss:// without use_opendal should use MicrosoftAzureBuilder, got: {}",
+            inner_desc
+        );
+    }
+
+    #[tokio::test]
+    async fn test_unsupported_scheme_rejected() {
+        use crate::object_store::StorageOptionsAccessor;
+        let provider = AzureBlobStoreProvider;
+        let url = Url::parse("wasbs://container@myaccount.blob.core.windows.net/path").unwrap();
+        let params = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("account_name".to_string(), "myaccount".to_string()),
+                    ("account_key".to_string(), "dGVzdA==".to_string()),
+                ]),
+            ))),
+            ..Default::default()
+        };
+
+        let err = provider
+            .new_store(url, &params)
+            .await
+            .expect_err("expected error for unsupported scheme");
+        assert!(
+            err.to_string().contains("Unsupported Azure scheme"),
+            "unexpected error: {}",
+            err
+        );
+    }
+
+    #[tokio::test]
+    async fn test_abfss_with_opendal_uses_azdls() {
+        use crate::object_store::StorageOptionsAccessor;
+        let provider = AzureBlobStoreProvider;
+        let url = Url::parse("abfss://testfs@testaccount.dfs.core.windows.net/data").unwrap();
+        let params = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("use_opendal".to_string(), "true".to_string()),
+                    ("account_name".to_string(), "testaccount".to_string()),
+                    ("account_key".to_string(), "dGVzdA==".to_string()),
+                ]),
+            ))),
+            ..Default::default()
+        };
+
+        let store = provider.new_store(url, &params).await.unwrap();
+        assert_eq!(store.scheme, "abfss");
+        assert!(!store.is_local());
+        assert!(store.is_cloud());
+        let inner_desc = store.inner.to_string();
+        assert!(
+            inner_desc.contains("Opendal") && inner_desc.contains("azdls"),
+            "abfss:// with use_opendal=true should use OpenDAL Azdls, got: {}",
+            inner_desc
+        );
+    }
+
+    #[test]
+    fn test_azdls_capabilities_differ_from_azblob() {
+        let common_opts = StorageOptions(HashMap::from([
+            ("account_name".to_string(), "testaccount".to_string()),
+            ("account_key".to_string(), "dGVzdA==".to_string()),
+            (
+                "endpoint".to_string(),
+                "https://testaccount.blob.core.windows.net".to_string(),
+            ),
+        ]));
+
+        // Build az:// operator (uses Azblob backend)
+        let az_url = Url::parse("az://test-container/path").unwrap();
+        let az_operator =
+            AzureBlobStoreProvider::build_opendal_operator(&az_url, &common_opts).unwrap();
+
+        // Build abfss:// operator (uses Azdls backend)
+        let abfss_url = Url::parse("abfss://testfs@testaccount.dfs.core.windows.net/data").unwrap();
+        let abfss_operator =
+            AzureBlobStoreProvider::build_opendal_operator(&abfss_url, &common_opts).unwrap();
+
+        let azblob_cap = az_operator.info().native_capability();
+        let azdls_cap = abfss_operator.info().native_capability();
+
+        // Both support basic operations
+        assert!(azblob_cap.read);
+        assert!(azdls_cap.read);
+        assert!(azblob_cap.write);
+        assert!(azdls_cap.write);
+        assert!(azblob_cap.list);
+        assert!(azdls_cap.list);
+
+        // Azdls supports rename and create_dir (HNS features); Azblob does not
+        assert!(azdls_cap.rename, "Azdls should support rename");
+        assert!(azdls_cap.create_dir, "Azdls should support create_dir");
+        assert!(!azblob_cap.rename, "Azblob should not support rename");
+    }
 }
diff --git a/rust/lance-io/src/object_store/providers/gcp.rs b/rust/lance-io/src/object_store/providers/gcp.rs
index 7bfe3715566..e0f42179b19 100644
--- a/rust/lance-io/src/object_store/providers/gcp.rs
+++ b/rust/lance-io/src/object_store/providers/gcp.rs
@@ -16,6 +16,7 @@ use url::Url;
 use crate::object_store::{
     DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore,
     ObjectStoreParams, ObjectStoreProvider, StorageOptions,
+    throttle::{AimdThrottleConfig, AimdThrottledStore},
 };
 use lance_core::error::{Error, Result};
 
@@ -58,12 +59,12 @@ impl GcsStoreProvider {
         base_path: &Url,
         storage_options: &StorageOptions,
     ) -> Result<Arc<dyn OSObjectStore>> {
-        let max_retries = storage_options.client_max_retries();
-        let retry_timeout = storage_options.client_retry_timeout();
+        // Use a low retry count since the AIMD throttle layer handles
+        // throttle recovery with its own retry loop.
         let retry_config = RetryConfig {
             backoff: Default::default(),
-            max_retries,
-            retry_timeout: Duration::from_secs(retry_timeout),
+            max_retries: storage_options.client_max_retries(),
+            retry_timeout: Duration::from_secs(storage_options.client_retry_timeout()),
         };
 
         let mut builder = GoogleCloudStorageBuilder::new()
@@ -91,7 +92,7 @@ impl ObjectStoreProvider for GcsStoreProvider {
     async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
         let block_size = params.block_size.unwrap_or(DEFAULT_CLOUD_BLOCK_SIZE);
         let mut storage_options =
-            StorageOptions(params.storage_options().cloned().unwrap_or_default());
+            StorageOptions::new(params.storage_options().cloned().unwrap_or_default());
         storage_options.with_env_gcs();
         let download_retry_count = storage_options.download_retry_count();
 
@@ -108,6 +109,19 @@ impl ObjectStoreProvider for GcsStoreProvider {
             self.build_google_cloud_store(&base_path, &storage_options)
                 .await?
         };
+        let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?;
+        let inner = if throttle_config.is_disabled() {
+            inner
+        } else if storage_options.client_max_retries() == 0 {
+            log::warn!(
+                "AIMD throttle disabled: the current implementation relies on the object store \
+                 client surfacing retry errors, which requires client_max_retries > 0. \
+                 No throttle or retry layer will be applied."
+            );
+            inner
+        } else {
+            Arc::new(AimdThrottledStore::new(inner, throttle_config)?) as Arc<dyn OSObjectStore>
+        };
 
         Ok(ObjectStore {
             inner,
diff --git a/rust/lance-io/src/object_store/storage_options.rs b/rust/lance-io/src/object_store/storage_options.rs
index b16281ea4be..7cff81ef213 100644
--- a/rust/lance-io/src/object_store/storage_options.rs
+++ b/rust/lance-io/src/object_store/storage_options.rs
@@ -94,7 +94,7 @@ pub trait StorageOptionsProvider: Send + Sync + fmt::Debug {
 
 /// StorageOptionsProvider implementation that fetches options from a LanceNamespace
 pub struct LanceNamespaceStorageOptionsProvider {
-    namespace: Arc<dyn LanceNamespace>,
+    namespace_client: Arc<dyn LanceNamespace>,
     table_id: Vec<String>,
 }
 
@@ -114,11 +114,11 @@ impl LanceNamespaceStorageOptionsProvider {
     /// Create a new LanceNamespaceStorageOptionsProvider
     ///
     /// # Arguments
-    /// * `namespace` - The namespace implementation to fetch storage options from
+    /// * `namespace_client` - The namespace implementation to fetch storage options from
     /// * `table_id` - The table identifier
-    pub fn new(namespace: Arc<dyn LanceNamespace>, table_id: Vec<String>) -> Self {
+    pub fn new(namespace_client: Arc<dyn LanceNamespace>, table_id: Vec<String>) -> Self {
         Self {
-            namespace,
+            namespace_client,
             table_id,
         }
     }
@@ -132,20 +132,24 @@ impl StorageOptionsProvider for LanceNamespaceStorageOptionsProvider {
             ..Default::default()
         };
 
-        let response = self.namespace.describe_table(request).await.map_err(|e| {
-            Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to fetch storage options: {}",
-                e
-            ))))
-        })?;
+        let response = self
+            .namespace_client
+            .describe_table(request)
+            .await
+            .map_err(|e| {
+                Error::io_source(Box::new(std::io::Error::other(format!(
+                    "Failed to fetch storage options: {}",
+                    e
+                ))))
+            })?;
 
         Ok(response.storage_options)
     }
 
     fn provider_id(&self) -> String {
         format!(
-            "LanceNamespaceStorageOptionsProvider {{ namespace: {}, table_id: {:?} }}",
-            self.namespace.namespace_id(),
+            "LanceNamespaceStorageOptionsProvider {{ namespace_client: {}, table_id: {:?} }}",
+            self.namespace_client.namespace_id(),
             self.table_id
         )
     }
@@ -310,14 +314,13 @@ impl StorageOptionsAccessor {
             }
         }
 
-        // If no provider, return initial options or error
+        // If no provider, return initial options or use defaults
         let Some(provider) = &self.provider else {
             return if let Some(initial) = &self.initial_options {
                 Ok(Some(super::StorageOptions(initial.clone())))
             } else {
-                Err(Error::io_source(Box::new(std::io::Error::other(
-                    "No storage options available",
-                ))))
+                // No provider and no initial options - use default credentials
+                Ok(Some(super::StorageOptions(HashMap::new())))
             };
         };
 
@@ -333,7 +336,6 @@ impl StorageOptionsAccessor {
         {
             return Ok(Some(super::StorageOptions(cached_opts.options.clone())));
         }
-
         log::debug!(
             "Refreshing storage options from provider: {}",
             provider.provider_id()
@@ -347,13 +349,18 @@ impl StorageOptionsAccessor {
         })?;
 
         let Some(options) = storage_options_map else {
-            // Provider returned None, fall back to initial options
+            // Provider returned None, fall back to initial options or use defaults
             if let Some(initial) = &self.initial_options {
                 return Ok(Some(super::StorageOptions(initial.clone())));
             }
-            return Err(Error::io_source(Box::new(std::io::Error::other(
-                "Provider returned no storage options",
-            ))));
+            // Provider returned None and no initial options - use default credentials
+            // This is valid when namespace doesn't vend credentials (e.g., directory namespace
+            // where environment credentials are used)
+            log::debug!(
+                "Provider {} returned no storage options, using default credentials",
+                provider.provider_id()
+            );
+            return Ok(Some(super::StorageOptions(HashMap::new())));
         };
 
         let expires_at_millis = options
diff --git a/rust/lance-io/src/object_store/throttle.rs b/rust/lance-io/src/object_store/throttle.rs
new file mode 100644
index 00000000000..046ab4b0ff4
--- /dev/null
+++ b/rust/lance-io/src/object_store/throttle.rs
@@ -0,0 +1,1628 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! AIMD-controlled token bucket rate limiter for ObjectStore operations.
+//!
+//! Wraps any [`object_store::ObjectStore`] with per-category token buckets
+//! whose fill rates are dynamically adjusted by AIMD controllers. When cloud
+//! stores return HTTP 429/503, the fill rate decreases multiplicatively. During
+//! sustained success windows, it increases additively.
+//!
+//! Operations are split into four independent categories — **read**, **write**,
+//! **delete**, **list** — each with its own AIMD controller and token bucket.
+//! This prevents a burst of reads from starving writes, and vice versa.
+//!
+//! # Example
+//!
+//! ```ignore
+//! use lance_io::object_store::throttle::{AimdThrottleConfig, AimdThrottledStore};
+//!
+//! let throttled = AimdThrottledStore::new(target, AimdThrottleConfig::default()).unwrap();
+//! ```
+
+use std::collections::HashMap;
+use std::fmt::{Debug, Display, Formatter};
+use std::ops::Range;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Bytes;
+use futures::StreamExt;
+use futures::stream::BoxStream;
+use lance_core::utils::aimd::{AimdConfig, AimdController, RequestOutcome};
+use object_store::path::Path;
+use object_store::{
+    GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
+    PutMultipartOptions, PutOptions, PutPayload, PutResult, Result as OSResult, UploadPart,
+};
+use rand::Rng;
+use tokio::sync::Mutex;
+use tracing::{debug, warn};
+
+/// Check whether an `object_store::Error` represents a throttle response
+/// (HTTP 429 / 503) from a cloud object store.
+///
+/// Regrettably, this information is not fully exposed by the `object_store` crate.
+/// There is no generic mechanism for a custom object store to return a throttle error.
+///
+/// However, the builtin object stores all use RetryError when retries are configured and
+/// throttle errors are returned.  Sadly, RetryError is not a public type, so we have to
+/// infer it from the error message.  This is potentially dangerous because these errors
+/// often include the URI itself and that URI could have any characters in it (e.g. if we
+/// look for 429 then we might match a 429 in a UUID).These error messages currently look like:
+///
+/// ", after ... retries, max_retries: ..., retry_timeout: ..."
+///
+/// So, as a crude heuristic, which should work for the builtin object stores, but won't
+/// work for custom object stores, we simply look for the string "retries, max_retries"
+/// in the error message.
+pub fn is_throttle_error(err: &object_store::Error) -> bool {
+    // Only Generic errors can carry throttle responses
+    if let object_store::Error::Generic { source, .. } = err {
+        source.to_string().contains("retries, max_retries")
+    } else {
+        false
+    }
+}
+
+/// Configuration for the AIMD-throttled ObjectStore wrapper.
+///
+/// Each operation category (read, write, delete, list) has its own AIMD config.
+/// Use [`with_aimd`](AimdThrottleConfig::with_aimd) to set all categories at
+/// once, or per-category methods like [`with_read_aimd`](AimdThrottleConfig::with_read_aimd)
+/// for fine-grained control.
+#[derive(Debug, Clone)]
+pub struct AimdThrottleConfig {
+    /// AIMD configuration for read operations (get, get_opts, get_range, get_ranges, head).
+    pub read: AimdConfig,
+    /// AIMD configuration for write operations (put, put_opts, put_multipart, copy, rename, etc.).
+    pub write: AimdConfig,
+    /// AIMD configuration for delete operations.
+    pub delete: AimdConfig,
+    /// AIMD configuration for list operations (list_with_delimiter).
+    pub list: AimdConfig,
+    /// Maximum tokens that can accumulate for bursts (shared across all categories).
+    pub burst_capacity: u32,
+    /// Maximum number of retries for throttle errors within the AIMD layer.
+    pub max_retries: usize,
+    /// Minimum backoff in milliseconds between retry attempts.
+    pub min_backoff_ms: u64,
+    /// Maximum backoff in milliseconds between retry attempts.
+    pub max_backoff_ms: u64,
+}
+
+impl Default for AimdThrottleConfig {
+    fn default() -> Self {
+        let aimd = AimdConfig::default();
+        Self {
+            read: aimd.clone(),
+            write: aimd.clone(),
+            delete: aimd.clone(),
+            list: aimd,
+            burst_capacity: 100,
+            max_retries: 3,
+            min_backoff_ms: 100,
+            max_backoff_ms: 300,
+        }
+    }
+}
+
+impl AimdThrottleConfig {
+    /// Set the AIMD configuration for all four operation categories at once.
+    pub fn with_aimd(self, aimd: AimdConfig) -> Self {
+        Self {
+            read: aimd.clone(),
+            write: aimd.clone(),
+            delete: aimd.clone(),
+            list: aimd,
+            ..self
+        }
+    }
+
+    /// Set the AIMD configuration for read operations.
+    pub fn with_read_aimd(self, aimd: AimdConfig) -> Self {
+        Self { read: aimd, ..self }
+    }
+
+    /// Set the AIMD configuration for write operations.
+    pub fn with_write_aimd(self, aimd: AimdConfig) -> Self {
+        Self {
+            write: aimd,
+            ..self
+        }
+    }
+
+    /// Set the AIMD configuration for delete operations.
+    pub fn with_delete_aimd(self, aimd: AimdConfig) -> Self {
+        Self {
+            delete: aimd,
+            ..self
+        }
+    }
+
+    /// Set the AIMD configuration for list operations.
+    pub fn with_list_aimd(self, aimd: AimdConfig) -> Self {
+        Self { list: aimd, ..self }
+    }
+
+    /// Returns `true` when the AIMD throttle layer should be bypassed entirely.
+    pub fn is_disabled(&self) -> bool {
+        self.max_retries == 0
+    }
+
+    pub fn with_burst_capacity(self, burst_capacity: u32) -> Self {
+        Self {
+            burst_capacity,
+            ..self
+        }
+    }
+
+    /// Build an `AimdThrottleConfig` from storage options and environment variables.
+    ///
+    /// Storage options take precedence over environment variables, which take
+    /// precedence over defaults. A single AIMD config is applied to all four
+    /// operation categories (read/write/delete/list).
+    ///
+    /// | Setting              | Storage Option Key               | Env Var                          | Default |
+    /// |----------------------|----------------------------------|----------------------------------|---------|
+    /// | Initial rate         | `lance_aimd_initial_rate`        | `LANCE_AIMD_INITIAL_RATE`        | 2000    |
+    /// | Min rate             | `lance_aimd_min_rate`            | `LANCE_AIMD_MIN_RATE`            | 1       |
+    /// | Max rate             | `lance_aimd_max_rate`            | `LANCE_AIMD_MAX_RATE`            | 5000    |
+    /// | Decrease factor      | `lance_aimd_decrease_factor`     | `LANCE_AIMD_DECREASE_FACTOR`     | 0.5     |
+    /// | Additive increment   | `lance_aimd_additive_increment`  | `LANCE_AIMD_ADDITIVE_INCREMENT`  | 300     |
+    /// | Burst capacity       | `lance_aimd_burst_capacity`      | `LANCE_AIMD_BURST_CAPACITY`      | 100     |
+    /// | Max retries          | `lance_aimd_max_retries`         | `LANCE_AIMD_MAX_RETRIES`         | 3       |
+    /// | Min backoff ms       | `lance_aimd_min_backoff_ms`      | `LANCE_AIMD_MIN_BACKOFF_MS`      | 100     |
+    /// | Max backoff ms       | `lance_aimd_max_backoff_ms`      | `LANCE_AIMD_MAX_BACKOFF_MS`      | 300     |
+    pub fn from_storage_options(
+        storage_options: Option<&HashMap<String, String>>,
+    ) -> lance_core::Result<Self> {
+        fn resolve_f64(
+            key: &str,
+            storage_options: Option<&HashMap<String, String>>,
+            default: f64,
+        ) -> lance_core::Result<f64> {
+            let env_key = key.to_ascii_uppercase();
+            if let Some(val) = storage_options.and_then(|opts| opts.get(key)) {
+                val.parse::<f64>().map_err(|_| {
+                    lance_core::Error::invalid_input(format!(
+                        "Invalid value for storage option '{key}': '{val}'"
+                    ))
+                })
+            } else if let Ok(val) = std::env::var(&env_key) {
+                val.parse::<f64>().map_err(|_| {
+                    lance_core::Error::invalid_input(format!(
+                        "Invalid value for env var '{env_key}': '{val}'"
+                    ))
+                })
+            } else {
+                Ok(default)
+            }
+        }
+
+        fn resolve_u32(
+            key: &str,
+            storage_options: Option<&HashMap<String, String>>,
+            default: u32,
+        ) -> lance_core::Result<u32> {
+            let env_key = key.to_ascii_uppercase();
+            if let Some(val) = storage_options.and_then(|opts| opts.get(key)) {
+                val.parse::<u32>().map_err(|_| {
+                    lance_core::Error::invalid_input(format!(
+                        "Invalid value for storage option '{key}': '{val}'"
+                    ))
+                })
+            } else if let Ok(val) = std::env::var(&env_key) {
+                val.parse::<u32>().map_err(|_| {
+                    lance_core::Error::invalid_input(format!(
+                        "Invalid value for env var '{env_key}': '{val}'"
+                    ))
+                })
+            } else {
+                Ok(default)
+            }
+        }
+
+        fn resolve_usize(
+            key: &str,
+            storage_options: Option<&HashMap<String, String>>,
+            default: usize,
+        ) -> lance_core::Result<usize> {
+            let env_key = key.to_ascii_uppercase();
+            if let Some(val) = storage_options.and_then(|opts| opts.get(key)) {
+                val.parse::<usize>().map_err(|_| {
+                    lance_core::Error::invalid_input(format!(
+                        "Invalid value for storage option '{key}': '{val}'"
+                    ))
+                })
+            } else if let Ok(val) = std::env::var(&env_key) {
+                val.parse::<usize>().map_err(|_| {
+                    lance_core::Error::invalid_input(format!(
+                        "Invalid value for env var '{env_key}': '{val}'"
+                    ))
+                })
+            } else {
+                Ok(default)
+            }
+        }
+
+        fn resolve_u64(
+            key: &str,
+            storage_options: Option<&HashMap<String, String>>,
+            default: u64,
+        ) -> lance_core::Result<u64> {
+            let env_key = key.to_ascii_uppercase();
+            if let Some(val) = storage_options.and_then(|opts| opts.get(key)) {
+                val.parse::<u64>().map_err(|_| {
+                    lance_core::Error::invalid_input(format!(
+                        "Invalid value for storage option '{key}': '{val}'"
+                    ))
+                })
+            } else if let Ok(val) = std::env::var(&env_key) {
+                val.parse::<u64>().map_err(|_| {
+                    lance_core::Error::invalid_input(format!(
+                        "Invalid value for env var '{env_key}': '{val}'"
+                    ))
+                })
+            } else {
+                Ok(default)
+            }
+        }
+
+        let initial_rate = resolve_f64("lance_aimd_initial_rate", storage_options, 2000.0)?;
+        let min_rate = resolve_f64("lance_aimd_min_rate", storage_options, 1.0)?;
+        let max_rate = resolve_f64("lance_aimd_max_rate", storage_options, 5000.0)?;
+        let decrease_factor = resolve_f64("lance_aimd_decrease_factor", storage_options, 0.5)?;
+        let additive_increment =
+            resolve_f64("lance_aimd_additive_increment", storage_options, 300.0)?;
+        let burst_capacity = resolve_u32("lance_aimd_burst_capacity", storage_options, 100)?;
+        let max_retries = resolve_usize("lance_aimd_max_retries", storage_options, 3)?;
+        let min_backoff_ms = resolve_u64("lance_aimd_min_backoff_ms", storage_options, 100)?;
+        let max_backoff_ms = resolve_u64("lance_aimd_max_backoff_ms", storage_options, 300)?;
+
+        let aimd = AimdConfig::default()
+            .with_initial_rate(initial_rate)
+            .with_min_rate(min_rate)
+            .with_max_rate(max_rate)
+            .with_decrease_factor(decrease_factor)
+            .with_additive_increment(additive_increment);
+
+        Ok(Self {
+            max_retries,
+            min_backoff_ms,
+            max_backoff_ms,
+            ..Self::default()
+                .with_aimd(aimd)
+                .with_burst_capacity(burst_capacity)
+        })
+    }
+}
+
+struct TokenBucketState {
+    tokens: f64,
+    last_refill: std::time::Instant,
+    rate: f64,
+}
+
+/// Per-category throttle state: an AIMD controller paired with a token bucket.
+struct OperationThrottle {
+    controller: AimdController,
+    bucket: Mutex<TokenBucketState>,
+    burst_capacity: f64,
+    max_retries: usize,
+    min_backoff_ms: u64,
+    max_backoff_ms: u64,
+}
+
+impl OperationThrottle {
+    fn new(
+        aimd_config: AimdConfig,
+        burst_capacity: f64,
+        max_retries: usize,
+        min_backoff_ms: u64,
+        max_backoff_ms: u64,
+    ) -> lance_core::Result<Self> {
+        let initial_rate = aimd_config.initial_rate;
+        let controller = AimdController::new(aimd_config)?;
+        Ok(Self {
+            controller,
+            bucket: Mutex::new(TokenBucketState {
+                tokens: burst_capacity,
+                last_refill: std::time::Instant::now(),
+                rate: initial_rate,
+            }),
+            burst_capacity,
+            max_retries,
+            min_backoff_ms,
+            max_backoff_ms,
+        })
+    }
+
+    /// Acquire a token from the bucket, sleeping if none are available.
+    ///
+    /// Each caller reserves a token immediately (allowing `tokens` to go
+    /// negative) so that concurrent waiters queue behind each other instead
+    /// of all waking at the same instant (thundering herd).
+    async fn acquire_token(&self) {
+        let sleep_duration = {
+            let mut bucket = self.bucket.lock().await;
+            let now = std::time::Instant::now();
+            let elapsed = now.duration_since(bucket.last_refill).as_secs_f64();
+            bucket.tokens = (bucket.tokens + elapsed * bucket.rate).min(self.burst_capacity);
+            bucket.last_refill = now;
+
+            // Reserve a token (may go negative to queue behind other waiters)
+            bucket.tokens -= 1.0;
+
+            if bucket.tokens >= 0.0 {
+                // Had a token available, no need to sleep
+                return;
+            }
+
+            // Sleep proportional to our position in the queue
+            std::time::Duration::from_secs_f64(-bucket.tokens / bucket.rate)
+        };
+
+        tokio::time::sleep(sleep_duration).await;
+    }
+
+    /// Update the bucket's fill rate from the controller.
+    async fn update_bucket_rate(&self, new_rate: f64) {
+        let mut bucket = self.bucket.lock().await;
+        bucket.rate = new_rate;
+    }
+
+    /// Classify a result and feed it back to the AIMD controller without
+    /// acquiring a token. Uses `try_lock` for the bucket update so that if the
+    /// bucket lock is contended the rate update is deferred to the next
+    /// `throttled()` call.
+    fn observe_outcome<T>(&self, result: &OSResult<T>) {
+        let outcome = match result {
+            Ok(_) => RequestOutcome::Success,
+            Err(err) if is_throttle_error(err) => {
+                debug!("Throttle error detected in stream");
+                RequestOutcome::Throttled
+            }
+            Err(_) => RequestOutcome::Success,
+        };
+        let prev_rate = self.controller.current_rate();
+        let new_rate = self.controller.record_outcome(outcome);
+        if new_rate < prev_rate {
+            warn!(
+                previous_rate = format!("{prev_rate:.1}"),
+                new_rate = format!("{new_rate:.1}"),
+                "AIMD throttle: rate reduced due to throttle errors"
+            );
+        }
+        if let Ok(mut bucket) = self.bucket.try_lock() {
+            bucket.rate = new_rate;
+        }
+    }
+
+    /// Execute an operation with throttling: acquire token, run, classify result.
+    /// On throttle errors, retries up to `max_retries` times with a random
+    /// backoff between `min_backoff_ms` and `max_backoff_ms` between attempts.
+    async fn throttled<T, F, Fut>(&self, f: F) -> OSResult<T>
+    where
+        F: Fn() -> Fut,
+        Fut: std::future::Future<Output = OSResult<T>>,
+    {
+        for attempt in 0..=self.max_retries {
+            self.acquire_token().await;
+            let result = f().await;
+            let outcome = match &result {
+                Ok(_) => RequestOutcome::Success,
+                Err(err) if is_throttle_error(err) => {
+                    debug!("Throttle error detected");
+                    RequestOutcome::Throttled
+                }
+                Err(_) => RequestOutcome::Success, // Non-throttle errors don't indicate capacity problems
+            };
+            let prev_rate = self.controller.current_rate();
+            let new_rate = self.controller.record_outcome(outcome);
+            if new_rate < prev_rate {
+                warn!(
+                    previous_rate = format!("{prev_rate:.1}"),
+                    new_rate = format!("{new_rate:.1}"),
+                    "AIMD throttle: rate reduced due to throttle errors"
+                );
+            }
+            self.update_bucket_rate(new_rate).await;
+
+            match &result {
+                Err(err) if is_throttle_error(err) && attempt < self.max_retries => {
+                    let backoff_ms =
+                        rand::rng().random_range(self.min_backoff_ms..=self.max_backoff_ms);
+                    debug!(
+                        attempt = attempt + 1,
+                        max_retries = self.max_retries,
+                        backoff_ms,
+                        "Retrying after throttle error"
+                    );
+                    tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await;
+                    continue;
+                }
+                _ => return result,
+            }
+        }
+        unreachable!()
+    }
+}
+
+impl Debug for OperationThrottle {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("OperationThrottle")
+            .field("controller", &self.controller)
+            .field("burst_capacity", &self.burst_capacity)
+            .finish()
+    }
+}
+
+/// A [`MultipartUpload`] wrapper that throttles and retries `put_part`,
+/// `complete`, and `abort`, feeding outcomes back to the write AIMD
+/// controller.
+struct ThrottledMultipartUpload {
+    target: Box<dyn MultipartUpload>,
+    write: Arc<OperationThrottle>,
+}
+
+impl Debug for ThrottledMultipartUpload {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ThrottledMultipartUpload").finish()
+    }
+}
+
+#[async_trait]
+impl MultipartUpload for ThrottledMultipartUpload {
+    fn put_part(&mut self, data: PutPayload) -> UploadPart {
+        let write = Arc::clone(&self.write);
+        // Call put_part synchronously to preserve part ordering regardless
+        // of which futures are awaited first.
+        let fut = self.target.put_part(data);
+        Box::pin(async move {
+            write.acquire_token().await;
+            let result = fut.await;
+            write.observe_outcome(&result);
+            result
+        })
+    }
+
+    async fn complete(&mut self) -> OSResult<PutResult> {
+        let target = &mut self.target;
+        for attempt in 0..=self.write.max_retries {
+            self.write.acquire_token().await;
+            let result = target.complete().await;
+            self.write.observe_outcome(&result);
+
+            match &result {
+                Err(err) if is_throttle_error(err) && attempt < self.write.max_retries => {
+                    let backoff_ms = rand::rng()
+                        .random_range(self.write.min_backoff_ms..=self.write.max_backoff_ms);
+                    tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await;
+                    continue;
+                }
+                _ => return result,
+            }
+        }
+        unreachable!()
+    }
+
+    async fn abort(&mut self) -> OSResult<()> {
+        let target = &mut self.target;
+        for attempt in 0..=self.write.max_retries {
+            self.write.acquire_token().await;
+            let result = target.abort().await;
+            self.write.observe_outcome(&result);
+
+            match &result {
+                Err(err) if is_throttle_error(err) && attempt < self.write.max_retries => {
+                    let backoff_ms = rand::rng()
+                        .random_range(self.write.min_backoff_ms..=self.write.max_backoff_ms);
+                    tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await;
+                    continue;
+                }
+                _ => return result,
+            }
+        }
+        unreachable!()
+    }
+}
+
+/// An ObjectStore wrapper that rate-limits operations using per-category token
+/// buckets whose fill rates are controlled by AIMD algorithms.
+///
+/// Operations are split into four independent categories:
+/// - **read**: `get`, `get_opts`, `get_range`, `get_ranges`, `head`
+/// - **write**: `put`, `put_opts`, `put_multipart`, `put_multipart_opts`, `copy`, `copy_if_not_exists`, `rename`, `rename_if_not_exists`
+/// - **delete**: `delete`
+/// - **list**: `list_with_delimiter`
+///
+/// Streaming operations (`list`, `list_with_offset`, `delete_stream`) do not acquire tokens,
+/// but observe each yielded item and feed the result back to the AIMD controller so it can
+/// adjust the rate for other operations in the same category.
+///
+/// This is not perfect but probably as close as we can get without moving the throttle into
+/// the object_store crate itself.
+pub struct AimdThrottledStore {
+    target: Arc<dyn ObjectStore>,
+    read: Arc<OperationThrottle>,
+    write: Arc<OperationThrottle>,
+    delete: Arc<OperationThrottle>,
+    list: Arc<OperationThrottle>,
+}
+
+impl Debug for AimdThrottledStore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("AimdThrottledStore")
+            .field("target", &self.target)
+            .field("read", &self.read)
+            .field("write", &self.write)
+            .field("delete", &self.delete)
+            .field("list", &self.list)
+            .finish()
+    }
+}
+
+impl Display for AimdThrottledStore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "AimdThrottledStore({})", self.target)
+    }
+}
+
+impl AimdThrottledStore {
+    pub fn new(
+        target: Arc<dyn ObjectStore>,
+        config: AimdThrottleConfig,
+    ) -> lance_core::Result<Self> {
+        let burst = config.burst_capacity as f64;
+        let max_retries = config.max_retries;
+        let min_backoff_ms = config.min_backoff_ms;
+        let max_backoff_ms = config.max_backoff_ms;
+        Ok(Self {
+            target,
+            read: Arc::new(OperationThrottle::new(
+                config.read,
+                burst,
+                max_retries,
+                min_backoff_ms,
+                max_backoff_ms,
+            )?),
+            write: Arc::new(OperationThrottle::new(
+                config.write,
+                burst,
+                max_retries,
+                min_backoff_ms,
+                max_backoff_ms,
+            )?),
+            delete: Arc::new(OperationThrottle::new(
+                config.delete,
+                burst,
+                max_retries,
+                min_backoff_ms,
+                max_backoff_ms,
+            )?),
+            list: Arc::new(OperationThrottle::new(
+                config.list,
+                burst,
+                max_retries,
+                min_backoff_ms,
+                max_backoff_ms,
+            )?),
+        })
+    }
+}
+
+#[async_trait]
+#[deny(clippy::missing_trait_methods)]
+impl ObjectStore for AimdThrottledStore {
+    async fn put(&self, location: &Path, bytes: PutPayload) -> OSResult<PutResult> {
+        self.write
+            .throttled(|| self.target.put(location, bytes.clone()))
+            .await
+    }
+
+    async fn put_opts(
+        &self,
+        location: &Path,
+        bytes: PutPayload,
+        opts: PutOptions,
+    ) -> OSResult<PutResult> {
+        self.write
+            .throttled(|| self.target.put_opts(location, bytes.clone(), opts.clone()))
+            .await
+    }
+
+    async fn put_multipart(&self, location: &Path) -> OSResult<Box<dyn MultipartUpload>> {
+        let target = self
+            .write
+            .throttled(|| self.target.put_multipart(location))
+            .await?;
+        Ok(Box::new(ThrottledMultipartUpload {
+            target,
+            write: Arc::clone(&self.write),
+        }))
+    }
+
+    async fn put_multipart_opts(
+        &self,
+        location: &Path,
+        opts: PutMultipartOptions,
+    ) -> OSResult<Box<dyn MultipartUpload>> {
+        let target = self
+            .write
+            .throttled(|| self.target.put_multipart_opts(location, opts.clone()))
+            .await?;
+        Ok(Box::new(ThrottledMultipartUpload {
+            target,
+            write: Arc::clone(&self.write),
+        }))
+    }
+
+    async fn get(&self, location: &Path) -> OSResult<GetResult> {
+        self.read.throttled(|| self.target.get(location)).await
+    }
+
+    async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+        self.read
+            .throttled(|| self.target.get_opts(location, options.clone()))
+            .await
+    }
+
+    async fn get_range(&self, location: &Path, range: Range<u64>) -> OSResult<Bytes> {
+        self.read
+            .throttled(|| self.target.get_range(location, range.clone()))
+            .await
+    }
+
+    async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+        self.read
+            .throttled(|| self.target.get_ranges(location, ranges))
+            .await
+    }
+
+    async fn head(&self, location: &Path) -> OSResult<ObjectMeta> {
+        self.read.throttled(|| self.target.head(location)).await
+    }
+
+    async fn delete(&self, location: &Path) -> OSResult<()> {
+        self.delete.throttled(|| self.target.delete(location)).await
+    }
+
+    fn delete_stream<'a>(
+        &'a self,
+        locations: BoxStream<'a, OSResult<Path>>,
+    ) -> BoxStream<'a, OSResult<Path>> {
+        self.target
+            .delete_stream(locations)
+            .map(|item| {
+                self.delete.observe_outcome(&item);
+                item
+            })
+            .boxed()
+    }
+
+    fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+        let throttle = Arc::clone(&self.list);
+        self.target
+            .list(prefix)
+            .map(move |item| {
+                throttle.observe_outcome(&item);
+                item
+            })
+            .boxed()
+    }
+
+    fn list_with_offset(
+        &self,
+        prefix: Option<&Path>,
+        offset: &Path,
+    ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+        let throttle = Arc::clone(&self.list);
+        self.target
+            .list_with_offset(prefix, offset)
+            .map(move |item| {
+                throttle.observe_outcome(&item);
+                item
+            })
+            .boxed()
+    }
+
+    async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult<ListResult> {
+        self.list
+            .throttled(|| self.target.list_with_delimiter(prefix))
+            .await
+    }
+
+    async fn copy(&self, from: &Path, to: &Path) -> OSResult<()> {
+        self.write.throttled(|| self.target.copy(from, to)).await
+    }
+
+    async fn rename(&self, from: &Path, to: &Path) -> OSResult<()> {
+        self.write.throttled(|| self.target.rename(from, to)).await
+    }
+
+    async fn rename_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
+        self.write
+            .throttled(|| self.target.rename_if_not_exists(from, to))
+            .await
+    }
+
+    async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
+        self.write
+            .throttled(|| self.target.copy_if_not_exists(from, to))
+            .await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use object_store::memory::InMemory;
+    use rstest::rstest;
+    use std::collections::VecDeque;
+    use std::sync::atomic::{AtomicU64, Ordering};
+
+    fn make_generic_error(msg: &str) -> object_store::Error {
+        object_store::Error::Generic {
+            store: "test",
+            source: msg.into(),
+        }
+    }
+
+    #[rstest]
+    #[case::retry_error("Error after 10 retries, max_retries: 10, retry_timeout: 180s", true)]
+    #[case::retries_in_message(
+        "request failed, after 3 retries, max_retries: 5, retry_timeout: 60s",
+        true
+    )]
+    #[case::not_found("Object not found", false)]
+    #[case::permission_denied("Access denied", false)]
+    #[case::timeout("Connection timed out", false)]
+    #[case::http_429_without_retries("HTTP 429 Too Many Requests", false)]
+    #[case::slowdown_without_retries("SlowDown: Please reduce your request rate", false)]
+    fn test_is_throttle_error(#[case] msg: &str, #[case] expected: bool) {
+        let err = make_generic_error(msg);
+        assert_eq!(
+            is_throttle_error(&err),
+            expected,
+            "is_throttle_error for '{}' should be {}",
+            msg,
+            expected
+        );
+    }
+
+    #[test]
+    fn test_non_generic_errors_are_not_throttle() {
+        let err = object_store::Error::NotFound {
+            path: "test".to_string(),
+            source: "not found".into(),
+        };
+        assert!(!is_throttle_error(&err));
+    }
+
+    #[tokio::test]
+    async fn test_basic_put_get_through_wrapper() {
+        let store = Arc::new(InMemory::new());
+        let config = AimdThrottleConfig::default();
+        let throttled = AimdThrottledStore::new(store, config).unwrap();
+
+        let path = Path::from("test/file.txt");
+        let data = PutPayload::from_static(b"hello world");
+        throttled.put(&path, data).await.unwrap();
+
+        let result = throttled.get(&path).await.unwrap();
+        let bytes = result.bytes().await.unwrap();
+        assert_eq!(bytes.as_ref(), b"hello world");
+    }
+
+    #[tokio::test]
+    async fn test_rate_decreases_on_throttle() {
+        let store = Arc::new(InMemory::new());
+        let config = AimdThrottleConfig::default().with_aimd(
+            AimdConfig::default()
+                .with_initial_rate(100.0)
+                .with_decrease_factor(0.5)
+                .with_window_duration(std::time::Duration::from_millis(10)),
+        );
+        let throttled = AimdThrottledStore::new(store, config).unwrap();
+
+        let initial_rate = throttled.read.controller.current_rate();
+        assert_eq!(initial_rate, 100.0);
+
+        // Simulate a throttle outcome directly
+        throttled
+            .read
+            .controller
+            .record_outcome(RequestOutcome::Throttled);
+
+        // Wait for window to expire and trigger evaluation
+        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+        throttled
+            .read
+            .controller
+            .record_outcome(RequestOutcome::Success);
+
+        let new_rate = throttled.read.controller.current_rate();
+        assert!(
+            new_rate < initial_rate,
+            "Rate should decrease after throttle: {} < {}",
+            new_rate,
+            initial_rate
+        );
+    }
+
+    #[tokio::test]
+    async fn test_rate_recovers_on_success() {
+        let store = Arc::new(InMemory::new());
+        let config = AimdThrottleConfig::default().with_aimd(
+            AimdConfig::default()
+                .with_initial_rate(100.0)
+                .with_decrease_factor(0.5)
+                .with_additive_increment(10.0)
+                .with_window_duration(std::time::Duration::from_millis(10)),
+        );
+        let throttled = AimdThrottledStore::new(store, config).unwrap();
+
+        // First decrease via throttle
+        throttled
+            .read
+            .controller
+            .record_outcome(RequestOutcome::Throttled);
+        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+        throttled
+            .read
+            .controller
+            .record_outcome(RequestOutcome::Success);
+        let decreased_rate = throttled.read.controller.current_rate();
+        assert_eq!(decreased_rate, 50.0);
+
+        // Now recover via success
+        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+        throttled
+            .read
+            .controller
+            .record_outcome(RequestOutcome::Success);
+        let recovered_rate = throttled.read.controller.current_rate();
+        assert_eq!(recovered_rate, 60.0);
+    }
+
+    #[tokio::test]
+    async fn test_as_dyn_object_store() {
+        let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+        let throttled: Arc<dyn ObjectStore> =
+            Arc::new(AimdThrottledStore::new(store, AimdThrottleConfig::default()).unwrap());
+
+        let path = Path::from("test/data.bin");
+        let data = PutPayload::from_static(b"test data");
+        throttled.put(&path, data).await.unwrap();
+
+        let result = throttled.get(&path).await.unwrap();
+        let bytes = result.bytes().await.unwrap();
+        assert_eq!(bytes.as_ref(), b"test data");
+    }
+
+    #[tokio::test]
+    async fn test_token_bucket_delays_when_exhausted() {
+        let store = Arc::new(InMemory::new());
+        // Very low rate and burst capacity to force waiting
+        let config = AimdThrottleConfig::default()
+            .with_burst_capacity(1)
+            .with_aimd(AimdConfig::default().with_initial_rate(10.0));
+        let throttled = Arc::new(AimdThrottledStore::new(store, config).unwrap());
+
+        let path = Path::from("test/file.txt");
+        let data = PutPayload::from_static(b"data");
+        throttled.put(&path, data).await.unwrap();
+
+        // After consuming the burst token, the next request should take ~100ms
+        // (1 token / 10 tokens-per-sec). We verify it takes at least 50ms.
+        let start = std::time::Instant::now();
+        let data2 = PutPayload::from_static(b"data2");
+        throttled.put(&path, data2).await.unwrap();
+        let elapsed = start.elapsed();
+
+        assert!(
+            elapsed >= std::time::Duration::from_millis(50),
+            "Expected delay for token refill, but elapsed was {:?}",
+            elapsed
+        );
+    }
+
+    #[tokio::test]
+    async fn test_list_observes_outcomes() {
+        let store = Arc::new(InMemory::new());
+        let config = AimdThrottleConfig::default();
+        let throttled = AimdThrottledStore::new(store.clone(), config).unwrap();
+
+        let path = Path::from("prefix/file.txt");
+        let data = PutPayload::from_static(b"data");
+        store.put(&path, data).await.unwrap();
+
+        let items: Vec<_> = throttled.list(Some(&Path::from("prefix"))).collect().await;
+        assert_eq!(items.len(), 1);
+        assert!(items[0].is_ok());
+    }
+
+    /// A mock store whose `list` stream yields a configurable sequence of
+    /// Ok / throttle-error items. Used to verify that the AIMD wrapper
+    /// observes errors surfaced inside list streams.
+    struct ThrottlingListMockStore {
+        inner: InMemory,
+        /// Number of throttle errors to inject at the start of each list call.
+        throttle_count: usize,
+    }
+
+    impl Display for ThrottlingListMockStore {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            write!(f, "ThrottlingListMockStore")
+        }
+    }
+
+    impl Debug for ThrottlingListMockStore {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            f.debug_struct("ThrottlingListMockStore").finish()
+        }
+    }
+
+    #[async_trait]
+    impl ObjectStore for ThrottlingListMockStore {
+        async fn put(&self, location: &Path, bytes: PutPayload) -> OSResult<PutResult> {
+            self.inner.put(location, bytes).await
+        }
+        async fn put_opts(
+            &self,
+            location: &Path,
+            bytes: PutPayload,
+            opts: PutOptions,
+        ) -> OSResult<PutResult> {
+            self.inner.put_opts(location, bytes, opts).await
+        }
+        async fn put_multipart(&self, location: &Path) -> OSResult<Box<dyn MultipartUpload>> {
+            self.inner.put_multipart(location).await
+        }
+        async fn put_multipart_opts(
+            &self,
+            location: &Path,
+            opts: PutMultipartOptions,
+        ) -> OSResult<Box<dyn MultipartUpload>> {
+            self.inner.put_multipart_opts(location, opts).await
+        }
+        async fn get(&self, location: &Path) -> OSResult<GetResult> {
+            self.inner.get(location).await
+        }
+        async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+            self.inner.get_opts(location, options).await
+        }
+        async fn get_range(&self, location: &Path, range: Range<u64>) -> OSResult<Bytes> {
+            self.inner.get_range(location, range).await
+        }
+        async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+            self.inner.get_ranges(location, ranges).await
+        }
+        async fn head(&self, location: &Path) -> OSResult<ObjectMeta> {
+            self.inner.head(location).await
+        }
+        async fn delete(&self, location: &Path) -> OSResult<()> {
+            self.inner.delete(location).await
+        }
+        fn delete_stream<'a>(
+            &'a self,
+            locations: BoxStream<'a, OSResult<Path>>,
+        ) -> BoxStream<'a, OSResult<Path>> {
+            self.inner.delete_stream(locations)
+        }
+        fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            let n = self.throttle_count;
+            let inner_stream = self.inner.list(prefix);
+            let errors = futures::stream::iter((0..n).map(|_| {
+                Err(object_store::Error::Generic {
+                    store: "ThrottlingListMock",
+                    source: "request failed, after 3 retries, max_retries: 5, retry_timeout: 60s"
+                        .into(),
+                })
+            }));
+            errors.chain(inner_stream).boxed()
+        }
+        fn list_with_offset(
+            &self,
+            prefix: Option<&Path>,
+            offset: &Path,
+        ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            self.inner.list_with_offset(prefix, offset)
+        }
+        async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult<ListResult> {
+            self.inner.list_with_delimiter(prefix).await
+        }
+        async fn copy(&self, from: &Path, to: &Path) -> OSResult<()> {
+            self.inner.copy(from, to).await
+        }
+        async fn rename(&self, from: &Path, to: &Path) -> OSResult<()> {
+            self.inner.rename(from, to).await
+        }
+        async fn rename_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
+            self.inner.rename_if_not_exists(from, to).await
+        }
+        async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
+            self.inner.copy_if_not_exists(from, to).await
+        }
+    }
+
+    #[tokio::test]
+    async fn test_list_stream_throttle_errors_decrease_rate() {
+        let mock = Arc::new(ThrottlingListMockStore {
+            inner: InMemory::new(),
+            throttle_count: 5,
+        });
+
+        // Seed a file so the real items come through after the errors.
+        mock.put(
+            &Path::from("prefix/file.txt"),
+            PutPayload::from_static(b"data"),
+        )
+        .await
+        .unwrap();
+
+        let config = AimdThrottleConfig::default().with_list_aimd(
+            AimdConfig::default()
+                .with_initial_rate(100.0)
+                .with_decrease_factor(0.5)
+                .with_window_duration(std::time::Duration::from_millis(10)),
+        );
+        let throttled = AimdThrottledStore::new(mock as Arc<dyn ObjectStore>, config).unwrap();
+
+        let initial_rate = throttled.list.controller.current_rate();
+        assert_eq!(initial_rate, 100.0);
+
+        let items: Vec<_> = throttled.list(Some(&Path::from("prefix"))).collect().await;
+
+        // 5 errors + 1 real item
+        assert_eq!(items.len(), 6);
+        assert!(items[0].is_err());
+        assert!(items[5].is_ok());
+
+        // Wait for the AIMD window to expire and trigger evaluation.
+        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+        throttled
+            .list
+            .controller
+            .record_outcome(RequestOutcome::Success);
+
+        let new_rate = throttled.list.controller.current_rate();
+        assert!(
+            new_rate < initial_rate,
+            "List rate should decrease after stream throttle errors: {} < {}",
+            new_rate,
+            initial_rate
+        );
+    }
+
+    #[tokio::test]
+    async fn test_per_category_independence() {
+        let store = Arc::new(InMemory::new());
+        let config = AimdThrottleConfig::default().with_aimd(
+            AimdConfig::default()
+                .with_initial_rate(100.0)
+                .with_decrease_factor(0.5)
+                .with_window_duration(std::time::Duration::from_millis(10)),
+        );
+        let throttled = AimdThrottledStore::new(store, config).unwrap();
+
+        // Push the read controller into a throttled state
+        throttled
+            .read
+            .controller
+            .record_outcome(RequestOutcome::Throttled);
+        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+        throttled
+            .read
+            .controller
+            .record_outcome(RequestOutcome::Success);
+
+        let read_rate = throttled.read.controller.current_rate();
+        let write_rate = throttled.write.controller.current_rate();
+        let delete_rate = throttled.delete.controller.current_rate();
+        let list_rate = throttled.list.controller.current_rate();
+
+        assert_eq!(read_rate, 50.0, "Read rate should have decreased");
+        assert_eq!(write_rate, 100.0, "Write rate should be unaffected");
+        assert_eq!(delete_rate, 100.0, "Delete rate should be unaffected");
+        assert_eq!(list_rate, 100.0, "List rate should be unaffected");
+    }
+
+    #[tokio::test]
+    async fn test_per_category_config() {
+        let store = Arc::new(InMemory::new());
+        let config = AimdThrottleConfig::default()
+            .with_read_aimd(AimdConfig::default().with_initial_rate(200.0))
+            .with_write_aimd(AimdConfig::default().with_initial_rate(100.0))
+            .with_delete_aimd(AimdConfig::default().with_initial_rate(50.0))
+            .with_list_aimd(AimdConfig::default().with_initial_rate(25.0));
+        let throttled = AimdThrottledStore::new(store, config).unwrap();
+
+        assert_eq!(throttled.read.controller.current_rate(), 200.0);
+        assert_eq!(throttled.write.controller.current_rate(), 100.0);
+        assert_eq!(throttled.delete.controller.current_rate(), 50.0);
+        assert_eq!(throttled.list.controller.current_rate(), 25.0);
+    }
+
+    /// A mock [`ObjectStore`] that measures request rate over a sliding window
+    /// and returns 503 errors when the rate exceeds a configurable threshold.
+    /// Write and metadata-only operations are not rate-limited.
+    struct RateLimitingMockStore {
+        inner: InMemory,
+        /// Timestamps of recent successful (admitted) requests.
+        timestamps: std::sync::Mutex<VecDeque<std::time::Instant>>,
+        /// Maximum requests allowed within `window`.
+        max_per_window: usize,
+        /// Sliding window duration.
+        window: std::time::Duration,
+        success_count: AtomicU64,
+        throttle_count: AtomicU64,
+    }
+
+    impl RateLimitingMockStore {
+        fn new(max_per_window: usize, window: std::time::Duration) -> Self {
+            Self {
+                inner: InMemory::new(),
+                timestamps: std::sync::Mutex::new(VecDeque::new()),
+                max_per_window,
+                window,
+                success_count: AtomicU64::new(0),
+                throttle_count: AtomicU64::new(0),
+            }
+        }
+
+        /// Returns `true` if the request is admitted, `false` if throttled.
+        fn check_rate(&self) -> bool {
+            let mut ts = self.timestamps.lock().unwrap();
+            let now = std::time::Instant::now();
+            while let Some(&front) = ts.front() {
+                if now.duration_since(front) > self.window {
+                    ts.pop_front();
+                } else {
+                    break;
+                }
+            }
+            if ts.len() >= self.max_per_window {
+                self.throttle_count.fetch_add(1, Ordering::Relaxed);
+                false
+            } else {
+                ts.push_back(now);
+                self.success_count.fetch_add(1, Ordering::Relaxed);
+                true
+            }
+        }
+
+        fn throttle_error() -> object_store::Error {
+            object_store::Error::Generic {
+                store: "RateLimitingMock",
+                source: "request failed, after 10 retries, max_retries: 10, retry_timeout: 180s"
+                    .into(),
+            }
+        }
+    }
+
+    impl Display for RateLimitingMockStore {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            write!(f, "RateLimitingMockStore")
+        }
+    }
+
+    impl Debug for RateLimitingMockStore {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            f.debug_struct("RateLimitingMockStore").finish()
+        }
+    }
+
+    #[async_trait]
+    impl ObjectStore for RateLimitingMockStore {
+        async fn put(&self, location: &Path, bytes: PutPayload) -> OSResult<PutResult> {
+            self.inner.put(location, bytes).await
+        }
+
+        async fn put_opts(
+            &self,
+            location: &Path,
+            bytes: PutPayload,
+            opts: PutOptions,
+        ) -> OSResult<PutResult> {
+            self.inner.put_opts(location, bytes, opts).await
+        }
+
+        async fn put_multipart(&self, location: &Path) -> OSResult<Box<dyn MultipartUpload>> {
+            self.inner.put_multipart(location).await
+        }
+
+        async fn put_multipart_opts(
+            &self,
+            location: &Path,
+            opts: PutMultipartOptions,
+        ) -> OSResult<Box<dyn MultipartUpload>> {
+            self.inner.put_multipart_opts(location, opts).await
+        }
+
+        async fn get(&self, location: &Path) -> OSResult<GetResult> {
+            if self.check_rate() {
+                self.inner.get(location).await
+            } else {
+                Err(Self::throttle_error())
+            }
+        }
+
+        async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+            if self.check_rate() {
+                self.inner.get_opts(location, options).await
+            } else {
+                Err(Self::throttle_error())
+            }
+        }
+
+        async fn get_range(&self, location: &Path, range: Range<u64>) -> OSResult<Bytes> {
+            if self.check_rate() {
+                self.inner.get_range(location, range).await
+            } else {
+                Err(Self::throttle_error())
+            }
+        }
+
+        async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+            if self.check_rate() {
+                self.inner.get_ranges(location, ranges).await
+            } else {
+                Err(Self::throttle_error())
+            }
+        }
+
+        async fn head(&self, location: &Path) -> OSResult<ObjectMeta> {
+            if self.check_rate() {
+                self.inner.head(location).await
+            } else {
+                Err(Self::throttle_error())
+            }
+        }
+
+        async fn delete(&self, location: &Path) -> OSResult<()> {
+            self.inner.delete(location).await
+        }
+
+        fn delete_stream<'a>(
+            &'a self,
+            locations: BoxStream<'a, OSResult<Path>>,
+        ) -> BoxStream<'a, OSResult<Path>> {
+            self.inner.delete_stream(locations)
+        }
+
+        fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            self.inner.list(prefix)
+        }
+
+        fn list_with_offset(
+            &self,
+            prefix: Option<&Path>,
+            offset: &Path,
+        ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            self.inner.list_with_offset(prefix, offset)
+        }
+
+        async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult<ListResult> {
+            self.inner.list_with_delimiter(prefix).await
+        }
+
+        async fn copy(&self, from: &Path, to: &Path) -> OSResult<()> {
+            self.inner.copy(from, to).await
+        }
+
+        async fn rename(&self, from: &Path, to: &Path) -> OSResult<()> {
+            self.inner.rename(from, to).await
+        }
+
+        async fn rename_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
+            self.inner.rename_if_not_exists(from, to).await
+        }
+
+        async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
+            self.inner.copy_if_not_exists(from, to).await
+        }
+    }
+
+    /// Verify that multiple concurrent readers sharing an AIMD-throttled store
+    /// converge to the backend's actual capacity.
+    ///
+    /// Setup:
+    /// - Mock backend allows 30 requests per 100ms (= 300 req/s).
+    /// - 5 reader tasks, each with their own [`AimdThrottledStore`] wrapping
+    ///   the shared mock.
+    /// - AIMD: 100ms window, initial rate 100 req/s, decrease 0.5, increase 2.
+    /// - Readers issue `head()` requests as fast as the throttle allows for 2s.
+    ///
+    /// Expected behaviour:
+    /// - Initial burst (100 burst tokens × 5 readers) overshoots the mock
+    ///   capacity, causing many 503s. Each reader's AIMD halves its rate.
+    /// - After the transient, each reader converges to ~60 req/s (300/5).
+    /// - Over 2 seconds, total successful requests should be in the range
+    ///   [300, 900] (theoretical max ≈ 600).
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn test_aimd_throttle_under_concurrent_load() {
+        let mock = Arc::new(RateLimitingMockStore::new(
+            30,
+            std::time::Duration::from_millis(100),
+        ));
+
+        // Seed a test file so head() succeeds when admitted.
+        let path = Path::from("test/data.bin");
+        mock.put(&path, PutPayload::from_static(b"test data"))
+            .await
+            .unwrap();
+
+        let aimd = AimdConfig::default()
+            .with_initial_rate(100.0)
+            .with_decrease_factor(0.5)
+            .with_additive_increment(2.0)
+            .with_window_duration(std::time::Duration::from_millis(100));
+        let throttle_config = AimdThrottleConfig::default()
+            .with_aimd(aimd)
+            .with_burst_capacity(100);
+
+        let num_readers = 5;
+        let test_duration = std::time::Duration::from_secs(2);
+        let mut handles = Vec::new();
+
+        for _ in 0..num_readers {
+            let store = Arc::new(
+                AimdThrottledStore::new(
+                    mock.clone() as Arc<dyn ObjectStore>,
+                    throttle_config.clone(),
+                )
+                .unwrap(),
+            );
+            let p = path.clone();
+            handles.push(tokio::spawn(async move {
+                let deadline = std::time::Instant::now() + test_duration;
+                let mut count = 0u64;
+                while std::time::Instant::now() < deadline {
+                    let _ = store.head(&p).await;
+                    count += 1;
+                }
+                count
+            }));
+        }
+
+        let mut total_reader_requests = 0u64;
+        for handle in handles {
+            total_reader_requests += handle.await.unwrap();
+        }
+
+        let successes = mock.success_count.load(Ordering::Relaxed);
+        let throttled = mock.throttle_count.load(Ordering::Relaxed);
+        let total_mock = successes + throttled;
+
+        // Mock-side count >= reader-side count because the AIMD layer retries
+        // throttle errors internally, causing multiple mock calls per reader call.
+        assert!(
+            total_mock >= total_reader_requests,
+            "Mock-side count ({total_mock}) should be >= reader-side count ({total_reader_requests})"
+        );
+
+        // Mock capacity is 30/100ms = 300 req/s. Over 2s the theoretical max is
+        // ~600 successful requests. With AIMD ramp-up, expect somewhat fewer.
+        assert!(
+            successes >= 300,
+            "Expected >= 300 successful requests over 2s, got {successes}"
+        );
+        assert!(
+            successes <= 900,
+            "Expected <= 900 successful requests, got {successes}"
+        );
+
+        // The initial burst exceeds mock capacity, so throttling must occur.
+        assert!(throttled > 0, "Expected some throttled requests but got 0");
+
+        // Without AIMD, raw tokio tasks against InMemory would fire 100k+ req/s.
+        // AIMD should keep the total well under 5000 over 2s.
+        assert!(
+            total_mock <= 5000,
+            "AIMD should limit total requests, got {total_mock}"
+        );
+    }
+
+    /// A mock store that returns a configurable number of throttle errors
+    /// before succeeding on `get` operations. Used to test the retry logic
+    /// inside `OperationThrottle::throttled()`.
+    struct RetryTestMockStore {
+        inner: InMemory,
+        /// Number of throttle errors remaining before success.
+        errors_remaining: std::sync::Mutex<usize>,
+        /// Total number of `get` calls observed.
+        get_call_count: AtomicU64,
+    }
+
+    impl RetryTestMockStore {
+        fn new(errors_before_success: usize) -> Self {
+            Self {
+                inner: InMemory::new(),
+                errors_remaining: std::sync::Mutex::new(errors_before_success),
+                get_call_count: AtomicU64::new(0),
+            }
+        }
+    }
+
+    impl Display for RetryTestMockStore {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            write!(f, "RetryTestMockStore")
+        }
+    }
+
+    impl Debug for RetryTestMockStore {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            f.debug_struct("RetryTestMockStore").finish()
+        }
+    }
+
+    #[async_trait]
+    impl ObjectStore for RetryTestMockStore {
+        async fn put(&self, location: &Path, bytes: PutPayload) -> OSResult<PutResult> {
+            self.inner.put(location, bytes).await
+        }
+        async fn put_opts(
+            &self,
+            location: &Path,
+            bytes: PutPayload,
+            opts: PutOptions,
+        ) -> OSResult<PutResult> {
+            self.inner.put_opts(location, bytes, opts).await
+        }
+        async fn put_multipart(&self, location: &Path) -> OSResult<Box<dyn MultipartUpload>> {
+            self.inner.put_multipart(location).await
+        }
+        async fn put_multipart_opts(
+            &self,
+            location: &Path,
+            opts: PutMultipartOptions,
+        ) -> OSResult<Box<dyn MultipartUpload>> {
+            self.inner.put_multipart_opts(location, opts).await
+        }
+        async fn get(&self, location: &Path) -> OSResult<GetResult> {
+            self.get_call_count.fetch_add(1, Ordering::Relaxed);
+            let should_error = {
+                let mut remaining = self.errors_remaining.lock().unwrap();
+                if *remaining > 0 {
+                    *remaining -= 1;
+                    true
+                } else {
+                    false
+                }
+            };
+            if should_error {
+                Err(object_store::Error::Generic {
+                    store: "RetryTestMock",
+                    source: "request failed, after 3 retries, max_retries: 3, retry_timeout: 30s"
+                        .into(),
+                })
+            } else {
+                self.inner.get(location).await
+            }
+        }
+        async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+            self.inner.get_opts(location, options).await
+        }
+        async fn get_range(&self, location: &Path, range: Range<u64>) -> OSResult<Bytes> {
+            self.inner.get_range(location, range).await
+        }
+        async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+            self.inner.get_ranges(location, ranges).await
+        }
+        async fn head(&self, location: &Path) -> OSResult<ObjectMeta> {
+            self.inner.head(location).await
+        }
+        async fn delete(&self, location: &Path) -> OSResult<()> {
+            self.inner.delete(location).await
+        }
+        fn delete_stream<'a>(
+            &'a self,
+            locations: BoxStream<'a, OSResult<Path>>,
+        ) -> BoxStream<'a, OSResult<Path>> {
+            self.inner.delete_stream(locations)
+        }
+        fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            self.inner.list(prefix)
+        }
+        fn list_with_offset(
+            &self,
+            prefix: Option<&Path>,
+            offset: &Path,
+        ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            self.inner.list_with_offset(prefix, offset)
+        }
+        async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult<ListResult> {
+            self.inner.list_with_delimiter(prefix).await
+        }
+        async fn copy(&self, from: &Path, to: &Path) -> OSResult<()> {
+            self.inner.copy(from, to).await
+        }
+        async fn rename(&self, from: &Path, to: &Path) -> OSResult<()> {
+            self.inner.rename(from, to).await
+        }
+        async fn rename_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
+            self.inner.rename_if_not_exists(from, to).await
+        }
+        async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
+            self.inner.copy_if_not_exists(from, to).await
+        }
+    }
+
+    #[tokio::test]
+    async fn test_throttled_retries_on_throttle_error_then_succeeds() {
+        // Mock returns 2 throttle errors then succeeds (within MAX_RETRIES=3)
+        let mock = Arc::new(RetryTestMockStore::new(2));
+        let path = Path::from("test/retry.txt");
+        mock.put(&path, PutPayload::from_static(b"retry data"))
+            .await
+            .unwrap();
+
+        let config = AimdThrottleConfig::default();
+        let throttled =
+            AimdThrottledStore::new(mock.clone() as Arc<dyn ObjectStore>, config).unwrap();
+
+        let result = throttled.get(&path).await;
+        assert!(result.is_ok(), "Expected success after retries");
+
+        let bytes = result.unwrap().bytes().await.unwrap();
+        assert_eq!(bytes.as_ref(), b"retry data");
+
+        // Should have called get 3 times total: 2 failures + 1 success
+        assert_eq!(mock.get_call_count.load(Ordering::Relaxed), 3);
+    }
+
+    #[tokio::test]
+    async fn test_throttled_fails_after_max_retries_exceeded() {
+        // Mock returns 4 throttle errors (more than MAX_RETRIES=3),
+        // so all 4 attempts (initial + 3 retries) will fail.
+        let mock = Arc::new(RetryTestMockStore::new(10));
+        let path = Path::from("test/fail.txt");
+        mock.put(&path, PutPayload::from_static(b"fail data"))
+            .await
+            .unwrap();
+
+        let config = AimdThrottleConfig::default();
+        let throttled =
+            AimdThrottledStore::new(mock.clone() as Arc<dyn ObjectStore>, config).unwrap();
+
+        let result = throttled.get(&path).await;
+        assert!(result.is_err(), "Expected error after max retries");
+        assert!(is_throttle_error(&result.unwrap_err()));
+
+        // Should have called get 4 times: initial attempt + 3 retries
+        assert_eq!(mock.get_call_count.load(Ordering::Relaxed), 4);
+    }
+
+    #[tokio::test]
+    async fn test_throttled_multipart_reorders_parts() {
+        let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+        let config = AimdThrottleConfig::default();
+        let throttled = AimdThrottledStore::new(store.clone(), config).unwrap();
+
+        let path = Path::from("test/multipart_ordering.bin");
+        let mut upload = throttled.put_multipart(&path).await.unwrap();
+
+        // Create futures for two parts in order: A then B.
+        let fut_a = upload.put_part(PutPayload::from_static(b"AAAA"));
+        let fut_b = upload.put_part(PutPayload::from_static(b"BBBB"));
+
+        // Await in REVERSE order. Part ordering should be determined by
+        // creation order (put_part call order), not by await order.
+        fut_b.await.unwrap();
+        fut_a.await.unwrap();
+
+        upload.complete().await.unwrap();
+
+        let result = store.get(&path).await.unwrap();
+        let bytes = result.bytes().await.unwrap();
+
+        assert_eq!(
+            bytes.as_ref(),
+            b"AAAABBBB",
+            "Parts were reordered! Got {:?} instead of AAAABBBB.",
+            std::str::from_utf8(&bytes).unwrap_or("<non-utf8>"),
+        );
+    }
+}
diff --git a/rust/lance-io/src/object_writer.rs b/rust/lance-io/src/object_writer.rs
index f6c9d8943e6..878687be91e 100644
--- a/rust/lance-io/src/object_writer.rs
+++ b/rust/lance-io/src/object_writer.rs
@@ -47,22 +47,35 @@ fn max_conn_reset_retries() -> u16 {
     })
 }
 
+/// Maximum part size in GCS and S3: 5GB.
+const MAX_UPLOAD_PART_SIZE: usize = 1024 * 1024 * 1024 * 5;
+
+/// Clamps a requested upload part size to the valid [5MB, 5GB] range.
+/// Returns the clamped value and whether clamping was necessary.
+fn clamp_initial_upload_size(raw: usize) -> (usize, bool) {
+    let clamped = raw.clamp(INITIAL_UPLOAD_STEP, MAX_UPLOAD_PART_SIZE);
+    (clamped, clamped != raw)
+}
+
 fn initial_upload_size() -> usize {
     static LANCE_INITIAL_UPLOAD_SIZE: OnceLock<usize> = OnceLock::new();
     *LANCE_INITIAL_UPLOAD_SIZE.get_or_init(|| {
-        std::env::var("LANCE_INITIAL_UPLOAD_SIZE")
+        let Some(raw) = std::env::var("LANCE_INITIAL_UPLOAD_SIZE")
             .ok()
             .and_then(|s| s.parse::<usize>().ok())
-            .inspect(|size| {
-                if *size < INITIAL_UPLOAD_STEP {
-                    // Minimum part size in GCS and S3
-                    panic!("LANCE_INITIAL_UPLOAD_SIZE must be at least 5MB");
-                } else if *size > 1024 * 1024 * 1024 * 5 {
-                    // Maximum part size in GCS and S3
-                    panic!("LANCE_INITIAL_UPLOAD_SIZE must be at most 5GB");
-                }
-            })
-            .unwrap_or(INITIAL_UPLOAD_STEP)
+        else {
+            return INITIAL_UPLOAD_STEP;
+        };
+        let (clamped, was_clamped) = clamp_initial_upload_size(raw);
+        if was_clamped {
+            // OnceLock caches the result, so this warning fires at most once per process.
+            tracing::warn!(
+                requested = raw,
+                clamped,
+                "LANCE_INITIAL_UPLOAD_SIZE must be between 5MB and 5GB; clamping to valid range"
+            );
+        }
+        clamped
     })
 }
 
@@ -500,11 +513,27 @@ impl Writer for ObjectWriter {
 }
 
 pub struct LocalWriter {
-    inner: tokio::io::BufWriter<tokio::fs::File>,
-    cursor: usize,
     path: Path,
+    state: LocalWriteState,
+}
+
+#[derive(Default)]
+enum LocalWriteState {
+    Writing(WritingState),
+    Finishing {
+        size: usize,
+        future: BoxFuture<'static, Result<WriteResult>>,
+    },
+    Done(WriteResult),
+    #[default]
+    Poisoned,
+}
+
+struct WritingState {
+    writer: tokio::io::BufWriter<tokio::fs::File>,
+    cursor: usize,
     /// Temp path that auto-deletes on drop. Set to `None` after `persist()`.
-    temp_path: Option<tempfile::TempPath>,
+    temp_path: tempfile::TempPath,
     io_tracker: Arc<IOTracker>,
 }
 
@@ -516,13 +545,57 @@ impl LocalWriter {
         io_tracker: Arc<IOTracker>,
     ) -> Self {
         Self {
-            inner: tokio::io::BufWriter::new(file),
-            cursor: 0,
             path,
-            temp_path: Some(temp_path),
-            io_tracker,
+            state: LocalWriteState::Writing(WritingState {
+                writer: tokio::io::BufWriter::new(file),
+                cursor: 0,
+                temp_path,
+                io_tracker,
+            }),
         }
     }
+
+    fn already_closed_err(path: &Path) -> io::Error {
+        io::Error::other(format!(
+            "cannot write to LocalWriter for {} after shutdown",
+            path
+        ))
+    }
+
+    fn poisoned_err(path: &Path) -> io::Error {
+        io::Error::other(format!("LocalWriter for {} is in poisoned state", path))
+    }
+
+    async fn persist(
+        temp_path: tempfile::TempPath,
+        final_path: Path,
+        size: usize,
+        io_tracker: Arc<IOTracker>,
+    ) -> Result<WriteResult> {
+        let local_path = crate::local::to_local_path(&final_path);
+        let e_tag = tokio::task::spawn_blocking(move || -> Result<String> {
+            temp_path.persist(&local_path).map_err(|e| {
+                Error::io(format!(
+                    "failed to persist temp file to {}: {}",
+                    local_path, e.error
+                ))
+            })?;
+
+            let metadata = std::fs::metadata(&local_path).map_err(|e| {
+                Error::io(format!("failed to read metadata for {}: {}", local_path, e))
+            })?;
+            Ok(get_etag(&metadata))
+        })
+        .await
+        .map_err(|e| Error::io(format!("spawn_blocking failed: {}", e)))??;
+
+        io_tracker.record_write("put", final_path, size as u64);
+
+        Ok(WriteResult {
+            size,
+            e_tag: Some(e_tag),
+        })
+    }
 }
 
 impl AsyncWrite for LocalWriter {
@@ -531,32 +604,82 @@ impl AsyncWrite for LocalWriter {
         cx: &mut std::task::Context<'_>,
         buf: &[u8],
     ) -> Poll<std::result::Result<usize, std::io::Error>> {
-        let poll = Pin::new(&mut self.inner).poll_write(cx, buf);
-        if let Poll::Ready(Ok(n)) = &poll {
-            self.cursor += *n;
+        if let LocalWriteState::Writing(state) = &mut self.state {
+            let poll = Pin::new(&mut state.writer).poll_write(cx, buf);
+            if let Poll::Ready(Ok(n)) = &poll {
+                state.cursor += *n;
+            }
+            poll
+        } else {
+            Poll::Ready(Err(Self::already_closed_err(&self.path)))
         }
-        poll
     }
 
     fn poll_flush(
         mut self: Pin<&mut Self>,
         cx: &mut std::task::Context<'_>,
     ) -> Poll<std::result::Result<(), std::io::Error>> {
-        Pin::new(&mut self.inner).poll_flush(cx)
+        if let LocalWriteState::Writing(state) = &mut self.state {
+            Pin::new(&mut state.writer).poll_flush(cx)
+        } else {
+            Poll::Ready(Err(Self::already_closed_err(&self.path)))
+        }
     }
 
     fn poll_shutdown(
         mut self: Pin<&mut Self>,
         cx: &mut std::task::Context<'_>,
     ) -> Poll<std::result::Result<(), std::io::Error>> {
-        Pin::new(&mut self.inner).poll_shutdown(cx)
+        let mut_self = &mut *self;
+        loop {
+            match &mut mut_self.state {
+                LocalWriteState::Writing(state) => {
+                    if Pin::new(&mut state.writer).poll_shutdown(cx).is_pending() {
+                        return Poll::Pending;
+                    }
+
+                    // Write is complete, we can transition to persisting.
+                    let LocalWriteState::Writing(state) =
+                        std::mem::replace(&mut mut_self.state, LocalWriteState::Poisoned)
+                    else {
+                        unreachable!()
+                    };
+                    let size = state.cursor;
+                    mut_self.state = LocalWriteState::Finishing {
+                        size,
+                        future: Box::pin(Self::persist(
+                            state.temp_path,
+                            mut_self.path.clone(),
+                            size,
+                            state.io_tracker,
+                        )),
+                    };
+                }
+                LocalWriteState::Finishing { future, .. } => match future.poll_unpin(cx) {
+                    Poll::Ready(Ok(result)) => mut_self.state = LocalWriteState::Done(result),
+                    Poll::Ready(Err(e)) => {
+                        return Poll::Ready(Err(io::Error::other(e)));
+                    }
+                    Poll::Pending => return Poll::Pending,
+                },
+                LocalWriteState::Done(_) => return Poll::Ready(Ok(())),
+                LocalWriteState::Poisoned => {
+                    return Poll::Ready(Err(Self::poisoned_err(&self.path)));
+                }
+            }
+        }
     }
 }
 
 #[async_trait]
 impl Writer for LocalWriter {
     async fn tell(&mut self) -> Result<usize> {
-        Ok(self.cursor)
+        match &mut self.state {
+            LocalWriteState::Writing(state) => Ok(state.cursor),
+            LocalWriteState::Finishing { size, .. } => Ok(*size),
+            LocalWriteState::Done(result) => Ok(result.size),
+            LocalWriteState::Poisoned => Err(Self::poisoned_err(&self.path).into()),
+        }
     }
 
     async fn shutdown(&mut self) -> Result<WriteResult> {
@@ -567,34 +690,10 @@ impl Writer for LocalWriter {
             ))
         })?;
 
-        let final_path = crate::local::to_local_path(&self.path);
-        let temp_path = self.temp_path.take().ok_or_else(|| {
-            Error::io(format!("local writer for {} already shut down", self.path))
-        })?;
-        let path_clone = self.path.clone();
-        let e_tag = tokio::task::spawn_blocking(move || -> Result<String> {
-            temp_path.persist(&final_path).map_err(|e| {
-                Error::io(format!(
-                    "failed to persist temp file to {}: {}",
-                    final_path, e.error
-                ))
-            })?;
-
-            let metadata = std::fs::metadata(&final_path).map_err(|e| {
-                Error::io(format!("failed to read metadata for {}: {}", path_clone, e))
-            })?;
-            Ok(get_etag(&metadata))
-        })
-        .await
-        .map_err(|e| Error::io(format!("spawn_blocking failed: {}", e)))??;
-
-        self.io_tracker
-            .record_write("put", self.path.clone(), self.cursor as u64);
-
-        Ok(WriteResult {
-            size: self.cursor,
-            e_tag: Some(e_tag),
-        })
+        match &self.state {
+            LocalWriteState::Done(result) => Ok(result.clone()),
+            _ => unreachable!(),
+        }
     }
 }
 
@@ -734,4 +833,39 @@ mod tests {
         assert!(!temp_file_path.exists());
         assert!(!file_path.exists());
     }
+
+    #[test]
+    fn clamp_initial_upload_size_below_min_is_clamped_up() {
+        assert_eq!(clamp_initial_upload_size(0), (INITIAL_UPLOAD_STEP, true));
+        assert_eq!(
+            clamp_initial_upload_size(INITIAL_UPLOAD_STEP - 1),
+            (INITIAL_UPLOAD_STEP, true)
+        );
+    }
+
+    #[test]
+    fn clamp_initial_upload_size_within_range_is_unchanged() {
+        assert_eq!(
+            clamp_initial_upload_size(INITIAL_UPLOAD_STEP),
+            (INITIAL_UPLOAD_STEP, false)
+        );
+        assert_eq!(
+            clamp_initial_upload_size(MAX_UPLOAD_PART_SIZE),
+            (MAX_UPLOAD_PART_SIZE, false)
+        );
+        let mid = INITIAL_UPLOAD_STEP * 8; // 40MB, in range
+        assert_eq!(clamp_initial_upload_size(mid), (mid, false));
+    }
+
+    #[test]
+    fn clamp_initial_upload_size_above_max_is_clamped_down() {
+        assert_eq!(
+            clamp_initial_upload_size(MAX_UPLOAD_PART_SIZE + 1),
+            (MAX_UPLOAD_PART_SIZE, true)
+        );
+        assert_eq!(
+            clamp_initial_upload_size(usize::MAX),
+            (MAX_UPLOAD_PART_SIZE, true)
+        );
+    }
 }
diff --git a/rust/lance-io/src/scheduler.rs b/rust/lance-io/src/scheduler.rs
index e27e9519425..d5c4f44f7fa 100644
--- a/rust/lance-io/src/scheduler.rs
+++ b/rust/lance-io/src/scheduler.rs
@@ -13,8 +13,9 @@ use std::ops::Range;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
 use std::time::Instant;
-use tokio::sync::{Notify, Semaphore, SemaphorePermit};
+use tokio::sync::Notify;
 
+use lance_core::utils::parse::str_is_truthy;
 use lance_core::{Error, Result};
 
 use crate::object_store::ObjectStore;
@@ -32,19 +33,6 @@ const BACKPRESSURE_DEBOUNCE: u64 = 60;
 static IOPS_COUNTER: AtomicU64 = AtomicU64::new(0);
 // Global counter of how many bytes were read by the scheduler
 static BYTES_READ_COUNTER: AtomicU64 = AtomicU64::new(0);
-// By default, we limit the number of IOPS across the entire process to 128
-//
-// In theory this is enough for ~10GBps on S3 following the guidelines to issue
-// 1 IOP per 80MBps.  In practice, I have noticed slightly better performance going
-// up to 256.
-//
-// However, non-S3 stores (e.g. GCS, Azure) can suffer significantly from too many
-// concurrent IOPS.  For safety, we set the default to 128 and let the user override
-// this if needed.
-//
-// Note: this only limits things that run through the scheduler.  It does not limit
-// IOPS from other sources like writing or commits.
-static DEFAULT_PROCESS_IOPS_LIMIT: i32 = 128;
 
 pub fn iops_counter() -> u64 {
     IOPS_COUNTER.load(Ordering::Acquire)
@@ -54,97 +42,6 @@ pub fn bytes_read_counter() -> u64 {
     BYTES_READ_COUNTER.load(Ordering::Acquire)
 }
 
-// There are two structures that control the I/O scheduler concurrency.  First,
-// we have a hard limit on the number of IOPS that can be issued concurrently.
-// This limit is process-wide.
-//
-// Second, we try and limit how many I/O requests can be buffered in memory without
-// being consumed by a decoder of some kind.  This limit is per-scheduler.  We cannot
-// make this limit process wide without introducing deadlock (because the decoder for
-// file 0 might be waiting on IOPS blocked by a queue filled with requests for file 1)
-// and vice-versa.
-//
-// There is also a per-scan limit on the number of IOPS that can be issued concurrently.
-//
-// The process-wide limit exists when users need a hard limit on the number of parallel
-// IOPS, e.g. due to port availability limits or to prevent multiple scans from saturating
-// the network.  (Note: a process-wide limit of X will not necessarily limit the number of
-// open TCP connections to exactly X.  The underlying object store may open more connections
-// anyways)
-//
-// However, it can be too tough in some cases, e.g. when some scans are reading from
-// cloud storage and other scans are reading from local disk.  In these cases users don't
-// need to set a process-limit and can rely on the per-scan limits.
-
-// The IopsQuota enforces the first of the above limits, it is the per-process hard cap
-// on the number of IOPS that can be issued concurrently.
-//
-// The per-scan limits are enforced by IoQueue
-struct IopsQuota {
-    // An Option is used here to avoid mutex overhead if no limit is set
-    iops_avail: Option<Semaphore>,
-}
-
-/// A reservation on the global IOPS quota
-///
-/// When the reservation is dropped, the IOPS quota is released unless
-/// [`Self::forget`] is called.
-struct IopsReservation<'a> {
-    value: Option<SemaphorePermit<'a>>,
-}
-
-impl IopsReservation<'_> {
-    // Forget the reservation, so it won't be released on drop
-    fn forget(&mut self) {
-        if let Some(value) = self.value.take() {
-            value.forget();
-        }
-    }
-}
-
-impl IopsQuota {
-    // By default, we throttle the number of scan IOPS across the entire process
-    //
-    // However, the user can disable this by setting the environment variable
-    // LANCE_PROCESS_IO_THREADS_LIMIT to zero (or a negative integer).
-    fn new() -> Self {
-        let initial_capacity = std::env::var("LANCE_PROCESS_IO_THREADS_LIMIT")
-            .map(|s| {
-                s.parse::<i32>().unwrap_or_else(|_| {
-                    log::warn!("Ignoring invalid LANCE_PROCESS_IO_THREADS_LIMIT: {}", s);
-                    DEFAULT_PROCESS_IOPS_LIMIT
-                })
-            })
-            .unwrap_or(DEFAULT_PROCESS_IOPS_LIMIT);
-        let iops_avail = if initial_capacity <= 0 {
-            None
-        } else {
-            Some(Semaphore::new(initial_capacity as usize))
-        };
-        Self { iops_avail }
-    }
-
-    // Return a reservation on the global IOPS quota
-    fn release(&self) {
-        if let Some(iops_avail) = self.iops_avail.as_ref() {
-            iops_avail.add_permits(1);
-        }
-    }
-
-    // Acquire a reservation on the global IOPS quota
-    async fn acquire(&self) -> IopsReservation<'_> {
-        if let Some(iops_avail) = self.iops_avail.as_ref() {
-            IopsReservation {
-                value: Some(iops_avail.acquire().await.unwrap()),
-            }
-        } else {
-            IopsReservation { value: None }
-        }
-    }
-}
-
-static IOPS_QUOTA: std::sync::LazyLock<IopsQuota> = std::sync::LazyLock::new(IopsQuota::new);
-
 // We want to allow requests that have a lower priority than any
 // currently in-flight request.  This helps avoid potential deadlocks
 // related to backpressure.  Unfortunately, it is quite expensive to
@@ -303,17 +200,8 @@ impl IoQueue {
     async fn pop(&self) -> Option<IoTask> {
         loop {
             {
-                // First, grab a reservation on the global IOPS quota
-                // If we then get a task to run, transfer the reservation
-                // to the task.  Otherwise, the reservation will be released
-                // when iop_res is dropped.
-                let mut iop_res = IOPS_QUOTA.acquire().await;
-                // Next, try and grab a reservation from the queue
                 let mut state = self.state.lock().unwrap();
                 if let Some(task) = state.next_task() {
-                    // Reservation successfully acquired, we will release the global
-                    // global reservation after task has run.
-                    iop_res.forget();
                     return Some(task);
                 }
 
@@ -501,7 +389,6 @@ impl IoTask {
             range_end = self.to_read.end,
             "File I/O completed"
         );
-        IOPS_QUOTA.release();
         (self.when_done)(bytes);
     }
 }
@@ -619,15 +506,21 @@ pub struct SchedulerConfig {
     /// This controls back pressure.  If data is not processed quickly enough then this
     /// buffer will fill up and the I/O loop will pause until the buffer is drained.
     pub io_buffer_size_bytes: u64,
-    /// Whether to use the new lite scheduler
-    pub use_lite_scheduler: bool,
+    /// Whether to use the lite scheduler.
+    ///
+    /// - `Some(true)` forces the lite scheduler (e.g. from env var or programmatic).
+    /// - `Some(false)` forces the standard scheduler.
+    /// - `None` defers to the object store's preference (see [`ObjectStore::prefers_lite_scheduler`]).
+    pub use_lite_scheduler: Option<bool>,
 }
 
 impl SchedulerConfig {
     pub fn new(io_buffer_size_bytes: u64) -> Self {
         Self {
             io_buffer_size_bytes,
-            use_lite_scheduler: std::env::var("LANCE_USE_LITE_SCHEDULER").is_ok(),
+            use_lite_scheduler: std::env::var("LANCE_USE_LITE_SCHEDULER")
+                .ok()
+                .map(|v| str_is_truthy(v.trim())),
         }
     }
 
@@ -635,7 +528,7 @@ impl SchedulerConfig {
     pub fn default_for_testing() -> Self {
         Self {
             io_buffer_size_bytes: 256 * 1024 * 1024,
-            use_lite_scheduler: false,
+            use_lite_scheduler: None,
         }
     }
 
@@ -647,7 +540,7 @@ impl SchedulerConfig {
 
     pub fn with_lite_scheduler(self) -> Self {
         Self {
-            use_lite_scheduler: true,
+            use_lite_scheduler: Some(true),
             ..self
         }
     }
@@ -662,7 +555,10 @@ impl ScanScheduler {
     /// * config - configuration settings for the scheduler
     pub fn new(object_store: Arc<ObjectStore>, config: SchedulerConfig) -> Arc<Self> {
         let io_capacity = object_store.io_parallelism();
-        let io_queue = if config.use_lite_scheduler {
+        let use_lite = config
+            .use_lite_scheduler
+            .unwrap_or_else(|| object_store.prefers_lite_scheduler());
+        let io_queue = if use_lite {
             let io_queue = Arc::new(lite::IoQueue::new(
                 io_capacity as u64,
                 config.io_buffer_size_bytes,
@@ -857,6 +753,11 @@ impl ScanScheduler {
     pub fn stats(&self) -> ScanStats {
         ScanStats::new(self.stats.as_ref())
     }
+
+    #[cfg(test)]
+    fn uses_lite_scheduler(&self) -> bool {
+        matches!(self.io_queue, IoQueueType::Lite(_))
+    }
 }
 
 impl Drop for ScanScheduler {
@@ -1234,7 +1135,7 @@ mod tests {
 
         let config = SchedulerConfig {
             io_buffer_size_bytes: 1024 * 1024,
-            use_lite_scheduler: false,
+            use_lite_scheduler: None,
         };
 
         let scan_scheduler = ScanScheduler::new(obj_store, config);
@@ -1325,7 +1226,7 @@ mod tests {
 
         let config = SchedulerConfig {
             io_buffer_size_bytes: 10,
-            use_lite_scheduler: false,
+            use_lite_scheduler: None,
         };
 
         let scan_scheduler = ScanScheduler::new(obj_store.clone(), config);
@@ -1400,7 +1301,7 @@ mod tests {
         // Ensure deadlock prevention timeout can be disabled
         let config = SchedulerConfig {
             io_buffer_size_bytes: 10,
-            use_lite_scheduler: false,
+            use_lite_scheduler: None,
         };
 
         let scan_scheduler = ScanScheduler::new(obj_store, config);
@@ -1488,6 +1389,55 @@ mod tests {
         assert_eq!(fut3.await.unwrap()[0].len(), 100);
     }
 
+    #[tokio::test]
+    async fn test_object_store_selects_scheduler() {
+        // A memory:// store should use the standard scheduler when config is None
+        let memory_store = Arc::new(ObjectStore::memory());
+        assert!(!memory_store.prefers_lite_scheduler());
+        let config = SchedulerConfig {
+            io_buffer_size_bytes: 256 * 1024 * 1024,
+            use_lite_scheduler: None,
+        };
+        let scheduler = ScanScheduler::new(memory_store.clone(), config);
+        assert!(!scheduler.uses_lite_scheduler());
+
+        // A file+uring:// store should use the lite scheduler when config is None
+        let uring_store = Arc::new(ObjectStore::new(
+            Arc::new(InMemory::new()),
+            Url::parse("file+uring:///tmp").unwrap(),
+            None,
+            None,
+            false,
+            false,
+            8,
+            DEFAULT_DOWNLOAD_RETRY_COUNT,
+            None,
+        ));
+        assert!(uring_store.prefers_lite_scheduler());
+        let config = SchedulerConfig {
+            io_buffer_size_bytes: 256 * 1024 * 1024,
+            use_lite_scheduler: None,
+        };
+        let scheduler = ScanScheduler::new(uring_store.clone(), config);
+        assert!(scheduler.uses_lite_scheduler());
+
+        // Explicit Some(false) overrides a file+uring:// store's preference
+        let config = SchedulerConfig {
+            io_buffer_size_bytes: 256 * 1024 * 1024,
+            use_lite_scheduler: Some(false),
+        };
+        let scheduler = ScanScheduler::new(uring_store, config);
+        assert!(!scheduler.uses_lite_scheduler());
+
+        // Explicit Some(true) overrides a memory:// store's preference
+        let config = SchedulerConfig {
+            io_buffer_size_bytes: 256 * 1024 * 1024,
+            use_lite_scheduler: Some(true),
+        };
+        let scheduler = ScanScheduler::new(memory_store, config);
+        assert!(scheduler.uses_lite_scheduler());
+    }
+
     #[test_log::test(tokio::test(flavor = "multi_thread"))]
     async fn stress_backpressure() {
         // This test ensures that the backpressure mechanism works correctly with
@@ -1503,7 +1453,7 @@ mod tests {
         // Only one request will be allowed in
         let config = SchedulerConfig {
             io_buffer_size_bytes: 1,
-            use_lite_scheduler: false,
+            use_lite_scheduler: None,
         };
         let scan_scheduler = ScanScheduler::new(obj_store.clone(), config);
         let file_scheduler = scan_scheduler
diff --git a/rust/lance-io/src/traits.rs b/rust/lance-io/src/traits.rs
index 9ad8d86c00c..eb83cf1a2ba 100644
--- a/rust/lance-io/src/traits.rs
+++ b/rust/lance-io/src/traits.rs
@@ -6,7 +6,7 @@ use std::ops::Range;
 use async_trait::async_trait;
 use bytes::Bytes;
 use deepsize::DeepSizeOf;
-use futures::future::BoxFuture;
+use futures::{StreamExt, future::BoxFuture, stream::BoxStream};
 use object_store::path::Path;
 use prost::Message;
 use tokio::io::{AsyncWrite, AsyncWriteExt};
@@ -19,6 +19,8 @@ pub trait ProtoStruct {
     type Proto: Message;
 }
 
+pub type ByteStream = BoxStream<'static, object_store::Result<Bytes>>;
+
 /// A trait for writing to a file on local file system or object store.
 #[async_trait]
 pub trait Writer: AsyncWrite + Unpin + Send {
@@ -67,6 +69,14 @@ pub trait WriteExt {
         minor_version: i16,
         magic: &[u8],
     ) -> Result<()>;
+
+    async fn copy_from_reader(&mut self, reader: &dyn Reader) -> Result<usize>;
+
+    async fn copy_range_from_reader(
+        &mut self,
+        reader: &dyn Reader,
+        range: Range<usize>,
+    ) -> Result<usize>;
 }
 
 #[async_trait]
@@ -95,6 +105,32 @@ impl<W: Writer + ?Sized> WriteExt for W {
         self.write_all(magic).await?;
         Ok(())
     }
+
+    async fn copy_from_reader(&mut self, reader: &dyn Reader) -> Result<usize> {
+        let mut stream = reader.get_stream().await?;
+        let mut copied = 0usize;
+        while let Some(chunk) = stream.next().await {
+            let bytes = chunk?;
+            copied += bytes.len();
+            self.write_all(&bytes).await?;
+        }
+        Ok(copied)
+    }
+
+    async fn copy_range_from_reader(
+        &mut self,
+        reader: &dyn Reader,
+        range: Range<usize>,
+    ) -> Result<usize> {
+        let mut stream = reader.get_range_stream(range).await?;
+        let mut copied = 0usize;
+        while let Some(chunk) = stream.next().await {
+            let bytes = chunk?;
+            copied += bytes.len();
+            self.write_all(&bytes).await?;
+        }
+        Ok(copied)
+    }
 }
 
 pub trait Reader: std::fmt::Debug + Send + Sync + DeepSizeOf {
@@ -119,4 +155,23 @@ pub trait Reader: std::fmt::Debug + Send + Sync + DeepSizeOf {
     /// By default this reads the size in a separate IOP but some implementations
     /// may not need the size beforehand.
     fn get_all(&self) -> BoxFuture<'_, object_store::Result<Bytes>>;
+
+    /// Read the entire object as a byte stream.
+    fn get_stream(&self) -> BoxFuture<'_, object_store::Result<ByteStream>> {
+        Box::pin(async move {
+            let bytes = self.get_all().await?;
+            Ok(futures::stream::once(async move { Ok(bytes) }).boxed())
+        })
+    }
+
+    /// Read a byte range as a byte stream.
+    fn get_range_stream(
+        &self,
+        range: Range<usize>,
+    ) -> BoxFuture<'_, object_store::Result<ByteStream>> {
+        Box::pin(async move {
+            let bytes = self.get_range(range).await?;
+            Ok(futures::stream::once(async move { Ok(bytes) }).boxed())
+        })
+    }
 }
diff --git a/rust/lance-io/src/uring.rs b/rust/lance-io/src/uring.rs
new file mode 100644
index 00000000000..06786178164
--- /dev/null
+++ b/rust/lance-io/src/uring.rs
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! io_uring-based I/O for disks with high IOPS capacity (e.g. NVMe)
+//!
+//! This module provides two implementations of the [`Reader`](crate::traits::Reader) trait
+//! using Linux's io_uring interface for asynchronous I/O.
+//!
+//! One of these uses a pool of dedicated background threads which each own an io_uring instance.
+//! Read requests are submitted to a background thread's pool.
+//!
+//! The other implementation uses a thread-local io_uring instance.  This only works if the future
+//! is polled by the same thread that submitted the request.  This means that the runtime must be
+//! a single-threaded runtime.
+//!
+//! # Configuration
+//!
+//! The io_uring reader is enabled by using the `file+uring://` URI scheme instead of `file://`.
+//! Additional tuning parameters are controlled by environment variables:
+//!
+//! - `LANCE_URING_CURRENT_THREAD` - Use thread-local io_uring (default: false)
+//! - `LANCE_URING_BLOCK_SIZE` - Block size in bytes (default: 4KB)
+//! - `LANCE_URING_IO_PARALLELISM` - Max concurrent operations (default: 128)
+//! - `LANCE_URING_QUEUE_DEPTH` - io_uring queue depth (default: 16K)
+//! - `LANCE_URING_THREAD_COUNT` - Number of io_uring threads to use (default: 2)
+//! - `LANCE_URING_SUBMIT_BATCH_SIZE` - Number of requests to batch before submitting (default: 128)
+//! - `LANCE_URING_POLL_TIMEOUT_MS` - Thread poll timeout in milliseconds (default: 10)
+//!
+//! Note: the block size and io parallelism are not actually used by the io_uring implementation.  These
+//! variables just control what the filesystem reports up to Lance.
+//!
+//! # Platform Support
+//!
+//! This module is only available on Linux and requires kernel 5.1 or newer.
+//! On other platforms, the code falls back to [`LocalObjectReader`](crate::local::LocalObjectReader).
+//!
+//! # Example
+//!
+//! ```no_run
+//! # use lance_io::object_store::ObjectStore;
+//! # async fn example() -> lance_core::Result<()> {
+//! // Enable io_uring by using the file+uring:// scheme
+//! let uri = "file+uring:///path/to/file.dat";
+//! let (store, path) = ObjectStore::from_uri(uri).await?;
+//! let reader = store.open(&path).await?;
+//!
+//! // Reader will use io_uring
+//! let data = reader.get_range(0..1024).await?;
+//! # Ok(())
+//! # }
+//! ```
+
+mod future;
+mod reader;
+mod requests;
+mod thread;
+
+// Thread-local io_uring implementation for current-thread runtimes
+pub(crate) mod current_thread;
+pub(crate) mod current_thread_future;
+
+#[cfg(test)]
+mod tests;
+
+use std::sync::LazyLock;
+
+pub(crate) use current_thread::UringCurrentThreadReader;
+pub use reader::UringReader;
+
+/// Default block size for io_uring reads (4KB)
+pub const DEFAULT_URING_BLOCK_SIZE: usize = 4 * 1024;
+
+/// Default I/O parallelism for io_uring (128 concurrent operations)
+pub const DEFAULT_URING_IO_PARALLELISM: usize = 128;
+
+/// Default io_uring queue depth (16K entries)
+pub const DEFAULT_URING_QUEUE_DEPTH: usize = 16 * 1024;
+
+/// Cached `LANCE_URING_BLOCK_SIZE` env var, read once at first access.
+pub(crate) static URING_BLOCK_SIZE: LazyLock<Option<usize>> = LazyLock::new(|| {
+    std::env::var("LANCE_URING_BLOCK_SIZE")
+        .ok()
+        .and_then(|s| s.parse().ok())
+});
diff --git a/rust/lance-io/src/uring/current_thread.rs b/rust/lance-io/src/uring/current_thread.rs
new file mode 100644
index 00000000000..bc09af058e6
--- /dev/null
+++ b/rust/lance-io/src/uring/current_thread.rs
@@ -0,0 +1,421 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Thread-local io_uring implementation for current-thread runtimes.
+//!
+//! This implementation creates a thread-local IoUring instance per thread
+//! and directly processes completions during future polling, eliminating
+//! the need for background threads and MPSC channels.
+
+use super::requests::{IoRequest, RequestState};
+use super::{DEFAULT_URING_BLOCK_SIZE, DEFAULT_URING_IO_PARALLELISM, URING_BLOCK_SIZE};
+use crate::local::to_local_path;
+use crate::traits::Reader;
+use crate::uring::DEFAULT_URING_QUEUE_DEPTH;
+use crate::utils::tracking_store::IOTracker;
+use bytes::{Bytes, BytesMut};
+use deepsize::DeepSizeOf;
+use futures::future::BoxFuture;
+use futures::{FutureExt, TryFutureExt};
+use io_uring::{IoUring, opcode, types};
+use lance_core::{Error, Result};
+use object_store::path::Path;
+
+use std::cell::{LazyCell, RefCell};
+use std::collections::HashMap;
+use std::fs::File;
+use std::future::Future;
+use std::io::{self, ErrorKind};
+use std::ops::Range;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+use tracing::instrument;
+
+// Re-use file handle types from reader.rs
+use super::reader::{CacheKey, CachedReaderData, HANDLE_CACHE, UringFileHandle};
+
+/// Global counter for generating unique user_data values
+static USER_DATA_COUNTER: AtomicU64 = AtomicU64::new(1);
+
+/// Thread-local io_uring instance with pending requests
+struct ThreadLocalUring {
+    ring: IoUring,
+    pending: HashMap<u64, Arc<IoRequest>>,
+}
+
+thread_local! {
+    static URING: LazyCell<RefCell<ThreadLocalUring>> = LazyCell::new(|| {
+        let queue_depth = std::env::var("LANCE_URING_QUEUE_DEPTH")
+            .ok()
+            .and_then(|s| s.parse().ok())
+            .unwrap_or(DEFAULT_URING_QUEUE_DEPTH);
+
+        let ring = IoUring::builder()
+            // Ensures work is only done in submit_and_wait
+            .setup_defer_taskrun()
+            // Enable perf. optimization when there is only one issuer thread
+            .setup_single_issuer()
+            .build(queue_depth as u32)
+            .expect("Failed to create io_uring");
+
+        log::debug!(
+            "Created thread-local io_uring with queue depth {}",
+            queue_depth
+        );
+
+        RefCell::new(ThreadLocalUring {
+            ring,
+            pending: HashMap::new(),
+        })
+    });
+}
+
+/// Push request to thread-local submission queue
+pub(super) fn push_request(request: Arc<IoRequest>) -> io::Result<()> {
+    URING.with(|cell| {
+        let mut uring = cell.borrow_mut();
+
+        // Generate unique user_data
+        let user_data = USER_DATA_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+        // Get buffer pointer, adjusting for any bytes already read (short read retry)
+        let (buffer_ptr, read_offset, read_length) = {
+            let state = request.state.lock().unwrap();
+            let br = state.bytes_read;
+            (
+                unsafe { state.buffer.as_ptr().add(br) as *mut u8 },
+                request.offset + br as u64,
+                (request.length - br) as u32,
+            )
+        };
+
+        // Prepare read operation
+        let read_op =
+            opcode::Read::new(types::Fd(request.fd), buffer_ptr, read_length).offset(read_offset);
+
+        // Get submission queue
+        let mut sq = uring.ring.submission();
+
+        // Check if SQ has space
+        if sq.is_full() {
+            drop(sq);
+            return Err(io::Error::new(
+                io::ErrorKind::WouldBlock,
+                "io_uring submission queue full",
+            ));
+        }
+
+        // Push to SQ
+        unsafe {
+            sq.push(&read_op.build().user_data(user_data))
+                .map_err(|_| io::Error::other("Failed to push to SQ"))?;
+        }
+        drop(sq);
+
+        // Track request in pending map
+        uring.pending.insert(user_data, request);
+
+        // Don't submit here - let the future handle submission
+
+        Ok(())
+    })
+}
+
+/// Process completions from thread-local IoUring
+pub(super) fn process_thread_local_completions() -> io::Result<usize> {
+    URING.with(|cell| {
+        let mut uring = cell.borrow_mut();
+        let mut completed = 0;
+        let mut retries: Vec<Arc<IoRequest>> = Vec::new();
+
+        // Collect completions first to avoid borrowing ring and pending simultaneously
+        let cqes: Vec<_> = uring
+            .ring
+            .completion()
+            .map(|cqe| (cqe.user_data(), cqe.result()))
+            .collect();
+
+        for (user_data, result) in cqes {
+            if let Some(request) = uring.pending.remove(&user_data) {
+                let mut state = request.state.lock().unwrap();
+
+                if result < 0 {
+                    // Kernel error
+                    state.err = Some(io::Error::from_raw_os_error(-result));
+                    state.completed = true;
+                } else if result == 0 {
+                    // EOF before full read completed
+                    let br = state.bytes_read;
+                    state.err = Some(io::Error::new(
+                        io::ErrorKind::UnexpectedEof,
+                        format!("unexpected EOF: read {} of {} bytes", br, request.length),
+                    ));
+                    state.buffer.truncate(br);
+                    state.completed = true;
+                } else {
+                    // Positive result: n bytes read
+                    let n = result as usize;
+                    state.bytes_read += n;
+                    let br = state.bytes_read;
+
+                    if br >= request.length {
+                        // Full read complete
+                        state.buffer.truncate(br);
+                        state.completed = true;
+                    } else {
+                        // Short read — need retry; don't mark completed or wake
+                        drop(state);
+                        retries.push(request);
+
+                        continue;
+                    }
+                }
+
+                // Wake waiting future
+                if let Some(waker) = state.waker.take() {
+                    drop(state);
+                    waker.wake();
+                }
+
+                completed += 1;
+            } else {
+                log::warn!("Received completion for unknown user_data: {}", user_data);
+            }
+        }
+
+        // Resubmit short-read retries
+        for request in retries {
+            // Generate unique user_data
+            let user_data = USER_DATA_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+            let (buffer_ptr, read_offset, read_length) = {
+                let state = request.state.lock().unwrap();
+                let br = state.bytes_read;
+                (
+                    unsafe { state.buffer.as_ptr().add(br) as *mut u8 },
+                    request.offset + br as u64,
+                    (request.length - br) as u32,
+                )
+            };
+
+            let read_op = opcode::Read::new(types::Fd(request.fd), buffer_ptr, read_length)
+                .offset(read_offset);
+
+            let mut sq = uring.ring.submission();
+            if sq.is_full() {
+                drop(sq);
+                request.fail(io::Error::new(
+                    io::ErrorKind::WouldBlock,
+                    "io_uring submission queue full during retry",
+                ));
+                continue;
+            }
+
+            unsafe {
+                if sq.push(&read_op.build().user_data(user_data)).is_err() {
+                    request.fail(io::Error::other("Failed to push short-read retry to SQ"));
+                    continue;
+                }
+            }
+            drop(sq);
+
+            uring.pending.insert(user_data, request);
+        }
+
+        if completed > 0 {
+            log::trace!("Processed {} completions", completed);
+        }
+
+        Ok(completed)
+    })
+}
+
+/// Submit all pending requests and wait with timeout 0 (non-blocking)
+pub(super) fn submit_and_wait_thread_local() -> io::Result<()> {
+    URING.with(|cell| {
+        let uring = cell.borrow_mut();
+        // Submit with wait=1 (do at least some work)
+        uring.ring.submit_and_wait(1)?;
+        Ok(())
+    })
+}
+
+/// Thread-local io_uring-based reader for current-thread runtimes
+#[derive(Debug)]
+pub struct UringCurrentThreadReader {
+    /// File handle
+    handle: Arc<UringFileHandle>,
+
+    /// Block size for I/O operations
+    block_size: usize,
+
+    /// File size (determined at open time)
+    size: usize,
+
+    /// I/O tracker for monitoring operations
+    io_tracker: Arc<IOTracker>,
+}
+
+impl DeepSizeOf for UringCurrentThreadReader {
+    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+        // Skip file handle (just a system resource)
+        // Only count the path's deep size
+        self.handle.path.as_ref().deep_size_of_children(context)
+    }
+}
+
+impl UringCurrentThreadReader {
+    /// Open a file with thread-local io_uring
+    ///
+    /// This reuses the file handle caching infrastructure from UringReader
+    #[instrument(level = "debug")]
+    pub(crate) async fn open(
+        path: &Path,
+        block_size: usize,
+        known_size: Option<usize>,
+        io_tracker: Arc<IOTracker>,
+    ) -> Result<Box<dyn Reader>> {
+        // Determine block size with environment variable override
+        let block_size = URING_BLOCK_SIZE.unwrap_or(block_size.max(DEFAULT_URING_BLOCK_SIZE));
+
+        let cache_key = CacheKey::new(path, block_size);
+
+        // Try to get from cache first
+        if let Some(data) = HANDLE_CACHE.get(&cache_key).await {
+            // Use known_size if provided, otherwise use cached size
+            let size = known_size.unwrap_or(data.size);
+            return Ok(Box::new(Self {
+                handle: data.handle,
+                block_size,
+                size,
+                io_tracker,
+            }) as Box<dyn Reader>);
+        }
+
+        // Cache miss - open file and get size
+        let path_clone = path.clone();
+        let local_path = to_local_path(path);
+
+        let data = tokio::task::spawn_blocking(move || {
+            let file = File::open(&local_path).map_err(|e| match e.kind() {
+                ErrorKind::NotFound => Error::not_found(path_clone.to_string()),
+                _ => e.into(),
+            })?;
+
+            // Get size from known_size or file metadata
+            let size = match known_size {
+                Some(s) => s,
+                None => file.metadata()?.len() as usize,
+            };
+
+            Ok::<_, Error>(CachedReaderData {
+                handle: Arc::new(UringFileHandle::new(file, path_clone)),
+                size,
+            })
+        })
+        .await??;
+
+        // Insert into cache
+        HANDLE_CACHE.insert(cache_key, data.clone()).await;
+
+        // Return new reader instance
+        Ok(Box::new(Self {
+            handle: data.handle.clone(),
+            block_size,
+            size: data.size,
+            io_tracker,
+        }) as Box<dyn Reader>)
+    }
+
+    /// Submit a read request and return a future
+    fn submit_read(
+        &self,
+        offset: u64,
+        length: usize,
+    ) -> Pin<Box<dyn Future<Output = object_store::Result<Bytes>> + Send>> {
+        let mut buffer = BytesMut::with_capacity(length);
+        unsafe {
+            buffer.set_len(length);
+        }
+
+        let request = Arc::new(IoRequest {
+            fd: self.handle.fd,
+            offset,
+            length,
+            thread_id: std::thread::current().id(),
+            state: Mutex::new(RequestState {
+                completed: false,
+                waker: None,
+                err: None,
+                buffer,
+                bytes_read: 0,
+            }),
+        });
+
+        match push_request(request.clone()) {
+            Ok(()) => Box::pin(super::current_thread_future::UringCurrentThreadFuture::new(
+                request,
+            )),
+            Err(e) => Box::pin(async move {
+                Err(object_store::Error::Generic {
+                    store: "io_uring_ct",
+                    source: Box::new(e),
+                })
+            }),
+        }
+    }
+}
+
+impl Reader for UringCurrentThreadReader {
+    fn path(&self) -> &Path {
+        &self.handle.path
+    }
+
+    fn block_size(&self) -> usize {
+        self.block_size
+    }
+
+    fn io_parallelism(&self) -> usize {
+        std::env::var("LANCE_URING_IO_PARALLELISM")
+            .ok()
+            .and_then(|s| s.parse().ok())
+            .unwrap_or(DEFAULT_URING_IO_PARALLELISM)
+    }
+
+    /// Returns the file size
+    fn size(&self) -> BoxFuture<'_, object_store::Result<usize>> {
+        Box::pin(async move { Ok(self.size) })
+    }
+
+    /// Read a range of bytes using thread-local io_uring
+    #[instrument(level = "debug", skip(self))]
+    fn get_range(&self, range: Range<usize>) -> BoxFuture<'static, object_store::Result<Bytes>> {
+        let io_tracker = self.io_tracker.clone();
+        let path = self.handle.path.clone();
+        let num_bytes = range.len() as u64;
+        let range_u64 = (range.start as u64)..(range.end as u64);
+
+        self.submit_read(range.start as u64, range.len())
+            .map_ok(move |bytes| {
+                io_tracker.record_read("get_range", path, num_bytes, Some(range_u64));
+                bytes
+            })
+            .boxed()
+    }
+
+    /// Read the entire file using thread-local io_uring
+    #[instrument(level = "debug", skip(self))]
+    fn get_all(&self) -> BoxFuture<'static, object_store::Result<Bytes>> {
+        let size = self.size;
+        let io_tracker = self.io_tracker.clone();
+        let path = self.handle.path.clone();
+
+        self.submit_read(0, size)
+            .map_ok(move |bytes| {
+                io_tracker.record_read("get_all", path, bytes.len() as u64, None);
+                bytes
+            })
+            .boxed()
+    }
+}
diff --git a/rust/lance-io/src/uring/current_thread_future.rs b/rust/lance-io/src/uring/current_thread_future.rs
new file mode 100644
index 00000000000..dbcf2242673
--- /dev/null
+++ b/rust/lance-io/src/uring/current_thread_future.rs
@@ -0,0 +1,102 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Future implementation for thread-local io_uring operations.
+//!
+//! This future actively processes completions during polling instead of
+//! relying on background tasks.
+
+use super::current_thread::{process_thread_local_completions, submit_and_wait_thread_local};
+use super::requests::IoRequest;
+use bytes::Bytes;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+/// Future that awaits completion of a thread-local io_uring read operation
+pub struct UringCurrentThreadFuture {
+    request: Arc<IoRequest>,
+}
+
+impl UringCurrentThreadFuture {
+    pub(super) fn new(request: Arc<IoRequest>) -> Self {
+        Self { request }
+    }
+}
+
+impl Future for UringCurrentThreadFuture {
+    type Output = object_store::Result<Bytes>;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        // Check thread safety
+        if self.request.thread_id != std::thread::current().id() {
+            panic!("Request thread ID does not match current thread ID");
+        }
+
+        // First, check if we've been completed by some other future polling for completions.
+        let mut state = self.request.state.lock().unwrap();
+
+        if state.completed {
+            // Take result and return Ready
+            match state.err.take() {
+                Some(err) => {
+                    return Poll::Ready(Err(object_store::Error::Generic {
+                        store: "io_uring_ct",
+                        source: Box::new(err),
+                    }));
+                }
+                None => {
+                    let br = state.bytes_read;
+                    state.buffer.truncate(br);
+                    let bytes = std::mem::take(&mut state.buffer).freeze();
+                    return Poll::Ready(Ok(bytes));
+                }
+            }
+        }
+
+        drop(state);
+
+        // If not, then we should do any available work and then process completions.
+        if let Err(e) = submit_and_wait_thread_local() {
+            log::debug!("Submit and wait error: {:?}", e);
+        }
+
+        if let Err(e) = process_thread_local_completions() {
+            log::warn!("Error processing completions: {:?}", e);
+        }
+
+        // Check if our request completed
+        let mut state = self.request.state.lock().unwrap();
+
+        if state.completed {
+            // Take result and return Ready
+            match state.err.take() {
+                Some(err) => {
+                    return Poll::Ready(Err(object_store::Error::Generic {
+                        store: "io_uring_ct",
+                        source: Box::new(err),
+                    }));
+                }
+                None => {
+                    let br = state.bytes_read;
+                    state.buffer.truncate(br);
+                    let bytes = std::mem::take(&mut state.buffer).freeze();
+                    return Poll::Ready(Ok(bytes));
+                }
+            }
+        }
+
+        // Not done yet - immediately wake and return Pending (don't store waker)
+        // which will force the future to be polled again.  This is intentionally
+        // a busy loop.  io_uring is intended for fast disks where read latency is
+        // so small that the cost of a true context switch (parking and unparking)
+        // would be too high.
+        //
+        // We are effectively doing a "yield" here while we wait for
+        // the io_uring thread to complete the request.
+        drop(state);
+        cx.waker().wake_by_ref();
+        Poll::Pending
+    }
+}
diff --git a/rust/lance-io/src/uring/future.rs b/rust/lance-io/src/uring/future.rs
new file mode 100644
index 00000000000..64b38993683
--- /dev/null
+++ b/rust/lance-io/src/uring/future.rs
@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Future implementation for io_uring read operations.
+
+use super::requests::IoRequest;
+use bytes::Bytes;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+/// Future that awaits completion of an io_uring read operation.
+///
+/// This future is woken by the io_uring thread when the operation completes.
+pub(super) struct UringReadFuture {
+    pub(super) request: Arc<IoRequest>,
+}
+
+impl Future for UringReadFuture {
+    type Output = object_store::Result<Bytes>;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let mut state = self.request.state.lock().unwrap();
+
+        if state.completed {
+            // Operation completed - take the result
+            match state.err.take() {
+                Some(err) => Poll::Ready(Err(object_store::Error::Generic {
+                    store: "io_uring",
+                    source: Box::new(err),
+                })),
+                None => {
+                    let br = state.bytes_read;
+                    state.buffer.truncate(br);
+                    let bytes = std::mem::take(&mut state.buffer).freeze();
+                    Poll::Ready(Ok(bytes))
+                }
+            }
+        } else {
+            // Operation not yet complete - store waker and return Pending
+            state.waker = Some(cx.waker().clone());
+            Poll::Pending
+        }
+    }
+}
diff --git a/rust/lance-io/src/uring/reader.rs b/rust/lance-io/src/uring/reader.rs
new file mode 100644
index 00000000000..0e7b0101ba8
--- /dev/null
+++ b/rust/lance-io/src/uring/reader.rs
@@ -0,0 +1,292 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! UringReader implementation.
+
+use super::future::UringReadFuture;
+use super::requests::IoRequest;
+use super::thread::{SUBMITTED_COUNTER, THREAD_SELECTOR, URING_THREADS};
+use super::{DEFAULT_URING_BLOCK_SIZE, DEFAULT_URING_IO_PARALLELISM, URING_BLOCK_SIZE};
+use crate::local::to_local_path;
+use crate::traits::Reader;
+use crate::uring::requests::RequestState;
+use crate::utils::tracking_store::IOTracker;
+use bytes::{Bytes, BytesMut};
+use deepsize::DeepSizeOf;
+use futures::future::BoxFuture;
+use futures::{FutureExt, TryFutureExt};
+use lance_core::{Error, Result};
+use object_store::path::Path;
+use std::fs::File;
+use std::future::Future;
+use std::io::{self, ErrorKind};
+use std::ops::Range;
+use std::os::unix::io::{AsRawFd, RawFd};
+use std::pin::Pin;
+use std::sync::atomic::Ordering;
+use std::sync::{Arc, LazyLock, Mutex};
+use std::time::Duration;
+use tracing::instrument;
+
+/// Cache key for UringReader instances.
+/// We cache by (path, block_size) because block_size affects reader behavior.
+#[derive(Clone, Debug, Hash, Eq, PartialEq)]
+pub(super) struct CacheKey {
+    path: String,
+    block_size: usize,
+}
+
+impl CacheKey {
+    pub(super) fn new(path: &Path, block_size: usize) -> Self {
+        Self {
+            path: path.to_string(),
+            block_size,
+        }
+    }
+}
+
+/// Data stored in the cache for each opened file.
+#[derive(Clone)]
+pub(super) struct CachedReaderData {
+    pub(super) handle: Arc<UringFileHandle>,
+    pub(super) size: usize,
+}
+
+/// Global cache of open file handles.
+/// Entries expire after 60 seconds to ensure files are eventually closed.
+pub(super) static HANDLE_CACHE: LazyLock<moka::future::Cache<CacheKey, CachedReaderData>> =
+    LazyLock::new(|| {
+        moka::future::Cache::builder()
+            .time_to_live(Duration::from_secs(60))
+            .max_capacity(10_000)
+            .build()
+    });
+
+/// File handle for io_uring operations.
+///
+/// Keeps the file alive and provides the raw file descriptor.
+#[derive(Debug)]
+pub(super) struct UringFileHandle {
+    /// The file (kept alive via Arc)
+    #[allow(unused)]
+    file: Arc<File>,
+
+    /// Raw file descriptor for io_uring
+    pub(super) fd: RawFd,
+
+    /// Object store path
+    pub(super) path: Path,
+}
+
+impl UringFileHandle {
+    pub(super) fn new(file: File, path: Path) -> Self {
+        let fd = file.as_raw_fd();
+        Self {
+            file: Arc::new(file),
+            fd,
+            path,
+        }
+    }
+}
+
+/// io_uring-based reader for local files.
+///
+/// This reader uses a dedicated process-wide thread running an io_uring event loop
+/// for high-performance asynchronous I/O.
+#[derive(Debug)]
+pub struct UringReader {
+    /// File handle
+    handle: Arc<UringFileHandle>,
+
+    /// Block size for I/O operations
+    block_size: usize,
+
+    /// File size (determined at open time)
+    size: usize,
+
+    /// I/O tracker for monitoring operations
+    io_tracker: Arc<IOTracker>,
+}
+
+impl DeepSizeOf for UringReader {
+    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+        // Skip file handle (just a system resource)
+        // Only count the path's deep size
+        self.handle.path.as_ref().deep_size_of_children(context)
+    }
+}
+
+impl UringReader {
+    /// Open a file with io_uring.
+    ///
+    /// This is the internal constructor used by ObjectStore.
+    #[instrument(level = "debug")]
+    pub(crate) async fn open(
+        path: &Path,
+        block_size: usize,
+        known_size: Option<usize>,
+        io_tracker: Arc<IOTracker>,
+    ) -> Result<Box<dyn Reader>> {
+        // Determine block size with environment variable override
+        let block_size = URING_BLOCK_SIZE.unwrap_or(block_size.max(DEFAULT_URING_BLOCK_SIZE));
+
+        let cache_key = CacheKey::new(path, block_size);
+
+        // Try to get from cache first
+        if let Some(data) = HANDLE_CACHE.get(&cache_key).await {
+            // Use known_size if provided, otherwise use cached size
+            let size = known_size.unwrap_or(data.size);
+            return Ok(Box::new(Self {
+                handle: data.handle,
+                block_size,
+                size,
+                io_tracker,
+            }) as Box<dyn Reader>);
+        }
+
+        // Cache miss - open file and get size
+        let path_clone = path.clone();
+        let local_path = to_local_path(path);
+
+        let data = tokio::task::spawn_blocking(move || {
+            let file = File::open(&local_path).map_err(|e| match e.kind() {
+                ErrorKind::NotFound => Error::not_found(path_clone.to_string()),
+                _ => e.into(),
+            })?;
+
+            // Get size from known_size or file metadata
+            let size = match known_size {
+                Some(s) => s,
+                None => file.metadata()?.len() as usize,
+            };
+
+            Ok::<_, Error>(CachedReaderData {
+                handle: Arc::new(UringFileHandle::new(file, path_clone)),
+                size,
+            })
+        })
+        .await??;
+
+        // Insert into cache
+        HANDLE_CACHE.insert(cache_key, data.clone()).await;
+
+        // Return new reader instance
+        Ok(Box::new(Self {
+            handle: data.handle.clone(),
+            block_size,
+            size: data.size,
+            io_tracker,
+        }) as Box<dyn Reader>)
+    }
+
+    /// Submit a read request to the io_uring thread via channel and return a future.
+    fn submit_read(
+        &self,
+        offset: u64,
+        length: usize,
+    ) -> Pin<Box<dyn Future<Output = object_store::Result<Bytes>> + Send>> {
+        let mut buffer = BytesMut::with_capacity(length);
+        unsafe {
+            buffer.set_len(length);
+        }
+
+        // Create IoRequest with all data
+        let request = Arc::new(IoRequest {
+            fd: self.handle.fd,
+            offset,
+            length,
+            thread_id: std::thread::current().id(),
+            state: Mutex::new(RequestState {
+                completed: false,
+                waker: None,
+                err: None,
+                buffer,
+                bytes_read: 0,
+            }),
+        });
+
+        // Increment submitted counter before sending to channel
+        SUBMITTED_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+        // Select thread in round-robin fashion
+        let thread_idx =
+            (THREAD_SELECTOR.fetch_add(1, Ordering::Relaxed) as usize) % URING_THREADS.len();
+
+        // Send to selected thread via channel
+        match URING_THREADS[thread_idx]
+            .request_tx
+            .send(Arc::clone(&request))
+        {
+            Ok(()) => {
+                // Return future that will be woken when operation completes
+                Box::pin(UringReadFuture { request })
+            }
+            Err(_) => {
+                // Thread died - decrement counter and return error future
+                SUBMITTED_COUNTER.fetch_sub(1, Ordering::Relaxed);
+                Box::pin(async move {
+                    Err(object_store::Error::Generic {
+                        store: "UringReader",
+                        source: Box::new(io::Error::new(
+                            io::ErrorKind::BrokenPipe,
+                            "io_uring thread died",
+                        )),
+                    })
+                })
+            }
+        }
+    }
+}
+
+impl Reader for UringReader {
+    fn path(&self) -> &Path {
+        &self.handle.path
+    }
+
+    fn block_size(&self) -> usize {
+        self.block_size
+    }
+
+    fn io_parallelism(&self) -> usize {
+        std::env::var("LANCE_URING_IO_PARALLELISM")
+            .ok()
+            .and_then(|s| s.parse().ok())
+            .unwrap_or(DEFAULT_URING_IO_PARALLELISM)
+    }
+
+    /// Returns the file size.
+    fn size(&self) -> BoxFuture<'_, object_store::Result<usize>> {
+        Box::pin(async move { Ok(self.size) })
+    }
+
+    /// Read a range of bytes using io_uring.
+    #[instrument(level = "debug", skip(self))]
+    fn get_range(&self, range: Range<usize>) -> BoxFuture<'static, object_store::Result<Bytes>> {
+        let io_tracker = self.io_tracker.clone();
+        let path = self.handle.path.clone();
+        let num_bytes = range.len() as u64;
+        let range_u64 = (range.start as u64)..(range.end as u64);
+
+        self.submit_read(range.start as u64, range.len())
+            .map_ok(move |bytes| {
+                io_tracker.record_read("get_range", path, num_bytes, Some(range_u64));
+                bytes
+            })
+            .boxed()
+    }
+
+    /// Read the entire file using io_uring.
+    #[instrument(level = "debug", skip(self))]
+    fn get_all(&self) -> BoxFuture<'static, object_store::Result<Bytes>> {
+        let size = self.size;
+        let io_tracker = self.io_tracker.clone();
+        let path = self.handle.path.clone();
+
+        self.submit_read(0, size)
+            .map_ok(move |bytes| {
+                io_tracker.record_read("get_all", path, bytes.len() as u64, None);
+                bytes
+            })
+            .boxed()
+    }
+}
diff --git a/rust/lance-io/src/uring/requests.rs b/rust/lance-io/src/uring/requests.rs
new file mode 100644
index 00000000000..fa257507dc1
--- /dev/null
+++ b/rust/lance-io/src/uring/requests.rs
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Protocol types for communication between UringReader and the io_uring thread.
+
+use bytes::BytesMut;
+use std::io;
+use std::os::unix::io::RawFd;
+use std::sync::Mutex;
+use std::task::Waker;
+use std::thread::ThreadId;
+
+pub(super) struct RequestState {
+    pub completed: bool,
+    pub waker: Option<Waker>,
+    pub err: Option<io::Error>,
+    pub buffer: BytesMut,
+    /// Accumulated bytes read across retries (for handling short reads).
+    pub bytes_read: usize,
+}
+
+/// I/O request object that contains all state for a single read operation.
+/// This is shared between the submitter, uring thread, and future via Arc.
+pub(super) struct IoRequest {
+    /// File descriptor to read from.
+    pub fd: RawFd,
+
+    /// Byte offset to start reading from.
+    pub offset: u64,
+
+    /// Number of bytes to read.
+    pub length: usize,
+
+    pub thread_id: ThreadId,
+
+    /// Completion flag - set to true when operation completes.
+    pub state: Mutex<RequestState>,
+}
+
+impl IoRequest {
+    /// Mark this request as failed with the given error.
+    ///
+    /// Sets the error, marks completed, and wakes any waiting future.
+    /// Used when a request cannot be submitted (e.g. SQ full).
+    pub(super) fn fail(&self, err: io::Error) {
+        let mut state = self.state.lock().unwrap();
+        state.err = Some(err);
+        state.completed = true;
+        if let Some(waker) = state.waker.take() {
+            drop(state);
+            waker.wake();
+        }
+    }
+}
diff --git a/rust/lance-io/src/uring/tests.rs b/rust/lance-io/src/uring/tests.rs
new file mode 100644
index 00000000000..19d931da629
--- /dev/null
+++ b/rust/lance-io/src/uring/tests.rs
@@ -0,0 +1,392 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Tests for io_uring reader implementation.
+
+use crate::object_store::ObjectStore;
+use lance_core::Result;
+use std::io::Write;
+use std::time::Duration;
+use tempfile::NamedTempFile;
+
+/// Helper to create a temporary file with test data
+fn create_test_file(size: usize) -> Result<(NamedTempFile, Vec<u8>)> {
+    let mut file = NamedTempFile::new()?;
+    let data: Vec<u8> = (0..size).map(|i| (i % 256) as u8).collect();
+    file.write_all(&data)?;
+    file.flush()?;
+    Ok((file, data))
+}
+
+#[tokio::test]
+async fn test_read_small_file() -> Result<()> {
+    let (file, expected_data) = create_test_file(1024)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Read entire file
+    let data = reader.get_all().await.unwrap();
+    assert_eq!(data.as_ref(), expected_data.as_slice());
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_read_range() -> Result<()> {
+    let (file, expected_data) = create_test_file(4096)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Read a range in the middle
+    let range = 1000..2000;
+    let data = reader.get_range(range.clone()).await.unwrap();
+    assert_eq!(data.as_ref(), &expected_data[range]);
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_read_multiple_ranges() -> Result<()> {
+    let (file, expected_data) = create_test_file(8192)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Read multiple ranges
+    let ranges = vec![0..100, 500..600, 2000..3000];
+    for range in ranges {
+        let data = reader.get_range(range.clone()).await.unwrap();
+        assert_eq!(data.as_ref(), &expected_data[range]);
+    }
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_file_size() -> Result<()> {
+    let size = 5000;
+    let (file, _) = create_test_file(size)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    assert_eq!(reader.size().await.unwrap(), size);
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_concurrent_reads() -> Result<()> {
+    let (file, expected_data) = create_test_file(16384)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+
+    // Perform multiple concurrent reads
+    let mut tasks = vec![];
+    for i in 0..10 {
+        let reader_clone = store.open(&path).await?;
+        let expected = expected_data.clone();
+        tasks.push(tokio::spawn(async move {
+            let range = (i * 1000)..((i + 1) * 1000);
+            let data = reader_clone.get_range(range.clone()).await.unwrap();
+            assert_eq!(data.as_ref(), &expected[range]);
+        }));
+    }
+
+    // Wait for all tasks
+    for task in tasks {
+        task.await.unwrap();
+    }
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_large_file_read() -> Result<()> {
+    // Test with a larger file (1MB)
+    let size = 1024 * 1024;
+    let (file, expected_data) = create_test_file(size)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Read entire file
+    let data = reader.get_all().await.unwrap();
+    assert_eq!(data.len(), size);
+    assert_eq!(data.as_ref(), expected_data.as_slice());
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_read_edge_cases() -> Result<()> {
+    let (file, expected_data) = create_test_file(4096)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Read from start
+    let data = reader.get_range(0..100).await.unwrap();
+    assert_eq!(data.as_ref(), &expected_data[0..100]);
+
+    // Read to end
+    let data = reader.get_range(4000..4096).await.unwrap();
+    assert_eq!(data.as_ref(), &expected_data[4000..4096]);
+
+    // Read single byte
+    let data = reader.get_range(2000..2001).await.unwrap();
+    assert_eq!(data.as_ref(), &expected_data[2000..2001]);
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_file_not_found() {
+    let uri = "file+uring:///nonexistent/file.dat";
+    let (store, path) = ObjectStore::from_uri(uri).await.unwrap();
+
+    // Should fail to open non-existent file
+    let result = store.open(&path).await;
+    assert!(result.is_err());
+}
+
+#[tokio::test]
+async fn test_block_size_and_parallelism() -> Result<()> {
+    let (file, _) = create_test_file(1024)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Check default values (or configured values)
+    assert!(reader.block_size() > 0);
+    assert!(reader.io_parallelism() > 0);
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_path() -> Result<()> {
+    let (file, _) = create_test_file(1024)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Verify path is preserved
+    assert_eq!(reader.path(), &path);
+
+    Ok(())
+}
+
+/// Test that reading past EOF returns an error.
+///
+/// This exercises the case where `known_size` passed to `open_with_size` is larger
+/// than the actual file, causing io_uring to hit EOF before the full read completes.
+#[tokio::test]
+async fn test_short_read_get_all() -> Result<()> {
+    let actual_size: usize = 8192;
+    let (file, _expected_data) = create_test_file(actual_size)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+
+    // Open with inflated known_size — the reader will think the file is 2x its real size
+    let inflated_size = actual_size * 2;
+    let reader = store.open_with_size(&path, inflated_size).await?;
+
+    // get_all() will submit a read for inflated_size bytes from an actual_size file.
+    // The kernel reads actual_size bytes then returns 0 (EOF) — this should be an error.
+    let result = reader.get_all().await;
+    assert!(result.is_err(), "reading past EOF should return an error");
+
+    Ok(())
+}
+
+/// Test that a range read extending past EOF returns an error.
+#[tokio::test]
+async fn test_short_read_get_range_past_eof() -> Result<()> {
+    let actual_size: usize = 8192;
+    let (file, _expected_data) = create_test_file(actual_size)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Request a range that starts inside the file but extends past EOF.
+    // File is 8192 bytes; reading 4096..16384 hits EOF — this should be an error.
+    let range_start = 4096;
+    let range_end = actual_size * 2; // 16384, well past EOF
+    let result = reader.get_range(range_start..range_end).await;
+    assert!(
+        result.is_err(),
+        "range extending past EOF should return an error"
+    );
+
+    Ok(())
+}
+
+/// Test that when push_to_sq fails (SQ full), the request's future returns
+/// an error instead of hanging forever.
+///
+/// This directly tests the thread-path scenario: create an IoUring with
+/// queue_depth=2, fill the SQ, then try to push a 3rd request. The 3rd
+/// request's future should return an error within the timeout.
+///
+/// BUG: currently the failed push silently drops the request, so the
+/// future hangs and the timeout fires.
+#[tokio::test]
+async fn test_retry_sq_full_thread() -> Result<()> {
+    use super::future::UringReadFuture;
+    use super::requests::{IoRequest, RequestState};
+    use super::thread::push_to_sq;
+    use bytes::BytesMut;
+    use io_uring::IoUring;
+    use std::collections::HashMap;
+    use std::os::unix::io::AsRawFd;
+    use std::sync::{Arc, Mutex};
+
+    let (file, _) = create_test_file(4096)?;
+    let fd = file.as_file().as_raw_fd();
+
+    // Create a tiny ring with queue_depth=2
+    let mut ring = IoUring::new(2).unwrap();
+    let mut pending: HashMap<u64, Arc<IoRequest>> = HashMap::new();
+
+    // Helper to create a request
+    let make_request = || {
+        Arc::new(IoRequest {
+            fd,
+            offset: 0,
+            length: 4096,
+            thread_id: std::thread::current().id(),
+            state: Mutex::new(RequestState {
+                completed: false,
+                waker: None,
+                err: None,
+                buffer: BytesMut::zeroed(4096),
+                bytes_read: 0,
+            }),
+        })
+    };
+
+    // Fill the SQ (capacity=2)
+    let _r1 = make_request();
+    let _r2 = make_request();
+    push_to_sq(&mut ring, &mut pending, _r1).unwrap();
+    push_to_sq(&mut ring, &mut pending, _r2).unwrap();
+
+    // 3rd push should fail — SQ is full
+    let r3 = make_request();
+    let push_result = push_to_sq(&mut ring, &mut pending, r3.clone());
+    assert!(push_result.is_err(), "3rd push should fail (SQ full)");
+
+    // r3's future should return an error, not hang forever.
+    // BUG: currently nobody sets completed=true or err on r3, so the future hangs.
+    let future = UringReadFuture { request: r3 };
+    let result = tokio::time::timeout(Duration::from_secs(2), future).await;
+    assert!(
+        result.is_ok(),
+        "future timed out — request was dropped without error on SQ-full push failure"
+    );
+
+    Ok(())
+}
+
+/// Test that when push_to_sq fails (SQ full) on the current-thread path,
+/// the request's future returns an error instead of hanging forever.
+///
+/// Uses UringCurrentThreadFuture (which will be a no-op poller since the
+/// thread-local URING has no knowledge of this request) after push_to_sq
+/// has already completed the request with an error.
+#[tokio::test(flavor = "current_thread")]
+async fn test_retry_sq_full_current_thread() -> Result<()> {
+    use super::current_thread_future::UringCurrentThreadFuture;
+    use super::requests::{IoRequest, RequestState};
+    use super::thread::push_to_sq;
+    use bytes::BytesMut;
+    use io_uring::IoUring;
+    use std::collections::HashMap;
+    use std::os::unix::io::AsRawFd;
+    use std::sync::{Arc, Mutex};
+
+    let (file, _) = create_test_file(4096)?;
+    let fd = file.as_file().as_raw_fd();
+
+    // Create a tiny ring with queue_depth=2
+    let mut ring = IoUring::new(2).unwrap();
+    let mut pending: HashMap<u64, Arc<IoRequest>> = HashMap::new();
+
+    let make_request = || {
+        Arc::new(IoRequest {
+            fd,
+            offset: 0,
+            length: 4096,
+            thread_id: std::thread::current().id(),
+            state: Mutex::new(RequestState {
+                completed: false,
+                waker: None,
+                err: None,
+                buffer: BytesMut::zeroed(4096),
+                bytes_read: 0,
+            }),
+        })
+    };
+
+    // Fill the SQ (capacity=2)
+    push_to_sq(&mut ring, &mut pending, make_request()).unwrap();
+    push_to_sq(&mut ring, &mut pending, make_request()).unwrap();
+
+    // 3rd push should fail — SQ is full
+    let r3 = make_request();
+    let push_result = push_to_sq(&mut ring, &mut pending, r3.clone());
+    assert!(push_result.is_err(), "3rd push should fail (SQ full)");
+
+    // r3's future should return an error, not hang forever.
+    let future = UringCurrentThreadFuture::new(r3);
+    let result = tokio::time::timeout(Duration::from_secs(2), future).await;
+    assert!(
+        result.is_ok(),
+        "future timed out — request was dropped without error on SQ-full push failure"
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_uring_not_enabled_with_file_scheme() -> Result<()> {
+    // Verify that files opened with file:// don't use uring
+    let (file, expected_data) = create_test_file(1024)?;
+    let file_path = file.path().to_str().unwrap();
+    // Use regular file:// scheme, should NOT use uring
+    let uri = format!("file://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Should still be able to read, just won't use uring
+    let data = reader.get_all().await.unwrap();
+    assert_eq!(data.as_ref(), expected_data.as_slice());
+
+    Ok(())
+}
diff --git a/rust/lance-io/src/uring/thread.rs b/rust/lance-io/src/uring/thread.rs
new file mode 100644
index 00000000000..d2ef197947d
--- /dev/null
+++ b/rust/lance-io/src/uring/thread.rs
@@ -0,0 +1,396 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Dedicated thread for io_uring operations.
+//!
+//! This module provides a background thread that owns an io_uring instance
+//! and processes read requests from a channel. Readers send requests via
+//! an MPSC channel, and the thread handles submission and completion processing.
+
+use super::DEFAULT_URING_QUEUE_DEPTH;
+use super::requests::IoRequest;
+use io_uring::{IoUring, opcode, types};
+use std::collections::HashMap;
+use std::io;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::mpsc::{Receiver, RecvTimeoutError, SyncSender, sync_channel};
+use std::sync::{Arc, LazyLock};
+use std::time::{Duration, Instant};
+
+/// Handle to the io_uring background thread.
+///
+/// This provides a channel sender for submitting read requests to the thread.
+pub(super) struct UringThreadHandle {
+    pub request_tx: SyncSender<Arc<IoRequest>>,
+}
+
+/// Lazy-initialized io_uring thread pool.
+///
+/// Multiple threads are spawned on first access and run until process exit.
+pub(super) static URING_THREADS: LazyLock<Vec<UringThreadHandle>> = LazyLock::new(|| {
+    let queue_depth = get_queue_depth();
+    let thread_count = get_thread_count();
+
+    let mut threads = Vec::with_capacity(thread_count);
+
+    for i in 0..thread_count {
+        let (tx, rx) = sync_channel(queue_depth);
+
+        std::thread::Builder::new()
+            .name(format!("lance-uring-{}", i))
+            .spawn(move || run_uring_thread(rx, queue_depth, i))
+            .expect("Failed to spawn io_uring thread");
+
+        threads.push(UringThreadHandle { request_tx: tx });
+    }
+
+    log::info!(
+        "io_uring thread pool spawned ({} threads, queue_depth={})",
+        thread_count,
+        queue_depth
+    );
+
+    threads
+});
+
+/// Atomic counter for round-robin thread selection.
+pub(super) static THREAD_SELECTOR: AtomicU64 = AtomicU64::new(0);
+
+/// Counter for generating unique user_data values.
+///
+/// Each io_uring operation needs a unique user_data ID to match completions
+/// with their corresponding requests.
+static USER_DATA_COUNTER: AtomicU64 = AtomicU64::new(1);
+
+/// Counter for requests that have been submitted to the thread but not yet received.
+///
+/// This tracks requests sitting in the channel queue waiting to be received by the thread.
+pub(super) static SUBMITTED_COUNTER: AtomicU64 = AtomicU64::new(0);
+
+/// Default batch size for submission - how many requests to batch before calling submit().
+const DEFAULT_SUBMIT_BATCH_SIZE: usize = 128;
+
+/// Default number of io_uring threads.
+const DEFAULT_URING_THREAD_COUNT: usize = 2;
+
+/// Get the configured queue depth from environment variable.
+fn get_queue_depth() -> usize {
+    std::env::var("LANCE_URING_QUEUE_DEPTH")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(DEFAULT_URING_QUEUE_DEPTH)
+}
+
+/// Get the configured poll timeout from environment variable.
+fn get_poll_timeout() -> Duration {
+    let timeout_ms = std::env::var("LANCE_URING_POLL_TIMEOUT_MS")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(10);
+
+    Duration::from_millis(timeout_ms)
+}
+
+/// Get the configured submit batch size from environment variable.
+fn get_submit_batch_size() -> usize {
+    std::env::var("LANCE_URING_SUBMIT_BATCH_SIZE")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(DEFAULT_SUBMIT_BATCH_SIZE)
+}
+
+/// Get the configured number of uring threads from environment variable.
+fn get_thread_count() -> usize {
+    std::env::var("LANCE_URING_THREAD_COUNT")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(DEFAULT_URING_THREAD_COUNT)
+}
+
+/// Main loop for the io_uring thread.
+///
+/// This thread:
+/// 1. Receives requests from the channel
+/// 2. Submits them to io_uring
+/// 3. Processes completions
+/// 4. Wakes futures via their wakers
+fn run_uring_thread(request_rx: Receiver<Arc<IoRequest>>, queue_depth: usize, thread_id: usize) {
+    // Create local io_uring instance
+    let mut ring = IoUring::builder()
+        // .setup_sqpoll(100)
+        .build(queue_depth as u32)
+        .expect("Failed to create io_uring");
+
+    let mut pending: HashMap<u64, Arc<IoRequest>> = HashMap::with_capacity(queue_depth);
+    let poll_timeout = get_poll_timeout();
+    let submit_batch_size = get_submit_batch_size();
+    let mut last_log = Instant::now();
+    let log_interval = Duration::from_millis(100);
+    let mut completed_iops = 0usize;
+    let mut completed_sectors = 0usize;
+    let mut min_in_flight = usize::MAX;
+
+    loop {
+        // Track minimum in-flight count
+        let in_flight = pending.len();
+        min_in_flight = min_in_flight.min(in_flight);
+
+        // Log in-flight requests every 100ms
+        let now = Instant::now();
+        if now.duration_since(last_log) >= log_interval {
+            let submitted = SUBMITTED_COUNTER.load(Ordering::Relaxed);
+            log::info!(
+                "io_uring[{}]: {} submitted, {} in flight (min {}), {} iops completed, {} sectors completed",
+                thread_id,
+                submitted,
+                in_flight,
+                min_in_flight,
+                completed_iops,
+                completed_sectors
+            );
+            last_log = now;
+            completed_iops = 0; // Reset counter after logging
+            completed_sectors = 0; // Reset counter after logging
+            min_in_flight = usize::MAX; // Reset min tracker
+        }
+
+        // Process all available completions first
+        let mut needs_submit = false;
+        let completions = process_completions(&mut ring, &mut pending);
+        match completions {
+            Ok(result) => {
+                completed_iops += result.iops;
+                completed_sectors += result.sectors;
+
+                // Resubmit any short-read retries
+                for request in result.retries {
+                    if let Err(e) = push_to_sq(&mut ring, &mut pending, request) {
+                        log::error!("Failed to resubmit short read: {}", e);
+                    } else {
+                        needs_submit = true;
+                    }
+                }
+            }
+            Err(e) => {
+                log::error!("Error processing io_uring completions: {}", e);
+            }
+        }
+
+        min_in_flight = min_in_flight.min(pending.len());
+
+        // Batch submit requests - keep pulling from channel and pushing to SQ
+        // until we hit batch size or channel is empty
+        let mut batch_count = 0;
+        loop {
+            // Try to receive new request
+            // Use recv_timeout only when pending is empty, otherwise use try_recv
+            let recv_result = if pending.is_empty() && batch_count == 0 {
+                // No operations in flight and no batch started - we can afford to wait with timeout
+                request_rx.recv_timeout(poll_timeout).map_err(|e| match e {
+                    RecvTimeoutError::Timeout => std::sync::mpsc::TryRecvError::Empty,
+                    RecvTimeoutError::Disconnected => std::sync::mpsc::TryRecvError::Disconnected,
+                })
+            } else {
+                // Operations in flight or batch in progress - busy loop with try_recv
+                request_rx.try_recv()
+            };
+
+            match recv_result {
+                Ok(request) => {
+                    // Decrement submitted counter when we receive the request from channel
+                    SUBMITTED_COUNTER.fetch_sub(1, Ordering::Relaxed);
+
+                    // Push to submission queue (but don't submit yet)
+                    if let Err(e) = push_to_sq(&mut ring, &mut pending, request) {
+                        log::error!("Failed to push to io_uring SQ: {}", e);
+                    } else {
+                        batch_count += 1;
+                    }
+
+                    // Break if we've hit the batch size limit
+                    if batch_count >= submit_batch_size {
+                        break;
+                    }
+                }
+                Err(std::sync::mpsc::TryRecvError::Empty) => {
+                    // No more requests in channel - break to submit the batch
+                    break;
+                }
+                Err(std::sync::mpsc::TryRecvError::Disconnected) => {
+                    // All senders dropped - submit batch and shutdown
+                    if batch_count > 0
+                        && let Err(e) = ring.submit()
+                    {
+                        log::error!(
+                            "io_uring[{}]: Failed to submit io_uring batch: {}",
+                            thread_id,
+                            e
+                        );
+                    }
+                    log::info!(
+                        "io_uring thread {} shutting down (channel disconnected)",
+                        thread_id
+                    );
+                    return;
+                }
+            }
+        }
+
+        // Submit if we have any requests (from channel or retries)
+        if (batch_count > 0 || needs_submit)
+            && let Err(e) = ring.submit()
+        {
+            log::error!(
+                "Failed to submit io_uring batch of {} requests: {}",
+                batch_count,
+                e
+            );
+        }
+    }
+}
+
+/// Push a read request to the io_uring submission queue (without submitting).
+///
+/// This generates a unique user_data ID, prepares the read operation,
+/// and pushes it to the SQ. The caller is responsible for calling ring.submit().
+pub(super) fn push_to_sq(
+    ring: &mut IoUring,
+    pending: &mut HashMap<u64, Arc<IoRequest>>,
+    request: Arc<IoRequest>,
+) -> io::Result<()> {
+    // Generate unique user_data
+    let user_data = USER_DATA_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+    // Get buffer pointer, adjusting for any bytes already read (short read retry)
+    let (buffer_ptr, read_offset, read_length) = {
+        let state = request.state.lock().unwrap();
+        let br = state.bytes_read;
+        (
+            unsafe { state.buffer.as_ptr().add(br) as *mut u8 },
+            request.offset + br as u64,
+            (request.length - br) as u32,
+        )
+    };
+
+    // Prepare read operation
+    let read_op =
+        opcode::Read::new(types::Fd(request.fd), buffer_ptr, read_length).offset(read_offset);
+
+    // Get submission queue
+    let mut sq = ring.submission();
+
+    // Check if SQ has space
+    if sq.is_full() {
+        drop(sq);
+        request.fail(io::Error::new(
+            io::ErrorKind::WouldBlock,
+            "io_uring submission queue full",
+        ));
+        return Err(io::Error::new(
+            io::ErrorKind::WouldBlock,
+            "io_uring submission queue full",
+        ));
+    }
+
+    // Push to SQ
+    unsafe {
+        if sq.push(&read_op.build().user_data(user_data)).is_err() {
+            drop(sq);
+            request.fail(io::Error::other("Failed to push to SQ"));
+            return Err(io::Error::other("Failed to push to SQ"));
+        }
+    }
+    drop(sq);
+
+    // Track request in pending map
+    pending.insert(user_data, request);
+
+    Ok(())
+}
+
+struct CompletionResult {
+    iops: usize,
+    sectors: usize,
+    retries: Vec<Arc<IoRequest>>,
+}
+
+/// Process all available completions from the io_uring.
+///
+/// This iterates through the completion queue, matches completions to requests,
+/// updates their state, and wakes any waiting futures. Short reads are collected
+/// into `retries` for resubmission; EOF before a full read is an error.
+///
+/// Returns completion stats and a list of requests needing resubmission.
+fn process_completions(
+    ring: &mut IoUring,
+    pending: &mut HashMap<u64, Arc<IoRequest>>,
+) -> io::Result<CompletionResult> {
+    let mut iops = 0;
+    let mut sectors = 0;
+    let mut retries = Vec::new();
+
+    // Process all available completions
+    for cqe in ring.completion() {
+        let user_data = cqe.user_data();
+        let result = cqe.result();
+
+        // Look up request
+        if let Some(request) = pending.remove(&user_data) {
+            let mut state = request.state.lock().unwrap();
+
+            if result < 0 {
+                // Kernel error
+                state.err = Some(io::Error::from_raw_os_error(-result));
+                state.completed = true;
+            } else if result == 0 {
+                // EOF before full read completed
+                let br = state.bytes_read;
+                state.err = Some(io::Error::new(
+                    io::ErrorKind::UnexpectedEof,
+                    format!("unexpected EOF: read {} of {} bytes", br, request.length),
+                ));
+                state.buffer.truncate(br);
+                state.completed = true;
+            } else {
+                // Positive result: n bytes read
+                let n = result as usize;
+                state.bytes_read += n;
+                let br = state.bytes_read;
+
+                if br >= request.length {
+                    // Full read complete
+                    state.buffer.truncate(br);
+                    state.completed = true;
+
+                    if request.length > 0 {
+                        let first_sector = request.offset / 4096;
+                        let last_sector = (request.offset + request.length as u64 - 1) / 4096;
+                        let num_sectors = (last_sector - first_sector + 1) as usize;
+                        sectors += num_sectors;
+                    }
+                } else {
+                    // Short read — need retry; don't mark completed or wake
+                    drop(state);
+                    retries.push(request);
+                    continue;
+                }
+            }
+
+            // Wake the future if it's waiting
+            if let Some(waker) = state.waker.take() {
+                drop(state); // Release lock before waking
+                waker.wake();
+            }
+
+            iops += 1;
+        } else {
+            log::warn!("Received completion for unknown user_data: {}", user_data);
+        }
+    }
+
+    Ok(CompletionResult {
+        iops,
+        sectors,
+        retries,
+    })
+}
diff --git a/rust/lance-io/src/utils.rs b/rust/lance-io/src/utils.rs
index 48909728e3a..48d28526a5e 100644
--- a/rust/lance-io/src/utils.rs
+++ b/rust/lance-io/src/utils.rs
@@ -233,6 +233,10 @@ impl PartialEq for CachedFileSize {
 impl Eq for CachedFileSize {}
 
 impl CachedFileSize {
+    /// Create a `CachedFileSize` from a raw byte count.
+    ///
+    /// Passing `0` is equivalent to calling [`unknown`](Self::unknown): the
+    /// type interprets zero as "size not yet known".
     pub fn new(size: u64) -> Self {
         Self(AtomicU64::new(size))
     }
@@ -307,4 +311,39 @@ mod tests {
         let actual: BytesWrapper = read_struct(&object_reader, pos).await.unwrap();
         assert_eq!(some_message, actual);
     }
+
+    #[tokio::test]
+    async fn test_copy_reader_to_writer() {
+        let store = ObjectStore::memory();
+        let src = Path::from("/src");
+        let dst = Path::from("/dst");
+        store.put(&src, b"abcdef").await.unwrap();
+
+        let reader = store.open(&src).await.unwrap();
+        let mut writer = store.create(&dst).await.unwrap();
+        let copied = writer.copy_from_reader(reader.as_ref()).await.unwrap();
+        writer.shutdown().await.unwrap();
+
+        assert_eq!(copied, 6);
+        assert_eq!(store.read_one_all(&dst).await.unwrap().as_ref(), b"abcdef");
+    }
+
+    #[tokio::test]
+    async fn test_copy_reader_range_to_writer() {
+        let store = ObjectStore::memory();
+        let src = Path::from("/src-range");
+        let dst = Path::from("/dst-range");
+        store.put(&src, b"abcdef").await.unwrap();
+
+        let reader = store.open(&src).await.unwrap();
+        let mut writer = store.create(&dst).await.unwrap();
+        let copied = writer
+            .copy_range_from_reader(reader.as_ref(), 2..5)
+            .await
+            .unwrap();
+        writer.shutdown().await.unwrap();
+
+        assert_eq!(copied, 3);
+        assert_eq!(store.read_one_all(&dst).await.unwrap().as_ref(), b"cde");
+    }
 }
diff --git a/rust/lance-io/src/utils/tracking_store.rs b/rust/lance-io/src/utils/tracking_store.rs
index dd8474b5683..706aba34615 100644
--- a/rust/lance-io/src/utils/tracking_store.rs
+++ b/rust/lance-io/src/utils/tracking_store.rs
@@ -190,9 +190,8 @@ macro_rules! assert_io_lt {
     };
 }
 
-// These fields are "dead code" because we just use them right now to display
-// in test failure messages through Debug. (The lint ignores Debug impls.)
-#[allow(dead_code)]
+// These request records only exist for test-only diagnostics.
+#[cfg(feature = "test-util")]
 #[derive(Clone)]
 pub struct IoRequestRecord {
     pub method: &'static str,
@@ -200,6 +199,7 @@ pub struct IoRequestRecord {
     pub range: Option<Range<u64>>,
 }
 
+#[cfg(feature = "test-util")]
 impl std::fmt::Debug for IoRequestRecord {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         // For example: "put /path/to/file range: 0-100"
diff --git a/rust/lance-linalg/Cargo.toml b/rust/lance-linalg/Cargo.toml
index c3cb18868d8..463dc7f02c9 100644
--- a/rust/lance-linalg/Cargo.toml
+++ b/rust/lance-linalg/Cargo.toml
@@ -62,5 +62,9 @@ harness = false
 name = "norm_l2"
 harness = false
 
+[[bench]]
+name = "dist_table"
+harness = false
+
 [lints]
 workspace = true
diff --git a/rust/lance-linalg/benches/cosine.rs b/rust/lance-linalg/benches/cosine.rs
index 57b26538d09..aa812339b86 100644
--- a/rust/lance-linalg/benches/cosine.rs
+++ b/rust/lance-linalg/benches/cosine.rs
@@ -8,6 +8,7 @@ use arrow_array::{
 use criterion::{Criterion, black_box, criterion_group, criterion_main};
 use lance_arrow::{ArrowFloatType, FloatArray, bfloat16::BFloat16Type};
 use lance_linalg::distance::cosine::{Cosine, cosine_distance_batch};
+use lance_linalg::distance::cosine_u8::{cosine_u8, cosine_u8_scalar};
 use num_traits::Float;
 
 #[cfg(target_os = "linux")]
@@ -76,6 +77,42 @@ fn bench_distance(c: &mut Criterion) {
             black_box(cosine_distance_batch(key.values(), target.values(), 8).collect::<Vec<_>>())
         })
     });
+
+    // u8 cosine benchmarks
+    {
+        use rand::Rng;
+        use std::iter::repeat_with;
+
+        const DIMENSION: usize = 1024;
+        const TOTAL: usize = 1024 * 1024;
+        let mut rng = rand::rng();
+        let key_u8: Vec<u8> = repeat_with(|| rng.random()).take(DIMENSION).collect();
+        let target_u8: Vec<u8> = repeat_with(|| rng.random())
+            .take(TOTAL * DIMENSION)
+            .collect();
+
+        c.bench_function("Cosine(u8, scalar)", |b| {
+            b.iter(|| {
+                black_box(
+                    target_u8
+                        .chunks_exact(DIMENSION)
+                        .map(|tgt| cosine_u8_scalar(&key_u8, tgt))
+                        .fold(0.0, |acc: f32, v| acc + v),
+                );
+            });
+        });
+
+        c.bench_function("Cosine(u8, SIMD)", |b| {
+            b.iter(|| {
+                black_box(
+                    target_u8
+                        .chunks_exact(DIMENSION)
+                        .map(|tgt| cosine_u8(&key_u8, tgt))
+                        .fold(0.0, |acc: f32, v| acc + v),
+                );
+            });
+        });
+    }
 }
 
 #[cfg(target_os = "linux")]
diff --git a/rust/lance-linalg/benches/dist_table.rs b/rust/lance-linalg/benches/dist_table.rs
new file mode 100644
index 00000000000..acd0c04c644
--- /dev/null
+++ b/rust/lance-linalg/benches/dist_table.rs
@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Benchmark of 4-bit LUT distance table summation (RaBitQ inner loop).
+//!
+//! Measures both the dispatched path (NEON on ARM, AVX2 on x86) and the
+//! scalar fallback, so the speedup is visible in a single benchmark run.
+
+use std::iter::repeat_with;
+
+use criterion::{Criterion, black_box, criterion_group, criterion_main};
+use lance_linalg::simd::dist_table::{BATCH_SIZE, sum_4bit_dist_table, sum_4bit_dist_table_scalar};
+use rand::Rng;
+
+fn bench_sum_4bit_dist_table(c: &mut Criterion) {
+    let mut rng = rand::rng();
+
+    // code_len = dim / 8 for 1-bit quantization
+    for (label, n_vectors, code_len) in [
+        ("32vec_dim128", 32_usize, 16_usize),
+        ("32vec_dim1536", 32, 192),
+        ("32vec_dim4096", 32, 512),
+        ("32vec_dim65536", 32, 8192),
+        ("16Kvec_dim128", 16_000, 16),
+        ("16Kvec_dim1536", 16_000, 192),
+    ] {
+        let n = n_vectors.div_ceil(BATCH_SIZE) * BATCH_SIZE;
+
+        let codes: Vec<u8> = repeat_with(|| rng.random::<u8>())
+            .take(n * code_len)
+            .collect();
+
+        let dist_table: Vec<u8> = repeat_with(|| rng.random::<u8>())
+            .take(BATCH_SIZE * code_len)
+            .collect();
+
+        let mut dists = vec![0u16; n];
+
+        // Dispatched path (NEON on ARM, AVX2 on x86)
+        c.bench_function(&format!("sum_4bit_dist_table/simd/{}", label), |b| {
+            b.iter(|| {
+                dists.fill(0);
+                sum_4bit_dist_table(n, code_len, &codes, &dist_table, &mut dists);
+                black_box(&dists);
+            })
+        });
+
+        // Scalar reference path
+        c.bench_function(&format!("sum_4bit_dist_table/scalar/{}", label), |b| {
+            b.iter(|| {
+                dists.fill(0);
+                sum_4bit_dist_table_scalar(code_len, &codes, &dist_table, &mut dists);
+                black_box(&dists);
+            })
+        });
+    }
+}
+
+criterion_group!(
+    name = benches;
+    config = Criterion::default().significance_level(0.1).sample_size(10);
+    targets = bench_sum_4bit_dist_table
+);
+criterion_main!(benches);
diff --git a/rust/lance-linalg/benches/dot.rs b/rust/lance-linalg/benches/dot.rs
index 17fd1b891eb..3b1beaba7bf 100644
--- a/rust/lance-linalg/benches/dot.rs
+++ b/rust/lance-linalg/benches/dot.rs
@@ -107,6 +107,40 @@ fn bench_distance(c: &mut Criterion) {
         });
     });
 
+    // u8 dot product benchmarks: scalar baseline vs SIMD dispatch
+    {
+        use lance_linalg::distance::dot_u8::{dot_u8, dot_u8_scalar};
+
+        for &dim in &[128, 256, 512, 1024] {
+            let num_vectors = 1024 * 1024 / dim; // ~1M elements total
+            let mut rng = rand::rng();
+            let key_u8: Vec<u8> = (0..dim).map(|_| rng.random()).collect();
+            let target_u8: Vec<u8> = (0..num_vectors * dim).map(|_| rng.random()).collect();
+
+            c.bench_function(&format!("Dot(u8, scalar, dim={dim})"), |b| {
+                b.iter(|| {
+                    black_box(
+                        target_u8
+                            .chunks(dim)
+                            .map(|y| dot_u8_scalar(key_u8.as_slice(), y))
+                            .collect::<Vec<_>>(),
+                    )
+                });
+            });
+
+            c.bench_function(&format!("Dot(u8, dispatch, dim={dim})"), |b| {
+                b.iter(|| {
+                    black_box(
+                        target_u8
+                            .chunks(dim)
+                            .map(|y| dot_u8(key_u8.as_slice(), y))
+                            .collect::<Vec<_>>(),
+                    )
+                });
+            });
+        }
+    }
+
     run_bench::<Float32Type>(c);
     c.bench_function("Dot(f32, SIMD)", |b| {
         let key = generate_random_array_with_seed::<Float32Type>(DIMENSION, [0; 32]);
diff --git a/rust/lance-linalg/benches/l2.rs b/rust/lance-linalg/benches/l2.rs
index 81d364cc9a9..ac869c08981 100644
--- a/rust/lance-linalg/benches/l2.rs
+++ b/rust/lance-linalg/benches/l2.rs
@@ -15,6 +15,7 @@ use rand::Rng;
 use pprof::criterion::{Output, PProfProfiler};
 
 use lance_arrow::{ArrowFloatType, FloatArray};
+use lance_linalg::distance::l2_u8::l2_u8;
 use lance_linalg::distance::{L2, l2::l2, l2_distance_batch, l2_distance_uint_scalar};
 use lance_testing::datagen::generate_random_array_with_seed;
 
@@ -157,6 +158,17 @@ fn bench_uint_distance(c: &mut Criterion) {
             );
         });
     });
+
+    c.bench_function("L2(u8, SIMD)", |b| {
+        b.iter(|| {
+            black_box(
+                target
+                    .chunks_exact(DIMENSION)
+                    .map(|tgt| l2_u8(&key, tgt) as f32)
+                    .fold(0.0, |acc, v| acc + v),
+            );
+        });
+    });
 }
 
 #[cfg(target_os = "linux")]
diff --git a/rust/lance-linalg/benches/norm_l2.rs b/rust/lance-linalg/benches/norm_l2.rs
index e8efd68a909..c43cb32b3d8 100644
--- a/rust/lance-linalg/benches/norm_l2.rs
+++ b/rust/lance-linalg/benches/norm_l2.rs
@@ -13,7 +13,7 @@ use num_traits::Float;
 use rand::Rng;
 
 use lance_arrow::{ArrowFloatType, FloatArray, bfloat16::BFloat16Type};
-use lance_linalg::distance::{norm_l2, norm_l2_impl};
+use lance_linalg::distance::{norm_l2, norm_l2_f64_simd, norm_l2_impl};
 use lance_testing::datagen::generate_random_array_with_seed;
 
 #[cfg(target_os = "linux")]
@@ -106,7 +106,7 @@ fn bench_distance(c: &mut Criterion) {
         c,
         target.as_slice(),
         norm_l2_impl::<f64, f32, 8>,
-        None, // TODO: implement SIMD for f64
+        Some(norm_l2_f64_simd),
     );
 }
 
diff --git a/rust/lance-linalg/build.rs b/rust/lance-linalg/build.rs
index 152163ed611..06e1439c77a 100644
--- a/rust/lance-linalg/build.rs
+++ b/rust/lance-linalg/build.rs
@@ -19,6 +19,7 @@ fn main() -> Result<(), String> {
     println!("cargo::rustc-check-cfg=cfg(kernel_support, values(\"avx512\"))");
 
     println!("cargo:rerun-if-changed=src/simd/f16.c");
+    println!("cargo:rerun-if-changed=src/simd/bf16.c");
     println!("cargo:rerun-if-changed=src/simd/dist_table.c");
 
     // Important: we don't use `cfg!(target_arch)` here because that is the target_arch
@@ -37,13 +38,16 @@ fn main() -> Result<(), String> {
     if target_arch == "aarch64" && target_os == "macos" {
         // Build a version with NEON
         build_f16_with_flags("neon", &["-mtune=apple-m1"]).unwrap();
+        build_bf16_with_flags("neon", &["-mtune=apple-m1"]).unwrap();
     } else if target_arch == "aarch64" && target_os == "ios" {
         // Build version with NEON
         // A13 bionic is the earliest supported iOS SOC
         build_f16_with_flags("neon", &["-mtune=apple-a13"]).unwrap();
+        build_bf16_with_flags("neon", &["-mtune=apple-a13"]).unwrap();
     } else if target_arch == "aarch64" && (target_os == "linux" || target_os == "android") {
         // Build a version with NEON
         build_f16_with_flags("neon", &["-march=armv8.2-a+fp16"]).unwrap();
+        build_bf16_with_flags("neon", &["-march=armv8.2-a+fp16"]).unwrap();
     } else if target_arch == "x86_64" {
         // Build a version with AVX512
         if let Err(err) = build_f16_with_flags("avx512", &["-march=sapphirerapids", "-mavx512fp16"])
@@ -59,6 +63,17 @@ fn main() -> Result<(), String> {
             // generated the AVX512 version of the f16 kernels.
             println!("cargo:rustc-cfg=kernel_support=\"avx512\"");
         };
+        // Build AVX-512 bf16 kernels (sapphirerapids has native vdpbf16ps)
+        if let Err(err) =
+            build_bf16_with_flags("avx512", &["-march=sapphirerapids", "-mavx512fp16"])
+        {
+            println!(
+                "cargo:warning=Skipping build of AVX-512 bf16 kernels. Error: {}",
+                err
+            );
+        } else {
+            println!("cargo:rustc-cfg=kernel_support=\"avx512\"");
+        };
         if let Err(err) = build_dist_table_with_flags("avx512", &["-march=native"]) {
             println!(
                 "cargo:warning=Skipping build of AVX-512 dist_table. Error: {}",
@@ -77,11 +92,20 @@ fn main() -> Result<(), String> {
                 err
             ));
         };
+        // Build AVX2 bf16 kernels (bf16-to-f32 is just a shift, auto-vectorizes well)
+        if let Err(err) = build_bf16_with_flags("avx2", &["-march=haswell"]) {
+            return Err(format!(
+                "Unable to build AVX2 bf16 kernels.  Received error: {}",
+                err
+            ));
+        };
         // There is no SSE instruction set for f16 -> f32 float conversion
     } else if target_arch == "loongarch64" {
         // Build a version with LSX and LASX
         build_f16_with_flags("lsx", &["-mlsx"]).unwrap();
         build_f16_with_flags("lasx", &["-mlasx"]).unwrap();
+        build_bf16_with_flags("lsx", &["-mlsx"]).unwrap();
+        build_bf16_with_flags("lasx", &["-mlasx"]).unwrap();
     } else {
         // Only error if fp16kernels was explicitly requested on unsupported platform.
         // This allows builds on iOS, Android, etc. when the feature is disabled.
@@ -128,6 +152,32 @@ fn build_f16_with_flags(suffix: &str, flags: &[&str]) -> Result<(), cc::Error> {
     builder.try_compile(&format!("f16_{}", suffix))
 }
 
+fn build_bf16_with_flags(suffix: &str, flags: &[&str]) -> Result<(), cc::Error> {
+    if cfg!(not(feature = "fp16kernels")) {
+        println!(
+            "cargo:warning=fp16kernels feature is not enabled, skipping build of bf16 kernels"
+        );
+        return Ok(());
+    }
+
+    let mut builder = cc::Build::new();
+    builder
+        .std("c17")
+        .file("src/simd/bf16.c")
+        .flag("-ffast-math")
+        .flag("-funroll-loops")
+        .flag("-O3")
+        .flag("-Wall")
+        .flag("-Wextra")
+        .flag(format!("-DSUFFIX=_{}", suffix).as_str());
+
+    for flag in flags {
+        builder.flag(flag);
+    }
+
+    builder.try_compile(&format!("bf16_{}", suffix))
+}
+
 fn build_dist_table_with_flags(suffix: &str, flags: &[&str]) -> Result<(), cc::Error> {
     let mut builder = cc::Build::new();
     builder
diff --git a/rust/lance-linalg/src/distance.rs b/rust/lance-linalg/src/distance.rs
index 84c81fe85ed..9dd57edb3f9 100644
--- a/rust/lance-linalg/src/distance.rs
+++ b/rust/lance-linalg/src/distance.rs
@@ -17,9 +17,12 @@ use arrow_array::{Array, ArrowPrimitiveType, FixedSizeListArray, Float32Array, L
 use arrow_schema::{ArrowError, DataType};
 
 pub mod cosine;
+pub mod cosine_u8;
 pub mod dot;
+pub mod dot_u8;
 pub mod hamming;
 pub mod l2;
+pub mod l2_u8;
 pub mod norm_l2;
 
 pub use cosine::*;
diff --git a/rust/lance-linalg/src/distance/cosine.rs b/rust/lance-linalg/src/distance/cosine.rs
index 89a86208793..be5bf436344 100644
--- a/rust/lance-linalg/src/distance/cosine.rs
+++ b/rust/lance-linalg/src/distance/cosine.rs
@@ -65,9 +65,72 @@ pub trait Cosine: Dot + Normalize {
     }
 }
 
-impl Cosine for u8 {}
+impl Cosine for u8 {
+    #[inline]
+    fn cosine(x: &[Self], other: &[Self]) -> f32 {
+        super::cosine_u8::cosine_u8(x, other)
+    }
+}
 
-impl Cosine for bf16 {}
+#[cfg(feature = "fp16kernels")]
+mod bf16_kernel {
+    use half::bf16;
+
+    // These are the `cosine_bf16` function in bf16.c. Our build.rs script compiles
+    // a version of this file for each SIMD level with different suffixes.
+    unsafe extern "C" {
+        #[cfg(target_arch = "aarch64")]
+        pub fn cosine_bf16_neon(x: *const bf16, x_norm: f32, y: *const bf16, dimension: u32)
+        -> f32;
+        #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
+        pub fn cosine_bf16_avx512(
+            x: *const bf16,
+            x_norm: f32,
+            y: *const bf16,
+            dimension: u32,
+        ) -> f32;
+        #[cfg(target_arch = "x86_64")]
+        pub fn cosine_bf16_avx2(x: *const bf16, x_norm: f32, y: *const bf16, dimension: u32)
+        -> f32;
+        #[cfg(target_arch = "loongarch64")]
+        pub fn cosine_bf16_lsx(x: *const bf16, x_norm: f32, y: *const bf16, dimension: u32) -> f32;
+        #[cfg(target_arch = "loongarch64")]
+        pub fn cosine_bf16_lasx(x: *const bf16, x_norm: f32, y: *const bf16, dimension: u32)
+        -> f32;
+    }
+}
+
+impl Cosine for bf16 {
+    fn cosine_fast(x: &[Self], x_norm: f32, y: &[Self]) -> f32 {
+        match *SIMD_SUPPORT {
+            #[cfg(all(feature = "fp16kernels", target_arch = "aarch64"))]
+            SimdSupport::Neon => unsafe {
+                bf16_kernel::cosine_bf16_neon(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32)
+            },
+            #[cfg(all(
+                feature = "fp16kernels",
+                kernel_support = "avx512",
+                target_arch = "x86_64"
+            ))]
+            SimdSupport::Avx512FP16 => unsafe {
+                bf16_kernel::cosine_bf16_avx512(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32)
+            },
+            #[cfg(all(feature = "fp16kernels", target_arch = "x86_64"))]
+            SimdSupport::Avx2 | SimdSupport::Avx512 => unsafe {
+                bf16_kernel::cosine_bf16_avx2(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32)
+            },
+            #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))]
+            SimdSupport::Lasx => unsafe {
+                bf16_kernel::cosine_bf16_lasx(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32)
+            },
+            #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))]
+            SimdSupport::Lsx => unsafe {
+                bf16_kernel::cosine_bf16_lsx(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32)
+            },
+            _ => cosine_scalar(x, x_norm, y),
+        }
+    }
+}
 
 #[cfg(feature = "fp16kernels")]
 mod kernel {
@@ -225,10 +288,49 @@ impl Cosine for f32 {
     }
 }
 
-impl Cosine for f64 {}
+impl Cosine for f64 {
+    #[inline]
+    fn cosine_fast(x: &[Self], x_norm: f32, y: &[Self]) -> f32 {
+        use crate::simd::f64::{f64x4, f64x8};
+        use crate::simd::{FloatSimd, SIMD};
+
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+        let mut y_norm8 = f64x8::zeros();
+        let mut xy8 = f64x8::zeros();
+        for i in (0..unrolled_len).step_by(8) {
+            unsafe {
+                let xv = f64x8::load_unaligned(x.as_ptr().add(i));
+                let yv = f64x8::load_unaligned(y.as_ptr().add(i));
+                xy8.multiply_add(xv, yv);
+                y_norm8.multiply_add(yv, yv);
+            }
+        }
+        let aligned_len = dim / 4 * 4;
+        let mut y_norm4 = f64x4::zeros();
+        let mut xy4 = f64x4::zeros();
+        for i in (unrolled_len..aligned_len).step_by(4) {
+            unsafe {
+                let xv = f64x4::load_unaligned(x.as_ptr().add(i));
+                let yv = f64x4::load_unaligned(y.as_ptr().add(i));
+                xy4.multiply_add(xv, yv);
+                y_norm4.multiply_add(yv, yv);
+            }
+        }
+        let tail_y_norm: Self = y[aligned_len..].iter().map(|&v| v * v).sum();
+        let tail_xy: Self = x[aligned_len..]
+            .iter()
+            .zip(y[aligned_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        let y_norm_sq = (y_norm8.reduce_sum() + y_norm4.reduce_sum() + tail_y_norm) as f32;
+        let xy = (xy8.reduce_sum() + xy4.reduce_sum() + tail_xy) as f32;
+        1.0 - xy / x_norm / y_norm_sq.sqrt()
+    }
+}
 
 /// Fallback non-SIMD implementation
-#[allow(dead_code)] // Does not fallback on aarch64.
 #[inline]
 fn cosine_scalar<T: Dot>(x: &[T], x_norm: f32, y: &[T]) -> f32 {
     let y_sq = dot(y, y);
diff --git a/rust/lance-linalg/src/distance/cosine_u8.rs b/rust/lance-linalg/src/distance/cosine_u8.rs
new file mode 100644
index 00000000000..b2d06d35c31
--- /dev/null
+++ b/rust/lance-linalg/src/distance/cosine_u8.rs
@@ -0,0 +1,369 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Unsigned int8 cosine distance with runtime-dispatched SIMD backends.
+//!
+//! Computes `1 - dot(a,b) / (‖a‖ × ‖b‖)` for u8 slices in a single
+//! pass over the data. The fused kernel maintains three accumulators
+//! simultaneously — `Σ(a·b)`, `Σ(a²)`, `Σ(b²)` — so memory is only
+//! traversed once instead of 2-3 times (norm + dot).
+//!
+//! Backends (selected at runtime, best available wins):
+//!   1. scalar     — portable reference, also used for tails
+//!   2. avx2       — zero-extend u8→i16, triple VPMADDWD, 32 elements/iter
+//!   3. avx512vnni — same with VPDPWSSD accumulation, 64 elements/iter
+
+use std::sync::OnceLock;
+
+/// Intermediate results from the fused u8 cosine kernel: (dot_ab, norm_a², norm_b²).
+///
+/// Separated from the final normalization so SIMD backends can be tested
+/// for exact integer equality before the f32 division.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct CosineAccumulators {
+    pub dot_ab: u32,
+    pub norm_a_sq: u32,
+    pub norm_b_sq: u32,
+}
+
+/// Portable scalar fused cosine accumulation.
+#[inline]
+pub fn cosine_u8_accum_scalar(a: &[u8], b: &[u8]) -> CosineAccumulators {
+    debug_assert_eq!(a.len(), b.len());
+    let (mut dot_ab, mut norm_a_sq, mut norm_b_sq) = (0u32, 0u32, 0u32);
+    for (&x, &y) in a.iter().zip(b.iter()) {
+        let (xu, yu) = (x as u32, y as u32);
+        dot_ab += xu * yu;
+        norm_a_sq += xu * xu;
+        norm_b_sq += yu * yu;
+    }
+    CosineAccumulators {
+        dot_ab,
+        norm_a_sq,
+        norm_b_sq,
+    }
+}
+
+/// Convert accumulators to cosine distance: `1 - dot / (‖a‖ × ‖b‖)`.
+#[inline]
+fn normalize(acc: CosineAccumulators) -> f32 {
+    let na = (acc.norm_a_sq as f32).sqrt();
+    let nb = (acc.norm_b_sq as f32).sqrt();
+    let denom = na * nb;
+    if denom == 0.0 {
+        // Both zero-norm → identical → distance 0.
+        // One zero-norm → undefined, but 0 is a safe sentinel.
+        return 0.0;
+    }
+    1.0 - acc.dot_ab as f32 / denom
+}
+
+/// Portable scalar u8 cosine distance.
+#[inline]
+pub fn cosine_u8_scalar(a: &[u8], b: &[u8]) -> f32 {
+    normalize(cosine_u8_accum_scalar(a, b))
+}
+
+#[cfg(target_arch = "x86_64")]
+mod x86 {
+    use super::CosineAccumulators;
+    use std::arch::x86_64::*;
+
+    /// Horizontal sum of all 8 × i32 lanes in a __m256i.
+    #[inline(always)]
+    unsafe fn hsum_epi32_avx2(v: __m256i) -> u32 {
+        let lo128 = _mm256_castsi256_si128(v);
+        let hi128 = _mm256_extracti128_si256(v, 1);
+        let mut sum128 = _mm_add_epi32(lo128, hi128);
+        sum128 = _mm_hadd_epi32(sum128, sum128);
+        sum128 = _mm_hadd_epi32(sum128, sum128);
+        _mm_cvtsi128_si32(sum128) as u32
+    }
+
+    /// AVX2 fused cosine: three VPMADDWD products per half, 32 elements/iter.
+    #[target_feature(enable = "avx2")]
+    pub unsafe fn cosine_u8_accum_avx2(a: &[u8], b: &[u8]) -> CosineAccumulators {
+        debug_assert_eq!(a.len(), b.len());
+        let n = a.len();
+        let mut acc_dot = _mm256_setzero_si256();
+        let mut acc_na = _mm256_setzero_si256();
+        let mut acc_nb = _mm256_setzero_si256();
+        let mut i = 0usize;
+
+        while i + 32 <= n {
+            let av = _mm256_loadu_si256(a.as_ptr().add(i) as *const __m256i);
+            let bv = _mm256_loadu_si256(b.as_ptr().add(i) as *const __m256i);
+
+            // Zero-extend each 128-bit half to 16 × i16.
+            let a_lo = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(av));
+            let a_hi = _mm256_cvtepu8_epi16(_mm256_extracti128_si256(av, 1));
+            let b_lo = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(bv));
+            let b_hi = _mm256_cvtepu8_epi16(_mm256_extracti128_si256(bv, 1));
+
+            // VPMADDWD: pairwise multiply i16 and accumulate pairs into i32.
+            acc_dot = _mm256_add_epi32(acc_dot, _mm256_madd_epi16(a_lo, b_lo));
+            acc_dot = _mm256_add_epi32(acc_dot, _mm256_madd_epi16(a_hi, b_hi));
+            acc_na = _mm256_add_epi32(acc_na, _mm256_madd_epi16(a_lo, a_lo));
+            acc_na = _mm256_add_epi32(acc_na, _mm256_madd_epi16(a_hi, a_hi));
+            acc_nb = _mm256_add_epi32(acc_nb, _mm256_madd_epi16(b_lo, b_lo));
+            acc_nb = _mm256_add_epi32(acc_nb, _mm256_madd_epi16(b_hi, b_hi));
+            i += 32;
+        }
+
+        let mut dot_ab = hsum_epi32_avx2(acc_dot);
+        let mut norm_a_sq = hsum_epi32_avx2(acc_na);
+        let mut norm_b_sq = hsum_epi32_avx2(acc_nb);
+
+        // Scalar tail
+        while i < n {
+            let (xu, yu) = (a[i] as u32, b[i] as u32);
+            dot_ab += xu * yu;
+            norm_a_sq += xu * xu;
+            norm_b_sq += yu * yu;
+            i += 1;
+        }
+
+        CosineAccumulators {
+            dot_ab,
+            norm_a_sq,
+            norm_b_sq,
+        }
+    }
+
+    /// AVX-512 VNNI fused cosine: VPDPWSSD for each product, 64 elements/iter.
+    #[target_feature(enable = "avx512f,avx512bw,avx512vnni")]
+    pub unsafe fn cosine_u8_accum_avx512_vnni(a: &[u8], b: &[u8]) -> CosineAccumulators {
+        debug_assert_eq!(a.len(), b.len());
+        let n = a.len();
+        let zeros = _mm512_setzero_si512();
+        let mut acc_dot = _mm512_setzero_si512();
+        let mut acc_na = _mm512_setzero_si512();
+        let mut acc_nb = _mm512_setzero_si512();
+        let mut i = 0usize;
+
+        while i + 64 <= n {
+            let av = _mm512_loadu_si512(a.as_ptr().add(i) as *const __m512i);
+            let bv = _mm512_loadu_si512(b.as_ptr().add(i) as *const __m512i);
+
+            // Zero-extend u8→i16 via interleave with zeros.
+            let a_lo = _mm512_unpacklo_epi8(av, zeros);
+            let a_hi = _mm512_unpackhi_epi8(av, zeros);
+            let b_lo = _mm512_unpacklo_epi8(bv, zeros);
+            let b_hi = _mm512_unpackhi_epi8(bv, zeros);
+
+            // VPDPWSSD: signed i16 multiply-add into i32 accumulator.
+            acc_dot = _mm512_dpwssd_epi32(acc_dot, a_lo, b_lo);
+            acc_dot = _mm512_dpwssd_epi32(acc_dot, a_hi, b_hi);
+            acc_na = _mm512_dpwssd_epi32(acc_na, a_lo, a_lo);
+            acc_na = _mm512_dpwssd_epi32(acc_na, a_hi, a_hi);
+            acc_nb = _mm512_dpwssd_epi32(acc_nb, b_lo, b_lo);
+            acc_nb = _mm512_dpwssd_epi32(acc_nb, b_hi, b_hi);
+            i += 64;
+        }
+
+        let mut dot_ab = _mm512_reduce_add_epi32(acc_dot) as u32;
+        let mut norm_a_sq = _mm512_reduce_add_epi32(acc_na) as u32;
+        let mut norm_b_sq = _mm512_reduce_add_epi32(acc_nb) as u32;
+
+        // Scalar tail
+        while i < n {
+            let (xu, yu) = (a[i] as u32, b[i] as u32);
+            dot_ab += xu * yu;
+            norm_a_sq += xu * xu;
+            norm_b_sq += yu * yu;
+            i += 1;
+        }
+
+        CosineAccumulators {
+            dot_ab,
+            norm_a_sq,
+            norm_b_sq,
+        }
+    }
+}
+
+type CosineU8AccumFn = fn(&[u8], &[u8]) -> CosineAccumulators;
+
+static DISPATCH: OnceLock<CosineU8AccumFn> = OnceLock::new();
+
+fn select_backend() -> CosineU8AccumFn {
+    #[cfg(target_arch = "x86_64")]
+    {
+        if is_x86_feature_detected!("avx512f")
+            && is_x86_feature_detected!("avx512bw")
+            && is_x86_feature_detected!("avx512vnni")
+        {
+            return |a, b| unsafe { x86::cosine_u8_accum_avx512_vnni(a, b) };
+        }
+
+        if is_x86_feature_detected!("avx2") {
+            return |a, b| unsafe { x86::cosine_u8_accum_avx2(a, b) };
+        }
+    }
+
+    cosine_u8_accum_scalar
+}
+
+/// Dispatched fused u8 cosine accumulation.
+#[inline]
+fn cosine_u8_accum(a: &[u8], b: &[u8]) -> CosineAccumulators {
+    (DISPATCH.get_or_init(select_backend))(a, b)
+}
+
+/// Dispatched u8 cosine distance, selecting the best available SIMD backend.
+///
+/// Returns `1 - dot(a,b) / (‖a‖ × ‖b‖)` computed in a single pass.
+#[inline]
+pub fn cosine_u8(a: &[u8], b: &[u8]) -> f32 {
+    normalize(cosine_u8_accum(a, b))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn fill_random(buf: &mut [u8], seed: &mut u32) {
+        for slot in buf.iter_mut() {
+            *seed = seed.wrapping_mul(1103515245).wrapping_add(12345);
+            *slot = (*seed >> 16) as u8;
+        }
+    }
+
+    const SIZES: &[usize] = &[
+        0, 1, 7, 15, 16, 31, 32, 33, 63, 64, 65, 127, 128, 255, 256, 1024, 4096, 4097,
+    ];
+
+    /// Verify SIMD backends produce identical integer accumulators to scalar.
+    fn check_all_backends_accum(a: &[u8], b: &[u8], case: &str) {
+        let reference = cosine_u8_accum_scalar(a, b);
+
+        #[cfg(target_arch = "x86_64")]
+        {
+            if is_x86_feature_detected!("avx2") {
+                let got = unsafe { x86::cosine_u8_accum_avx2(a, b) };
+                assert_eq!(got, reference, "avx2 [{case}] n={}", a.len());
+            }
+
+            if is_x86_feature_detected!("avx512f")
+                && is_x86_feature_detected!("avx512bw")
+                && is_x86_feature_detected!("avx512vnni")
+            {
+                let got = unsafe { x86::cosine_u8_accum_avx512_vnni(a, b) };
+                assert_eq!(got, reference, "avx512_vnni [{case}] n={}", a.len());
+            }
+        }
+
+        let dispatched = cosine_u8_accum(a, b);
+        assert_eq!(dispatched, reference, "dispatch [{case}] n={}", a.len());
+    }
+
+    #[test]
+    fn random_inputs_across_sizes_and_seeds() {
+        let mut a = vec![0u8; 4097];
+        let mut b = vec![0u8; 4097];
+
+        for seed_idx in 0..4u32 {
+            let mut seed = 0xC0FFEE_u32.wrapping_add(seed_idx.wrapping_mul(7919));
+            for &n in SIZES {
+                fill_random(&mut a[..n], &mut seed);
+                fill_random(&mut b[..n], &mut seed);
+                check_all_backends_accum(&a[..n], &b[..n], "random");
+            }
+        }
+    }
+
+    #[test]
+    fn boundary_values() {
+        let mut a = vec![0u8; 4097];
+        let mut b = vec![0u8; 4097];
+
+        for &n in SIZES {
+            a[..n].fill(u8::MAX);
+            b[..n].fill(u8::MAX);
+            check_all_backends_accum(&a[..n], &b[..n], "max-max");
+
+            a[..n].fill(u8::MAX);
+            b[..n].fill(0);
+            check_all_backends_accum(&a[..n], &b[..n], "max-0");
+
+            a[..n].fill(0);
+            b[..n].fill(u8::MAX);
+            check_all_backends_accum(&a[..n], &b[..n], "0-max");
+
+            a[..n].fill(0);
+            b[..n].fill(0);
+            check_all_backends_accum(&a[..n], &b[..n], "0-0");
+
+            for i in 0..n {
+                a[i] = if i & 1 == 0 { 0 } else { u8::MAX };
+                b[i] = if i & 1 == 0 { u8::MAX } else { 0 };
+            }
+            check_all_backends_accum(&a[..n], &b[..n], "alt 0/max");
+        }
+    }
+
+    #[test]
+    fn one_sided_zeros() {
+        let mut a = vec![0u8; 4097];
+        let mut b = vec![0u8; 4097];
+
+        for &n in SIZES {
+            let mut seed = 0xDEAD_BEEF_u32;
+            fill_random(&mut a[..n], &mut seed);
+            b[..n].fill(0);
+            check_all_backends_accum(&a[..n], &b[..n], "b=0");
+
+            a[..n].fill(0);
+            fill_random(&mut b[..n], &mut seed);
+            check_all_backends_accum(&a[..n], &b[..n], "a=0");
+        }
+    }
+
+    #[test]
+    fn cosine_known_values() {
+        // Identical vectors → distance 0
+        let v = [10u8, 20, 30, 40];
+        assert_eq!(cosine_u8(&v, &v), 0.0);
+
+        // Orthogonal-ish: one vector all in first half, other in second half
+        let a = [255u8, 255, 0, 0];
+        let b = [0u8, 0, 255, 255];
+        assert_eq!(cosine_u8(&a, &b), 1.0); // dot=0 → distance=1
+
+        // Zero vectors → distance 0 (by convention)
+        assert_eq!(cosine_u8(&[0, 0], &[0, 0]), 0.0);
+        assert_eq!(cosine_u8(&[0, 0], &[1, 2]), 0.0);
+    }
+
+    #[test]
+    fn cosine_distance_symmetry() {
+        let mut a = vec![0u8; 256];
+        let mut b = vec![0u8; 256];
+        let mut seed = 0xBEEF_u32;
+        fill_random(&mut a, &mut seed);
+        fill_random(&mut b, &mut seed);
+
+        let d1 = cosine_u8(&a, &b);
+        let d2 = cosine_u8(&b, &a);
+        assert_eq!(d1, d2);
+    }
+
+    #[test]
+    fn cosine_distance_range() {
+        // Cosine distance should be in [0, 1] for non-negative u8 inputs
+        // (all u8 values are ≥ 0, so dot product is always ≥ 0).
+        let mut a = vec![0u8; 1024];
+        let mut b = vec![0u8; 1024];
+
+        for seed_idx in 0..8u32 {
+            let mut seed = 0xABCD_u32.wrapping_add(seed_idx.wrapping_mul(31));
+            fill_random(&mut a, &mut seed);
+            fill_random(&mut b, &mut seed);
+            let d = cosine_u8(&a, &b);
+            assert!(
+                (0.0..=1.0).contains(&d),
+                "cosine distance {d} out of [0,1] range"
+            );
+        }
+    }
+}
diff --git a/rust/lance-linalg/src/distance/dot.rs b/rust/lance-linalg/src/distance/dot.rs
index 2951bc4643a..0dfa2178b9d 100644
--- a/rust/lance-linalg/src/distance/dot.rs
+++ b/rust/lance-linalg/src/distance/dot.rs
@@ -75,10 +75,56 @@ pub trait Dot: Num {
     fn dot(x: &[Self], y: &[Self]) -> f32;
 }
 
+#[cfg(feature = "fp16kernels")]
+mod bf16_kernel {
+    use half::bf16;
+
+    // These are the `dot_bf16` function in bf16.c. Our build.rs script compiles
+    // a version of this file for each SIMD level with different suffixes.
+    unsafe extern "C" {
+        #[cfg(target_arch = "aarch64")]
+        pub fn dot_bf16_neon(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
+        #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
+        pub fn dot_bf16_avx512(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
+        #[cfg(target_arch = "x86_64")]
+        pub fn dot_bf16_avx2(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
+        #[cfg(target_arch = "loongarch64")]
+        pub fn dot_bf16_lsx(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
+        #[cfg(target_arch = "loongarch64")]
+        pub fn dot_bf16_lasx(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
+    }
+}
+
 impl Dot for bf16 {
     #[inline]
     fn dot(x: &[Self], y: &[Self]) -> f32 {
-        dot_scalar::<Self, f32, 32>(x, y)
+        match *SIMD_SUPPORT {
+            #[cfg(all(feature = "fp16kernels", target_arch = "aarch64"))]
+            SimdSupport::Neon => unsafe {
+                bf16_kernel::dot_bf16_neon(x.as_ptr(), y.as_ptr(), x.len() as u32)
+            },
+            #[cfg(all(
+                feature = "fp16kernels",
+                kernel_support = "avx512",
+                target_arch = "x86_64"
+            ))]
+            SimdSupport::Avx512FP16 => unsafe {
+                bf16_kernel::dot_bf16_avx512(x.as_ptr(), y.as_ptr(), x.len() as u32)
+            },
+            #[cfg(all(feature = "fp16kernels", target_arch = "x86_64"))]
+            SimdSupport::Avx2 | SimdSupport::Avx512 => unsafe {
+                bf16_kernel::dot_bf16_avx2(x.as_ptr(), y.as_ptr(), x.len() as u32)
+            },
+            #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))]
+            SimdSupport::Lasx => unsafe {
+                bf16_kernel::dot_bf16_lasx(x.as_ptr(), y.as_ptr(), x.len() as u32)
+            },
+            #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))]
+            SimdSupport::Lsx => unsafe {
+                bf16_kernel::dot_bf16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32)
+            },
+            _ => dot_scalar::<Self, f32, 32>(x, y),
+        }
     }
 }
 
@@ -145,18 +191,51 @@ impl Dot for f32 {
 impl Dot for f64 {
     #[inline]
     fn dot(x: &[Self], y: &[Self]) -> f32 {
-        dot_scalar::<Self, Self, 8>(x, y) as f32
+        dot_f64_simd(x, y)
     }
 }
 
+/// Explicit SIMD dot product for f64.
+#[inline]
+fn dot_f64_simd(x: &[f64], y: &[f64]) -> f32 {
+    use crate::simd::f64::{f64x4, f64x8};
+    use crate::simd::{FloatSimd, SIMD};
+
+    let dim = x.len();
+    let unrolled_len = dim / 8 * 8;
+
+    let mut acc8 = f64x8::zeros();
+    for i in (0..unrolled_len).step_by(8) {
+        unsafe {
+            let a = f64x8::load_unaligned(x.as_ptr().add(i));
+            let b = f64x8::load_unaligned(y.as_ptr().add(i));
+            acc8.multiply_add(a, b);
+        }
+    }
+
+    let aligned_len = dim / 4 * 4;
+    let mut acc4 = f64x4::zeros();
+    for i in (unrolled_len..aligned_len).step_by(4) {
+        unsafe {
+            let a = f64x4::load_unaligned(x.as_ptr().add(i));
+            let b = f64x4::load_unaligned(y.as_ptr().add(i));
+            acc4.multiply_add(a, b);
+        }
+    }
+
+    let tail: f64 = x[aligned_len..]
+        .iter()
+        .zip(y[aligned_len..].iter())
+        .map(|(&a, &b)| a * b)
+        .sum();
+
+    (acc8.reduce_sum() + acc4.reduce_sum() + tail) as f32
+}
+
 impl Dot for u8 {
     #[inline]
     fn dot(x: &[Self], y: &[Self]) -> f32 {
-        // TODO: this is not optimized for auto vectorization yet.
-        x.iter()
-            .zip(y.iter())
-            .map(|(&x_i, &y_i)| x_i as u32 * y_i as u32)
-            .sum::<u32>() as f32
+        super::dot_u8::dot_u8(x, y) as f32
     }
 }
 
diff --git a/rust/lance-linalg/src/distance/dot_u8.rs b/rust/lance-linalg/src/distance/dot_u8.rs
new file mode 100644
index 00000000000..de5522cddfe
--- /dev/null
+++ b/rust/lance-linalg/src/distance/dot_u8.rs
@@ -0,0 +1,255 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Unsigned int8 dot product with runtime-dispatched SIMD backends.
+//!
+//! Used by Scalar Quantization (SQ) distance computation. SQ stores each
+//! vector dimension as a u8 after linearly mapping [min, max] → [0, 255].
+//! Distance computation between SQ-encoded vectors reduces to a u8 × u8
+//! dot product plus precomputed per-vector scalar terms.
+//!
+//! Backends (selected at runtime, best available wins):
+//!   1. scalar     — portable reference, also used for tails
+//!   2. avx2       — VPMADDWD on u16-widened halves, 32 elements/iter
+//!   3. avx512vnni — VPDPBUSD with XOR-0x80 bias trick, 64 elements/iter
+//!
+//! ## The VNNI bias trick
+//!
+//! VPDPBUSD expects one unsigned and one signed operand, but SQ vectors
+//! are u8 × u8. We bias `b` into the signed domain via XOR 0x80 (equivalent
+//! to subtracting 128 when reinterpreted as i8), feed `a` directly as
+//! unsigned, and correct by adding 128·Σa at the end:
+//!
+//!   DPBUSD(a, b ⊕ 0x80) = Σ a·(b − 128) = Σ a·b − 128·Σa
+//!
+//! The Σa term uses VPSADBW, which dispatches to port 5 while VPDPBUSD
+//! runs on port 0 on Intel. The two instructions execute in parallel,
+//! making the correction effectively free.
+
+use std::sync::OnceLock;
+
+/// Portable scalar u8 dot product, also used for SIMD tail elements.
+#[inline]
+pub fn dot_u8_scalar(a: &[u8], b: &[u8]) -> u32 {
+    debug_assert_eq!(a.len(), b.len());
+    a.iter()
+        .zip(b.iter())
+        .map(|(&x, &y)| x as u32 * y as u32)
+        .sum()
+}
+
+#[cfg(target_arch = "x86_64")]
+mod x86 {
+    use std::arch::x86_64::*;
+
+    /// AVX2 path: zero-extend u8→u16, then VPMADDWD. 32 elements/iter.
+    #[target_feature(enable = "avx2")]
+    pub unsafe fn dot_u8_avx2(a: &[u8], b: &[u8]) -> u32 {
+        debug_assert_eq!(a.len(), b.len());
+        let n = a.len();
+        let mut acc = _mm256_setzero_si256();
+        let mut i = 0usize;
+
+        while i + 32 <= n {
+            let av = _mm256_loadu_si256(a.as_ptr().add(i) as *const __m256i);
+            let bv = _mm256_loadu_si256(b.as_ptr().add(i) as *const __m256i);
+
+            // Zero-extend each 128-bit half to 16 × u16. Values ≤ 255 fit
+            // in i16 as positive, so VPMADDWD gives correct results.
+            let a_lo = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(av));
+            let a_hi = _mm256_cvtepu8_epi16(_mm256_extracti128_si256(av, 1));
+            let b_lo = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(bv));
+            let b_hi = _mm256_cvtepu8_epi16(_mm256_extracti128_si256(bv, 1));
+
+            acc = _mm256_add_epi32(acc, _mm256_madd_epi16(a_lo, b_lo));
+            acc = _mm256_add_epi32(acc, _mm256_madd_epi16(a_hi, b_hi));
+            i += 32;
+        }
+
+        let lo128 = _mm256_castsi256_si128(acc);
+        let hi128 = _mm256_extracti128_si256(acc, 1);
+        let mut sum128 = _mm_add_epi32(lo128, hi128);
+        sum128 = _mm_hadd_epi32(sum128, sum128);
+        sum128 = _mm_hadd_epi32(sum128, sum128);
+        let mut result = _mm_cvtsi128_si32(sum128) as u32;
+
+        while i < n {
+            result += a[i] as u32 * b[i] as u32;
+            i += 1;
+        }
+        result
+    }
+
+    /// AVX-512 VNNI path (Ice Lake+, Zen 4+). 64 elements/iter.
+    ///
+    /// VPDPBUSD expects (unsigned, signed) operands but SQ stores u8×u8.
+    /// We XOR b with 0x80 to map it to i8, then correct: result + 128·Σa.
+    /// The Σa term (VPSADBW, port 5) runs in parallel with VPDPBUSD (port 0).
+    #[target_feature(enable = "avx512f,avx512bw,avx512vnni")]
+    pub unsafe fn dot_u8_avx512_vnni(a: &[u8], b: &[u8]) -> u32 {
+        debug_assert_eq!(a.len(), b.len());
+        let n = a.len();
+
+        let mut acc_dot = _mm512_setzero_si512();
+        let mut acc_suma = _mm512_setzero_si512();
+        let sign_flip = _mm512_set1_epi8(0x80u8 as i8);
+        let zeros = _mm512_setzero_si512();
+        let mut i = 0usize;
+
+        while i + 64 <= n {
+            let av = _mm512_loadu_si512(a.as_ptr().add(i) as *const __m512i);
+            let bv = _mm512_loadu_si512(b.as_ptr().add(i) as *const __m512i);
+            let b_biased = _mm512_xor_si512(bv, sign_flip);
+            acc_dot = _mm512_dpbusd_epi32(acc_dot, av, b_biased);
+            acc_suma = _mm512_add_epi64(acc_suma, _mm512_sad_epu8(av, zeros));
+            i += 64;
+        }
+
+        let biased_dot = _mm512_reduce_add_epi32(acc_dot);
+        let sum_a = _mm512_reduce_add_epi64(acc_suma);
+        let mut result = (biased_dot as i64 + 128 * sum_a) as u32;
+
+        while i < n {
+            result += a[i] as u32 * b[i] as u32;
+            i += 1;
+        }
+        result
+    }
+}
+
+type DotU8Fn = fn(&[u8], &[u8]) -> u32;
+
+static DISPATCH: OnceLock<DotU8Fn> = OnceLock::new();
+
+fn select_backend() -> DotU8Fn {
+    #[cfg(target_arch = "x86_64")]
+    {
+        if is_x86_feature_detected!("avx512f")
+            && is_x86_feature_detected!("avx512bw")
+            && is_x86_feature_detected!("avx512vnni")
+        {
+            return |a, b| unsafe { x86::dot_u8_avx512_vnni(a, b) };
+        }
+
+        if is_x86_feature_detected!("avx2") {
+            return |a, b| unsafe { x86::dot_u8_avx2(a, b) };
+        }
+    }
+
+    dot_u8_scalar
+}
+
+/// Dispatched u8 dot product, selecting the best available SIMD backend.
+#[inline]
+pub fn dot_u8(a: &[u8], b: &[u8]) -> u32 {
+    (DISPATCH.get_or_init(select_backend))(a, b)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn fill_random(buf: &mut [u8], seed: &mut u32) {
+        for slot in buf.iter_mut() {
+            *seed = seed.wrapping_mul(1103515245).wrapping_add(12345);
+            *slot = (*seed >> 16) as u8;
+        }
+    }
+
+    const SIZES: &[usize] = &[
+        0, 1, 7, 15, 16, 31, 32, 33, 63, 64, 65, 127, 128, 255, 256, 1024, 4096, 4097,
+    ];
+
+    fn check_all_backends(a: &[u8], b: &[u8], case: &str) {
+        let reference = dot_u8_scalar(a, b);
+
+        #[cfg(target_arch = "x86_64")]
+        {
+            if is_x86_feature_detected!("avx2") {
+                let got = unsafe { x86::dot_u8_avx2(a, b) };
+                assert_eq!(got, reference, "avx2 [{case}] n={}", a.len());
+            }
+
+            if is_x86_feature_detected!("avx512f")
+                && is_x86_feature_detected!("avx512bw")
+                && is_x86_feature_detected!("avx512vnni")
+            {
+                let got = unsafe { x86::dot_u8_avx512_vnni(a, b) };
+                assert_eq!(got, reference, "avx512_vnni [{case}] n={}", a.len());
+            }
+        }
+
+        assert_eq!(dot_u8(a, b), reference, "dispatch [{case}] n={}", a.len());
+    }
+
+    #[test]
+    fn random_inputs_across_sizes_and_seeds() {
+        let mut a = vec![0u8; 4097];
+        let mut b = vec![0u8; 4097];
+
+        for seed_idx in 0..4u32 {
+            let mut seed = 0xC0FFEE_u32.wrapping_add(seed_idx.wrapping_mul(7919));
+            for &n in SIZES {
+                fill_random(&mut a[..n], &mut seed);
+                fill_random(&mut b[..n], &mut seed);
+                check_all_backends(&a[..n], &b[..n], "random");
+            }
+        }
+    }
+
+    #[test]
+    fn boundary_values() {
+        let mut a = vec![0u8; 4097];
+        let mut b = vec![0u8; 4097];
+
+        for &n in SIZES {
+            a[..n].fill(u8::MAX);
+            b[..n].fill(u8::MAX);
+            check_all_backends(&a[..n], &b[..n], "max*max");
+
+            a[..n].fill(u8::MAX);
+            b[..n].fill(0);
+            check_all_backends(&a[..n], &b[..n], "max*0");
+
+            a[..n].fill(0);
+            b[..n].fill(u8::MAX);
+            check_all_backends(&a[..n], &b[..n], "0*max");
+
+            for i in 0..n {
+                a[i] = if i & 1 == 0 { 0 } else { u8::MAX };
+                b[i] = if i & 1 == 0 { u8::MAX } else { 0 };
+            }
+            check_all_backends(&a[..n], &b[..n], "alt 0/max");
+        }
+    }
+
+    #[test]
+    fn one_sided_zeros() {
+        let mut a = vec![0u8; 4097];
+        let mut b = vec![0u8; 4097];
+
+        for &n in SIZES {
+            let mut seed = 0xDEAD_BEEF_u32;
+            fill_random(&mut a[..n], &mut seed);
+            b[..n].fill(0);
+            check_all_backends(&a[..n], &b[..n], "b=0");
+
+            a[..n].fill(0);
+            fill_random(&mut b[..n], &mut seed);
+            check_all_backends(&a[..n], &b[..n], "a=0");
+        }
+    }
+
+    #[test]
+    fn all_ones_pattern() {
+        let mut a = vec![0u8; 4097];
+        let mut b = vec![0u8; 4097];
+
+        for &n in SIZES {
+            a[..n].fill(1);
+            b[..n].fill(1);
+            check_all_backends(&a[..n], &b[..n], "1*1");
+            assert_eq!(dot_u8_scalar(&a[..n], &b[..n]), n as u32);
+        }
+    }
+}
diff --git a/rust/lance-linalg/src/distance/l2.rs b/rust/lance-linalg/src/distance/l2.rs
index 7523ef4603f..36855bfee18 100644
--- a/rust/lance-linalg/src/distance/l2.rs
+++ b/rust/lance-linalg/src/distance/l2.rs
@@ -93,15 +93,60 @@ pub fn l2_scalar<
 impl L2 for u8 {
     #[inline]
     fn l2(x: &[Self], y: &[Self]) -> f32 {
-        l2_distance_uint_scalar(x, y)
+        super::l2_u8::l2_u8(x, y) as f32
+    }
+}
+
+#[cfg(feature = "fp16kernels")]
+mod bf16_kernel {
+    use half::bf16;
+
+    // These are the `l2_bf16` function in bf16.c. Our build.rs script compiles
+    // a version of this file for each SIMD level with different suffixes.
+    unsafe extern "C" {
+        #[cfg(target_arch = "aarch64")]
+        pub fn l2_bf16_neon(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
+        #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
+        pub fn l2_bf16_avx512(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
+        #[cfg(target_arch = "x86_64")]
+        pub fn l2_bf16_avx2(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
+        #[cfg(target_arch = "loongarch64")]
+        pub fn l2_bf16_lsx(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
+        #[cfg(target_arch = "loongarch64")]
+        pub fn l2_bf16_lasx(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
     }
 }
 
 impl L2 for bf16 {
     #[inline]
     fn l2(x: &[Self], y: &[Self]) -> f32 {
-        // TODO: add SIMD support
-        l2_scalar::<Self, f32, 16>(x, y)
+        match *SIMD_SUPPORT {
+            #[cfg(all(feature = "fp16kernels", target_arch = "aarch64"))]
+            SimdSupport::Neon => unsafe {
+                bf16_kernel::l2_bf16_neon(x.as_ptr(), y.as_ptr(), x.len() as u32)
+            },
+            #[cfg(all(
+                feature = "fp16kernels",
+                kernel_support = "avx512",
+                target_arch = "x86_64"
+            ))]
+            SimdSupport::Avx512FP16 => unsafe {
+                bf16_kernel::l2_bf16_avx512(x.as_ptr(), y.as_ptr(), x.len() as u32)
+            },
+            #[cfg(all(feature = "fp16kernels", target_arch = "x86_64"))]
+            SimdSupport::Avx2 | SimdSupport::Avx512 => unsafe {
+                bf16_kernel::l2_bf16_avx2(x.as_ptr(), y.as_ptr(), x.len() as u32)
+            },
+            #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))]
+            SimdSupport::Lasx => unsafe {
+                bf16_kernel::l2_bf16_lasx(x.as_ptr(), y.as_ptr(), x.len() as u32)
+            },
+            #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))]
+            SimdSupport::Lsx => unsafe {
+                bf16_kernel::l2_bf16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32)
+            },
+            _ => l2_scalar::<Self, f32, 16>(x, y),
+        }
     }
 }
 
@@ -170,7 +215,151 @@ impl L2 for f32 {
 impl L2 for f64 {
     #[inline]
     fn l2(x: &[Self], y: &[Self]) -> f32 {
-        l2_scalar::<Self, Self, 8>(x, y) as f32
+        l2_f64_simd(x, y)
+    }
+}
+
+/// Explicit SIMD L2 distance for f64.
+#[inline]
+fn l2_f64_simd(x: &[f64], y: &[f64]) -> f32 {
+    use crate::simd::f64::{f64x4, f64x8};
+    use crate::simd::{FloatSimd, SIMD};
+
+    let dim = x.len();
+    let unrolled_len = dim / 8 * 8;
+
+    let mut acc8 = f64x8::zeros();
+    for i in (0..unrolled_len).step_by(8) {
+        unsafe {
+            let a = f64x8::load_unaligned(x.as_ptr().add(i));
+            let b = f64x8::load_unaligned(y.as_ptr().add(i));
+            let diff = a - b;
+            acc8.multiply_add(diff, diff);
+        }
+    }
+
+    let aligned_len = dim / 4 * 4;
+    let mut acc4 = f64x4::zeros();
+    for i in (unrolled_len..aligned_len).step_by(4) {
+        unsafe {
+            let a = f64x4::load_unaligned(x.as_ptr().add(i));
+            let b = f64x4::load_unaligned(y.as_ptr().add(i));
+            let diff = a - b;
+            acc4.multiply_add(diff, diff);
+        }
+    }
+
+    let tail: f64 = x[aligned_len..]
+        .iter()
+        .zip(y[aligned_len..].iter())
+        .map(|(&a, &b)| {
+            let diff = a - b;
+            diff * diff
+        })
+        .sum();
+
+    (acc8.reduce_sum() + acc4.reduce_sum() + tail) as f32
+}
+
+/// Accumulate squared differences for one dimension into per-target results.
+///
+/// Separated into its own function so that LLVM sees `row` and `result`
+/// as non-aliasing via the function signature (`&[f32]` vs `&mut [f32]`),
+/// enabling packed SIMD vectorization (vbroadcastss + vsubps + vfmadd231ps).
+#[inline(never)]
+fn accumulate_l2_dimension(q: f32, row: &[f32], result: &mut [f32]) {
+    for (dist, &target) in result.iter_mut().zip(row.iter()) {
+        let diff = q - target;
+        *dist += diff * diff;
+    }
+}
+
+/// Pre-transposed target vectors for batched L2 distance computation.
+///
+/// Stores targets in SoA layout `[dimension][num_targets]` so the inner
+/// distance loop iterates over targets contiguously. The AoS-to-SoA
+/// transpose is done once at construction; callers should reuse the
+/// struct across many queries to amortize that cost.
+///
+/// **Cache constraint**: this is designed for cases where
+/// `num_targets × dimension × 4` fits in L1 cache (~32 KB), such as PQ
+/// sub-vector codebooks (e.g. 256 centroids × 16 dims = 16 KB).
+/// For large target sets the SoA layout causes L1 thrashing and
+/// [`l2_distance_batch`] with its AoS per-target locality is faster.
+#[derive(Debug, Clone)]
+pub struct L2Prepared {
+    transposed: Vec<f32>,
+    dimension: usize,
+    num_targets: usize,
+}
+
+impl L2Prepared {
+    /// Transpose `targets` from AoS `[num_targets][dimension]` to SoA layout.
+    pub fn new(targets: &[f32], dimension: usize) -> Self {
+        let num_targets = targets.len() / dimension;
+        debug_assert_eq!(targets.len(), num_targets * dimension);
+
+        let mut transposed = vec![0.0f32; targets.len()];
+        for t in 0..num_targets {
+            for d in 0..dimension {
+                transposed[d * num_targets + t] = targets[t * dimension + d];
+            }
+        }
+
+        Self {
+            transposed,
+            dimension,
+            num_targets,
+        }
+    }
+
+    /// Compute L2 distances from `query` to every target, writing into `out`.
+    ///
+    /// `out` must have length `num_targets`. It will be zeroed before accumulation.
+    pub fn distances_into(&self, query: &[f32], out: &mut [f32]) {
+        debug_assert_eq!(query.len(), self.dimension);
+        debug_assert_eq!(out.len(), self.num_targets);
+
+        out.fill(0.0);
+        for (d, &q) in query.iter().enumerate() {
+            let row = &self.transposed[d * self.num_targets..][..self.num_targets];
+            accumulate_l2_dimension(q, row, out);
+        }
+    }
+
+    /// Compute L2 distances from `query` to every target.
+    pub fn distances(&self, query: &[f32]) -> Vec<f32> {
+        let mut result = vec![0.0f32; self.num_targets];
+        self.distances_into(query, &mut result);
+        result
+    }
+
+    /// Return the index of the nearest target to `query`, using `buf` as scratch space.
+    ///
+    /// `buf` must have length `num_targets`.
+    pub fn nearest_into(&self, query: &[f32], buf: &mut [f32]) -> Option<u32> {
+        self.distances_into(query, buf);
+        crate::kernels::argmin_value_float(buf.iter().copied()).map(|(idx, _)| idx)
+    }
+
+    /// Return the index of the nearest target to `query`.
+    pub fn nearest(&self, query: &[f32]) -> Option<u32> {
+        self.nearest_into(query, &mut vec![0.0f32; self.num_targets])
+    }
+
+    /// Number of targets in this set.
+    pub fn num_targets(&self) -> usize {
+        self.num_targets
+    }
+
+    /// Dimension of each target vector.
+    pub fn dimension(&self) -> usize {
+        self.dimension
+    }
+
+    /// Size of the internal buffer in bytes.
+    pub fn size_bytes(&self) -> usize {
+        self.transposed.len() * std::mem::size_of::<f32>()
     }
 }
 
@@ -445,4 +634,101 @@ mod tests {
             (255_u32.pow(2) * 2048) as f32
         );
     }
+
+    #[test]
+    fn test_l2_targets_matches_scalar() {
+        let cases = vec![
+            (16, 8),   // small target count
+            (16, 16),  // exact SIMD width
+            (16, 256), // PQ-like: 256 centroids, 16-dim sub-vectors
+            (16, 17),  // one remainder
+            (16, 31),  // 15 remainder
+            (1, 32),   // dim=1
+            (3, 20),   // odd dimension
+            (128, 64), // larger dimension
+        ];
+
+        for (dim, num_targets) in cases {
+            let query: Vec<f32> = (0..dim).map(|i| (i as f32) * 0.1 + 0.05).collect();
+            let targets: Vec<f32> = (0..dim * num_targets)
+                .map(|i| ((i * 7 + 3) % 100) as f32 * 0.01)
+                .collect();
+
+            let expected: Vec<f32> = targets
+                .chunks_exact(dim)
+                .map(|v| l2_scalar::<f32, f32, 16>(&query, v))
+                .collect();
+
+            let prepared = L2Prepared::new(&targets, dim);
+            let actual = prepared.distances(&query);
+
+            assert_eq!(
+                actual.len(),
+                expected.len(),
+                "length mismatch for dim={dim}, num_targets={num_targets}"
+            );
+            for (i, (a, e)) in actual.iter().zip(expected.iter()).enumerate() {
+                assert!(
+                    approx::relative_eq!(a, e, max_relative = 1e-6),
+                    "mismatch at index {i} for dim={dim}, num_targets={num_targets}: \
+                     prepared={a}, scalar={e}"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_l2_targets_zeros() {
+        let dim = 16;
+        let num_targets = 32;
+        let query = vec![0.0f32; dim];
+        let targets = vec![0.0f32; dim * num_targets];
+
+        let prepared = L2Prepared::new(&targets, dim);
+        let distances = prepared.distances(&query);
+        assert_eq!(distances.len(), num_targets);
+        for d in &distances {
+            assert_eq!(*d, 0.0);
+        }
+    }
+
+    #[test]
+    fn test_l2_targets_known_values() {
+        let dim = 2;
+        let query = vec![1.0f32, 0.0];
+
+        // 16 targets: [1,0], [0,1], [2,0], [0,0], then 12x [0,0]
+        let mut targets = vec![1.0, 0.0, 0.0, 1.0, 2.0, 0.0, 0.0, 0.0];
+        for _ in 4..16 {
+            targets.extend_from_slice(&[0.0, 0.0]);
+        }
+
+        let prepared = L2Prepared::new(&targets, dim);
+        let distances = prepared.distances(&query);
+        assert_eq!(distances.len(), 16);
+        assert_relative_eq!(distances[0], 0.0);
+        assert_relative_eq!(distances[1], 2.0);
+        assert_relative_eq!(distances[2], 1.0);
+        assert_relative_eq!(distances[3], 1.0);
+        for d in &distances[4..] {
+            assert_relative_eq!(*d, 1.0);
+        }
+    }
+
+    #[test]
+    fn test_l2_targets_reuse() {
+        // Verify that the same L2Prepared can be queried multiple times
+        let dim = 4;
+        let targets = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
+        let prepared = L2Prepared::new(&targets, dim);
+
+        let q1 = vec![1.0, 2.0, 3.0, 4.0];
+        let q2 = vec![5.0, 6.0, 7.0, 8.0];
+
+        let d1 = prepared.distances(&q1);
+        let d2 = prepared.distances(&q2);
+
+        assert_relative_eq!(d1[0], 0.0); // q1 == target[0]
+        assert_relative_eq!(d2[1], 0.0); // q2 == target[1]
+    }
 }
diff --git a/rust/lance-linalg/src/distance/l2_u8.rs b/rust/lance-linalg/src/distance/l2_u8.rs
new file mode 100644
index 00000000000..1b111f91338
--- /dev/null
+++ b/rust/lance-linalg/src/distance/l2_u8.rs
@@ -0,0 +1,272 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Unsigned int8 squared L2 distance with runtime-dispatched SIMD backends.
+//!
+//! Computes `Σ(a[i] - b[i])²` for u8 slices, returning a u32 result.
+//! Used by Scalar Quantization (SQ) distance computation where both L2
+//! and Cosine metric types operate on quantized u8 codes.
+//!
+//! Backends (selected at runtime, best available wins):
+//!   1. scalar     — portable reference, also used for tails
+//!   2. avx2       — VPSADBW-style abs diff + VPMADDWD squaring, 32 elements/iter
+//!   3. avx512vnni — same approach with VPDPWSSD accumulation, 64 elements/iter
+//!
+//! ## Algorithm
+//!
+//! Each SIMD backend computes |a - b| per element using saturating
+//! subtraction: `max(a,b) - min(a,b) = (a ⊖ b) | (b ⊖ a)` where ⊖
+//! is unsigned saturating subtraction. The absolute differences are then
+//! zero-extended to i16 and squared via VPMADDWD (which also pairwise-
+//! accumulates adjacent products into i32).
+
+use std::sync::OnceLock;
+
+/// Portable scalar u8 squared L2 distance, also used for SIMD tail elements.
+#[inline]
+pub fn l2_u8_scalar(a: &[u8], b: &[u8]) -> u32 {
+    debug_assert_eq!(a.len(), b.len());
+    a.iter()
+        .zip(b.iter())
+        .map(|(&x, &y)| (x.abs_diff(y) as u32).pow(2))
+        .sum()
+}
+
+#[cfg(target_arch = "x86_64")]
+mod x86 {
+    use std::arch::x86_64::*;
+
+    /// Horizontal sum of all 8 × i32 lanes in a __m256i.
+    #[inline(always)]
+    unsafe fn hsum_epi32_avx2(v: __m256i) -> u32 {
+        let lo128 = _mm256_castsi256_si128(v);
+        let hi128 = _mm256_extracti128_si256(v, 1);
+        let mut sum128 = _mm_add_epi32(lo128, hi128);
+        sum128 = _mm_hadd_epi32(sum128, sum128);
+        sum128 = _mm_hadd_epi32(sum128, sum128);
+        _mm_cvtsi128_si32(sum128) as u32
+    }
+
+    /// AVX2 path: saturating-sub abs diff, unpack to i16, VPMADDWD to square.
+    /// 32 elements/iter.
+    #[target_feature(enable = "avx2")]
+    pub unsafe fn l2_u8_avx2(a: &[u8], b: &[u8]) -> u32 {
+        debug_assert_eq!(a.len(), b.len());
+        let n = a.len();
+        let zeros = _mm256_setzero_si256();
+        let mut acc = _mm256_setzero_si256();
+        let mut i = 0usize;
+
+        while i + 32 <= n {
+            let av = _mm256_loadu_si256(a.as_ptr().add(i) as *const __m256i);
+            let bv = _mm256_loadu_si256(b.as_ptr().add(i) as *const __m256i);
+
+            // |a - b| via saturating subtraction: max(a-b, 0) | max(b-a, 0)
+            let abs_diff = _mm256_or_si256(_mm256_subs_epu8(av, bv), _mm256_subs_epu8(bv, av));
+
+            // Zero-extend u8→i16 via interleave with zeros.
+            // unpacklo/hi within each 128-bit lane.
+            let diff_lo = _mm256_unpacklo_epi8(abs_diff, zeros);
+            let diff_hi = _mm256_unpackhi_epi8(abs_diff, zeros);
+
+            // VPMADDWD squares adjacent i16 pairs and sums into i32.
+            acc = _mm256_add_epi32(acc, _mm256_madd_epi16(diff_lo, diff_lo));
+            acc = _mm256_add_epi32(acc, _mm256_madd_epi16(diff_hi, diff_hi));
+            i += 32;
+        }
+
+        let mut result = hsum_epi32_avx2(acc);
+
+        // Scalar tail
+        while i < n {
+            let d = a[i].abs_diff(b[i]) as u32;
+            result += d * d;
+            i += 1;
+        }
+        result
+    }
+
+    /// AVX-512 VNNI path: abs diff + VPDPWSSD for fused square-accumulate.
+    /// 64 elements/iter.
+    #[target_feature(enable = "avx512f,avx512bw,avx512vnni")]
+    pub unsafe fn l2_u8_avx512_vnni(a: &[u8], b: &[u8]) -> u32 {
+        debug_assert_eq!(a.len(), b.len());
+        let n = a.len();
+        let zeros = _mm512_setzero_si512();
+        let mut acc = _mm512_setzero_si512();
+        let mut i = 0usize;
+
+        while i + 64 <= n {
+            let av = _mm512_loadu_si512(a.as_ptr().add(i) as *const __m512i);
+            let bv = _mm512_loadu_si512(b.as_ptr().add(i) as *const __m512i);
+
+            // |a - b| via saturating subtraction
+            let abs_diff = _mm512_or_si512(_mm512_subs_epu8(av, bv), _mm512_subs_epu8(bv, av));
+
+            // Zero-extend u8→i16 via interleave with zeros
+            let diff_lo = _mm512_unpacklo_epi8(abs_diff, zeros);
+            let diff_hi = _mm512_unpackhi_epi8(abs_diff, zeros);
+
+            // VPDPWSSD: signed i16 pairwise multiply-add into i32 accumulator.
+            // diff values are 0..255, fitting in i16 as positive, so signed is fine.
+            acc = _mm512_dpwssd_epi32(acc, diff_lo, diff_lo);
+            acc = _mm512_dpwssd_epi32(acc, diff_hi, diff_hi);
+            i += 64;
+        }
+
+        let mut result = _mm512_reduce_add_epi32(acc) as u32;
+
+        // Scalar tail
+        while i < n {
+            let d = a[i].abs_diff(b[i]) as u32;
+            result += d * d;
+            i += 1;
+        }
+        result
+    }
+}
+
+type L2U8Fn = fn(&[u8], &[u8]) -> u32;
+
+static DISPATCH: OnceLock<L2U8Fn> = OnceLock::new();
+
+fn select_backend() -> L2U8Fn {
+    #[cfg(target_arch = "x86_64")]
+    {
+        if is_x86_feature_detected!("avx512f")
+            && is_x86_feature_detected!("avx512bw")
+            && is_x86_feature_detected!("avx512vnni")
+        {
+            return |a, b| unsafe { x86::l2_u8_avx512_vnni(a, b) };
+        }
+
+        if is_x86_feature_detected!("avx2") {
+            return |a, b| unsafe { x86::l2_u8_avx2(a, b) };
+        }
+    }
+
+    l2_u8_scalar
+}
+
+/// Dispatched u8 squared L2 distance, selecting the best available SIMD backend.
+#[inline]
+pub fn l2_u8(a: &[u8], b: &[u8]) -> u32 {
+    (DISPATCH.get_or_init(select_backend))(a, b)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn fill_random(buf: &mut [u8], seed: &mut u32) {
+        for slot in buf.iter_mut() {
+            *seed = seed.wrapping_mul(1103515245).wrapping_add(12345);
+            *slot = (*seed >> 16) as u8;
+        }
+    }
+
+    const SIZES: &[usize] = &[
+        0, 1, 7, 15, 16, 31, 32, 33, 63, 64, 65, 127, 128, 255, 256, 1024, 4096, 4097,
+    ];
+
+    fn check_all_backends(a: &[u8], b: &[u8], case: &str) {
+        let reference = l2_u8_scalar(a, b);
+
+        #[cfg(target_arch = "x86_64")]
+        {
+            if is_x86_feature_detected!("avx2") {
+                let got = unsafe { x86::l2_u8_avx2(a, b) };
+                assert_eq!(got, reference, "avx2 [{case}] n={}", a.len());
+            }
+
+            if is_x86_feature_detected!("avx512f")
+                && is_x86_feature_detected!("avx512bw")
+                && is_x86_feature_detected!("avx512vnni")
+            {
+                let got = unsafe { x86::l2_u8_avx512_vnni(a, b) };
+                assert_eq!(got, reference, "avx512_vnni [{case}] n={}", a.len());
+            }
+        }
+
+        assert_eq!(l2_u8(a, b), reference, "dispatch [{case}] n={}", a.len());
+    }
+
+    #[test]
+    fn random_inputs_across_sizes_and_seeds() {
+        let mut a = vec![0u8; 4097];
+        let mut b = vec![0u8; 4097];
+
+        for seed_idx in 0..4u32 {
+            let mut seed = 0xC0FFEE_u32.wrapping_add(seed_idx.wrapping_mul(7919));
+            for &n in SIZES {
+                fill_random(&mut a[..n], &mut seed);
+                fill_random(&mut b[..n], &mut seed);
+                check_all_backends(&a[..n], &b[..n], "random");
+            }
+        }
+    }
+
+    #[test]
+    fn boundary_values() {
+        let mut a = vec![0u8; 4097];
+        let mut b = vec![0u8; 4097];
+
+        for &n in SIZES {
+            // max diff: |255 - 0|² × n
+            a[..n].fill(u8::MAX);
+            b[..n].fill(0);
+            check_all_backends(&a[..n], &b[..n], "max-0");
+
+            a[..n].fill(0);
+            b[..n].fill(u8::MAX);
+            check_all_backends(&a[..n], &b[..n], "0-max");
+
+            // identical vectors: distance = 0
+            a[..n].fill(u8::MAX);
+            b[..n].fill(u8::MAX);
+            check_all_backends(&a[..n], &b[..n], "max-max");
+            assert_eq!(l2_u8_scalar(&a[..n], &b[..n]), 0);
+
+            // zeros
+            a[..n].fill(0);
+            b[..n].fill(0);
+            check_all_backends(&a[..n], &b[..n], "0-0");
+            assert_eq!(l2_u8_scalar(&a[..n], &b[..n]), 0);
+
+            // alternating
+            for i in 0..n {
+                a[i] = if i & 1 == 0 { 0 } else { u8::MAX };
+                b[i] = if i & 1 == 0 { u8::MAX } else { 0 };
+            }
+            check_all_backends(&a[..n], &b[..n], "alt 0/max");
+        }
+    }
+
+    #[test]
+    fn one_sided_zeros() {
+        let mut a = vec![0u8; 4097];
+        let mut b = vec![0u8; 4097];
+
+        for &n in SIZES {
+            let mut seed = 0xDEAD_BEEF_u32;
+            fill_random(&mut a[..n], &mut seed);
+            b[..n].fill(0);
+            check_all_backends(&a[..n], &b[..n], "b=0");
+
+            a[..n].fill(0);
+            fill_random(&mut b[..n], &mut seed);
+            check_all_backends(&a[..n], &b[..n], "a=0");
+        }
+    }
+
+    #[test]
+    fn known_values() {
+        // 3² + 1² = 10
+        assert_eq!(l2_u8_scalar(&[10, 20], &[7, 21]), 10);
+        assert_eq!(l2_u8(&[10, 20], &[7, 21]), 10);
+
+        // max single-element distance: 255² = 65025
+        assert_eq!(l2_u8(&[0], &[255]), 65025);
+        assert_eq!(l2_u8(&[255], &[0]), 65025);
+    }
+}
diff --git a/rust/lance-linalg/src/distance/norm_l2.rs b/rust/lance-linalg/src/distance/norm_l2.rs
index 97ee4cb1ce9..cd604ac8c05 100644
--- a/rust/lance-linalg/src/distance/norm_l2.rs
+++ b/rust/lance-linalg/src/distance/norm_l2.rs
@@ -80,10 +80,54 @@ impl Normalize for f16 {
     }
 }
 
+#[cfg(feature = "fp16kernels")]
+mod bf16_kernel {
+    use half::bf16;
+
+    unsafe extern "C" {
+        #[cfg(target_arch = "aarch64")]
+        pub fn norm_l2_bf16_neon(ptr: *const bf16, len: u32) -> f32;
+        #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
+        pub fn norm_l2_bf16_avx512(ptr: *const bf16, len: u32) -> f32;
+        #[cfg(target_arch = "x86_64")]
+        pub fn norm_l2_bf16_avx2(ptr: *const bf16, len: u32) -> f32;
+        #[cfg(target_arch = "loongarch64")]
+        pub fn norm_l2_bf16_lsx(ptr: *const bf16, len: u32) -> f32;
+        #[cfg(target_arch = "loongarch64")]
+        pub fn norm_l2_bf16_lasx(ptr: *const bf16, len: u32) -> f32;
+    }
+}
+
 impl Normalize for bf16 {
     #[inline]
     fn norm_l2(vector: &[Self]) -> f32 {
-        norm_l2_impl::<Self, f32, 32>(vector)
+        match *SIMD_SUPPORT {
+            #[cfg(all(feature = "fp16kernels", target_arch = "aarch64"))]
+            SimdSupport::Neon => unsafe {
+                bf16_kernel::norm_l2_bf16_neon(vector.as_ptr(), vector.len() as u32)
+            },
+            #[cfg(all(
+                feature = "fp16kernels",
+                kernel_support = "avx512",
+                target_arch = "x86_64"
+            ))]
+            SimdSupport::Avx512FP16 => unsafe {
+                bf16_kernel::norm_l2_bf16_avx512(vector.as_ptr(), vector.len() as u32)
+            },
+            #[cfg(all(feature = "fp16kernels", target_arch = "x86_64"))]
+            SimdSupport::Avx2 | SimdSupport::Avx512 => unsafe {
+                bf16_kernel::norm_l2_bf16_avx2(vector.as_ptr(), vector.len() as u32)
+            },
+            #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))]
+            SimdSupport::Lasx => unsafe {
+                bf16_kernel::norm_l2_bf16_lasx(vector.as_ptr(), vector.len() as u32)
+            },
+            #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))]
+            SimdSupport::Lsx => unsafe {
+                bf16_kernel::norm_l2_bf16_lsx(vector.as_ptr(), vector.len() as u32)
+            },
+            _ => norm_l2_impl::<Self, f32, 32>(vector),
+        }
     }
 }
 
@@ -97,8 +141,40 @@ impl Normalize for f32 {
 impl Normalize for f64 {
     #[inline]
     fn norm_l2(vector: &[Self]) -> f32 {
-        norm_l2_impl::<Self, Self, 8>(vector) as f32
+        norm_l2_f64_simd(vector)
+    }
+}
+
+/// Explicit SIMD implementation of L2 norm for f64.
+///
+/// Two-level unrolling: f64x8 main loop, f64x4 remainder, scalar tail.
+#[inline]
+pub fn norm_l2_f64_simd(vector: &[f64]) -> f32 {
+    use crate::simd::f64::{f64x4, f64x8};
+    use crate::simd::{FloatSimd, SIMD};
+
+    let dim = vector.len();
+    let unrolled_len = dim / 8 * 8;
+
+    let mut acc8 = f64x8::zeros();
+    for i in (0..unrolled_len).step_by(8) {
+        unsafe {
+            let v = f64x8::load_unaligned(vector.as_ptr().add(i));
+            acc8.multiply_add(v, v);
+        }
     }
+
+    let aligned_len = dim / 4 * 4;
+    let mut acc4 = f64x4::zeros();
+    for i in (unrolled_len..aligned_len).step_by(4) {
+        unsafe {
+            let v = f64x4::load_unaligned(vector.as_ptr().add(i));
+            acc4.multiply_add(v, v);
+        }
+    }
+
+    let tail: f64 = vector[aligned_len..].iter().map(|&v| v * v).sum();
+    (acc8.reduce_sum() + acc4.reduce_sum() + tail).sqrt() as f32
 }
 
 /// NOTE: this is only pub for benchmarking purposes
diff --git a/rust/lance-linalg/src/simd.rs b/rust/lance-linalg/src/simd.rs
index 0526d531b8f..91dc1c6959d 100644
--- a/rust/lance-linalg/src/simd.rs
+++ b/rust/lance-linalg/src/simd.rs
@@ -16,6 +16,7 @@ use std::ops::{Add, AddAssign, Mul, Sub, SubAssign};
 
 pub mod dist_table;
 pub mod f32;
+pub mod f64;
 pub mod i32;
 pub mod u8;
 
diff --git a/rust/lance-linalg/src/simd/bf16.c b/rust/lance-linalg/src/simd/bf16.c
new file mode 100644
index 00000000000..cfbe5dee971
--- /dev/null
+++ b/rust/lance-linalg/src/simd/bf16.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+#include <stddef.h>
+#include <stdint.h>
+#include <math.h>
+#include <string.h>
+
+// Because we might be compiling this library multiple times, we need to
+// add a suffix to each of the function names.
+#define FUNC_CAT_INNER(A, B) A##B
+#define FUNC_CAT(A, B) FUNC_CAT_INNER(A, B)
+#define FUNC(N) FUNC_CAT(N, SUFFIX)
+
+// Convert a bf16 value (stored as uint16_t) to float32.
+// BF16 is the upper 16 bits of an IEEE 754 float32, so we just shift left.
+static inline float bf16_to_f32(uint16_t v) {
+  uint32_t bits = (uint32_t)v << 16;
+  float f;
+  memcpy(&f, &bits, sizeof(f));
+  return f;
+}
+
+float FUNC(norm_l2_bf16)(const uint16_t *data, uint32_t dimension) {
+  float sum = 0;
+
+#pragma clang loop unroll(enable) vectorize(enable) interleave(enable)
+  for (uint32_t i = 0; i < dimension; i++) {
+    float v = bf16_to_f32(data[i]);
+    sum += v * v;
+  }
+  return sqrtf(sum);
+}
+
+/// @brief Dot product of two bf16 vectors.
+/// @param x A bf16 vector (stored as uint16_t)
+/// @param y A bf16 vector (stored as uint16_t)
+/// @param dimension The dimension of the vectors
+/// @return The dot product of the two vectors.
+float FUNC(dot_bf16)(const uint16_t *x, const uint16_t *y, uint32_t dimension) {
+  float sum = 0;
+
+#pragma clang loop unroll(enable) interleave(enable) vectorize(enable)
+  for (uint32_t i = 0; i < dimension; i++) {
+    sum += bf16_to_f32(x[i]) * bf16_to_f32(y[i]);
+  }
+  return sum;
+}
+
+float FUNC(l2_bf16)(const uint16_t *x, const uint16_t *y, uint32_t dimension) {
+  float sum = 0.0;
+
+#pragma clang loop unroll(enable) interleave(enable) vectorize(enable)
+  for (uint32_t i = 0; i < dimension; i++) {
+    float s = bf16_to_f32(x[i]) - bf16_to_f32(y[i]);
+    sum += s * s;
+  }
+  return sum;
+}
+
+float FUNC(cosine_bf16)(const uint16_t *x, float x_norm, const uint16_t *y, uint32_t dimension) {
+  float dot = 0.0;
+  float l2_y = 0.0;
+
+  // Combine the loop to reduce overhead of the bf16 to fp32 conversion.
+#pragma clang loop unroll(enable) interleave(enable) vectorize(enable)
+  for (uint32_t i = 0; i < dimension; i++) {
+    float y_i = bf16_to_f32(y[i]);
+    dot += bf16_to_f32(x[i]) * y_i;
+    l2_y += y_i * y_i;
+  }
+
+  return 1.0 - dot / (x_norm * sqrtf(l2_y));
+}
diff --git a/rust/lance-linalg/src/simd/dist_table.rs b/rust/lance-linalg/src/simd/dist_table.rs
index 2c1609072ea..689e949106e 100644
--- a/rust/lance-linalg/src/simd/dist_table.rs
+++ b/rust/lance-linalg/src/simd/dist_table.rs
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+#[cfg(target_arch = "aarch64")]
+use std::arch::aarch64::*;
 #[cfg(target_arch = "x86_64")]
 use std::arch::x86_64::*;
 
@@ -57,13 +59,28 @@ pub fn sum_4bit_dist_table(
                 )
             }
         },
+        #[cfg(target_arch = "aarch64")]
+        SimdSupport::Neon => unsafe {
+            for i in (0..n).step_by(BATCH_SIZE) {
+                sum_dist_table_32bytes_batch_neon(
+                    &codes[i * code_len..(i + BATCH_SIZE) * code_len],
+                    dist_table,
+                    &mut dists[i..i + BATCH_SIZE],
+                )
+            }
+        },
         _ => sum_4bit_dist_table_scalar(code_len, codes, dist_table, dists),
     }
 }
 
 #[inline]
 #[allow(unused)]
-fn sum_4bit_dist_table_scalar(code_len: usize, codes: &[u8], dist_table: &[u8], dists: &mut [u16]) {
+pub fn sum_4bit_dist_table_scalar(
+    code_len: usize,
+    codes: &[u8],
+    dist_table: &[u8],
+    dists: &mut [u16],
+) {
     for (vec_block_idx, blocks) in codes.chunks_exact(BATCH_SIZE * code_len).enumerate() {
         for (sub_vec_idx, block) in blocks.chunks_exact(BATCH_SIZE).enumerate() {
             let current_dist_table = &dist_table[sub_vec_idx * 2 * 16..(sub_vec_idx * 2 + 1) * 16];
@@ -159,6 +176,77 @@ unsafe fn sum_dist_table_32bytes_batch_avx2(codes: &[u8], dist_table: &[u8], dis
     _mm256_storeu_si256(dists.as_mut_ptr().add(16) as *mut __m256i, dis1);
 }
 
+#[cfg(target_arch = "aarch64")]
+#[inline]
+unsafe fn sum_dist_table_32bytes_batch_neon(codes: &[u8], dist_table: &[u8], dists: &mut [u16]) {
+    let low_mask = vdupq_n_u8(0x0f);
+
+    // 8 accumulators: 4 per 128-bit "lane" (lo = bytes 0..16, hi = bytes 16..32 of each block)
+    let mut accu0_lo = vdupq_n_u16(0);
+    let mut accu1_lo = vdupq_n_u16(0);
+    let mut accu2_lo = vdupq_n_u16(0);
+    let mut accu3_lo = vdupq_n_u16(0);
+    let mut accu0_hi = vdupq_n_u16(0);
+    let mut accu1_hi = vdupq_n_u16(0);
+    let mut accu2_hi = vdupq_n_u16(0);
+    let mut accu3_hi = vdupq_n_u16(0);
+
+    let codes_ptr = codes.as_ptr();
+    let dt_ptr = dist_table.as_ptr();
+
+    for i in (0..codes.len()).step_by(32) {
+        // Process lo lane: bytes [i..i+16]
+        let c_lo = vld1q_u8(codes_ptr.add(i));
+        let lut_lo = vld1q_u8(dt_ptr.add(i));
+
+        let lo_lo = vandq_u8(c_lo, low_mask);
+        let hi_lo = vshrq_n_u8::<4>(c_lo);
+
+        let res_lo_lo = vqtbl1q_u8(lut_lo, lo_lo);
+        let res_hi_lo = vqtbl1q_u8(lut_lo, hi_lo);
+
+        accu0_lo = vaddq_u16(accu0_lo, vreinterpretq_u16_u8(res_lo_lo));
+        accu1_lo = vaddq_u16(accu1_lo, vshrq_n_u16::<8>(vreinterpretq_u16_u8(res_lo_lo)));
+        accu2_lo = vaddq_u16(accu2_lo, vreinterpretq_u16_u8(res_hi_lo));
+        accu3_lo = vaddq_u16(accu3_lo, vshrq_n_u16::<8>(vreinterpretq_u16_u8(res_hi_lo)));
+
+        // Process hi lane: bytes [i+16..i+32]
+        let c_hi = vld1q_u8(codes_ptr.add(i + 16));
+        let lut_hi = vld1q_u8(dt_ptr.add(i + 16));
+
+        let lo_hi = vandq_u8(c_hi, low_mask);
+        let hi_hi = vshrq_n_u8::<4>(c_hi);
+
+        let res_lo_hi = vqtbl1q_u8(lut_hi, lo_hi);
+        let res_hi_hi = vqtbl1q_u8(lut_hi, hi_hi);
+
+        accu0_hi = vaddq_u16(accu0_hi, vreinterpretq_u16_u8(res_lo_hi));
+        accu1_hi = vaddq_u16(accu1_hi, vshrq_n_u16::<8>(vreinterpretq_u16_u8(res_lo_hi)));
+        accu2_hi = vaddq_u16(accu2_hi, vreinterpretq_u16_u8(res_hi_hi));
+        accu3_hi = vaddq_u16(accu3_hi, vshrq_n_u16::<8>(vreinterpretq_u16_u8(res_hi_hi)));
+    }
+
+    // Merge: clean even bytes by subtracting the odd-byte bleed
+    accu0_lo = vsubq_u16(accu0_lo, vshlq_n_u16::<8>(accu1_lo));
+    accu0_hi = vsubq_u16(accu0_hi, vshlq_n_u16::<8>(accu1_hi));
+
+    // Cross-lane merge: add lo and hi lane accumulators
+    // This is the NEON equivalent of AVX2's permute2f128 + blend + add
+    let dis0_even = vaddq_u16(accu0_lo, accu0_hi);
+    let dis0_odd = vaddq_u16(accu1_lo, accu1_hi);
+    vst1q_u16(dists.as_mut_ptr(), dis0_even);
+    vst1q_u16(dists.as_mut_ptr().add(8), dis0_odd);
+
+    // Same for hi-nibble accumulators (vectors 16..31)
+    accu2_lo = vsubq_u16(accu2_lo, vshlq_n_u16::<8>(accu3_lo));
+    accu2_hi = vsubq_u16(accu2_hi, vshlq_n_u16::<8>(accu3_hi));
+
+    let dis1_even = vaddq_u16(accu2_lo, accu2_hi);
+    let dis1_odd = vaddq_u16(accu3_lo, accu3_hi);
+    vst1q_u16(dists.as_mut_ptr().add(16), dis1_even);
+    vst1q_u16(dists.as_mut_ptr().add(24), dis1_odd);
+}
+
 // We implement the AVX512 version in C because AVX512 is not stable yet in Rust,
 // implement it in Rust once we upgrade rust to 1.89.0.
 unsafe extern "C" {
@@ -214,4 +302,83 @@ mod tests {
         // so the distance is 2 * (dist_table[0x6] + dist_table[0xb + 16]) = 2*(7 + 12) = 38
         assert_eq!(dists[1], 38);
     }
+
+    /// Test that the SIMD path (NEON on ARM, AVX2 on x86) produces identical
+    /// results to the scalar reference across a range of dimensions, including
+    /// very large ones (up to DIM=65536).
+    ///
+    /// Note: dist_table values are capped to avoid u16 overflow, matching
+    /// production behavior where values are quantized to a small range.
+    /// (The scalar path uses saturating_add while SIMD uses wrapping add,
+    /// so they diverge on overflow — but overflow never occurs with real
+    /// quantized data.)
+    #[test]
+    fn test_simd_matches_scalar_varied_dimensions() {
+        use rand::{Rng, SeedableRng};
+        let mut rng = rand::rngs::StdRng::seed_from_u64(42);
+
+        // code_len = dim / 8 for 1-bit quantization; we test various code_lens
+        // directly since that's what the function sees.
+        // code_len=16 → DIM=128, code_len=192 → DIM=1536,
+        // code_len=512 → DIM=4096, code_len=8192 → DIM=65536
+        for code_len in [2, 16, 96, 192, 512, 1024, 8192] {
+            let n = BATCH_SIZE; // 32 vectors per batch
+
+            // Each code byte produces 2 lookups; cap values so
+            // 2 * code_len * max_val < u16::MAX.
+            let max_val = (u16::MAX as usize / (2 * code_len)).min(255) as u8;
+
+            let codes: Vec<u8> = (0..n * code_len).map(|_| rng.random::<u8>()).collect();
+            let dist_table: Vec<u8> = (0..BATCH_SIZE * code_len)
+                .map(|_| rng.random_range(0..=max_val))
+                .collect();
+
+            let mut expected = vec![0u16; n];
+            sum_4bit_dist_table_scalar(code_len, &codes, &dist_table, &mut expected);
+
+            let mut actual = vec![0u16; n];
+            sum_4bit_dist_table(n, code_len, &codes, &dist_table, &mut actual);
+
+            assert_eq!(
+                actual,
+                expected,
+                "SIMD and scalar mismatch for code_len={} (DIM={})",
+                code_len,
+                code_len * 8,
+            );
+        }
+    }
+
+    /// Test with multiple batches to verify accumulation across batch boundaries.
+    #[test]
+    fn test_simd_matches_scalar_multi_batch() {
+        use rand::{Rng, SeedableRng};
+        let mut rng = rand::rngs::StdRng::seed_from_u64(123);
+
+        for code_len in [16, 192, 1024] {
+            let n = BATCH_SIZE * 10; // 320 vectors = 10 batches
+
+            let max_val = (u16::MAX as usize / (2 * code_len)).min(255) as u8;
+
+            let codes: Vec<u8> = (0..n * code_len).map(|_| rng.random::<u8>()).collect();
+            let dist_table: Vec<u8> = (0..BATCH_SIZE * code_len)
+                .map(|_| rng.random_range(0..=max_val))
+                .collect();
+
+            let mut expected = vec![0u16; n];
+            sum_4bit_dist_table_scalar(code_len, &codes, &dist_table, &mut expected);
+
+            let mut actual = vec![0u16; n];
+            sum_4bit_dist_table(n, code_len, &codes, &dist_table, &mut actual);
+
+            assert_eq!(
+                actual,
+                expected,
+                "SIMD and scalar mismatch for multi-batch code_len={} (DIM={}, n={})",
+                code_len,
+                code_len * 8,
+                n,
+            );
+        }
+    }
 }
diff --git a/rust/lance-linalg/src/simd/f16.c b/rust/lance-linalg/src/simd/f16.c
index bc0356d87e9..7bba4de3dc5 100644
--- a/rust/lance-linalg/src/simd/f16.c
+++ b/rust/lance-linalg/src/simd/f16.c
@@ -22,7 +22,6 @@
 #define FUNC_CAT(A, B) FUNC_CAT_INNER(A, B)
 #define FUNC(N) FUNC_CAT(N, SUFFIX)
 
-// TODO: I wonder if we could re-purpose this macro to compile bf16 kernels?
 #if defined(__clang__)
 // Note: we use __fp16 instead of _Float16 because Clang < 15.0.0 does not
 // support it well for most targets. __fp16 works for our purposes here since
diff --git a/rust/lance-linalg/src/simd/f64.rs b/rust/lance-linalg/src/simd/f64.rs
new file mode 100644
index 00000000000..32c0d389e5b
--- /dev/null
+++ b/rust/lance-linalg/src/simd/f64.rs
@@ -0,0 +1,886 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! `f64x4` and `f64x8` SIMD types for f64 distance computations.
+
+use std::fmt::Formatter;
+
+#[cfg(target_arch = "aarch64")]
+use std::arch::aarch64::*;
+#[cfg(target_arch = "loongarch64")]
+use std::arch::loongarch64::*;
+#[cfg(target_arch = "x86_64")]
+use std::arch::x86_64::*;
+#[cfg(target_arch = "loongarch64")]
+use std::mem::transmute;
+use std::ops::{Add, AddAssign, Mul, Sub, SubAssign};
+
+use super::{FloatSimd, SIMD};
+
+/// 4 of 64-bit `f64` values. Uses 256-bit SIMD if possible.
+#[allow(non_camel_case_types)]
+#[cfg(target_arch = "x86_64")]
+#[derive(Clone, Copy)]
+pub struct f64x4(std::arch::x86_64::__m256d);
+
+#[allow(non_camel_case_types)]
+#[cfg(target_arch = "aarch64")]
+#[derive(Clone, Copy)]
+pub struct f64x4(float64x2x2_t);
+
+#[allow(non_camel_case_types)]
+#[cfg(target_arch = "loongarch64")]
+#[derive(Clone, Copy)]
+pub struct f64x4(v4f64);
+
+impl std::fmt::Debug for f64x4 {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut arr = [0.0_f64; 4];
+        unsafe {
+            self.store_unaligned(arr.as_mut_ptr());
+        }
+        write!(f, "f64x4({:?})", arr)
+    }
+}
+
+impl From<&[f64]> for f64x4 {
+    fn from(value: &[f64]) -> Self {
+        unsafe { Self::load_unaligned(value.as_ptr()) }
+    }
+}
+
+impl<'a> From<&'a [f64; 4]> for f64x4 {
+    fn from(value: &'a [f64; 4]) -> Self {
+        unsafe { Self::load_unaligned(value.as_ptr()) }
+    }
+}
+
+impl SIMD<f64, 4> for f64x4 {
+    fn splat(val: f64) -> Self {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            Self(_mm256_set1_pd(val))
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            Self(float64x2x2_t(vdupq_n_f64(val), vdupq_n_f64(val)))
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            Self(transmute(lasx_xvreplgr2vr_d(transmute(val))))
+        }
+    }
+
+    fn zeros() -> Self {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            Self(_mm256_setzero_pd())
+        }
+        #[cfg(target_arch = "aarch64")]
+        {
+            Self::splat(0.0)
+        }
+        #[cfg(target_arch = "loongarch64")]
+        {
+            Self::splat(0.0)
+        }
+    }
+
+    #[inline]
+    unsafe fn load(ptr: *const f64) -> Self {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            Self(_mm256_load_pd(ptr))
+        }
+        #[cfg(target_arch = "aarch64")]
+        {
+            Self::load_unaligned(ptr)
+        }
+        #[cfg(target_arch = "loongarch64")]
+        {
+            Self(transmute(lasx_xvld::<0>(transmute(ptr))))
+        }
+    }
+
+    #[inline]
+    unsafe fn load_unaligned(ptr: *const f64) -> Self {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            Self(_mm256_loadu_pd(ptr))
+        }
+        #[cfg(target_arch = "aarch64")]
+        {
+            Self(vld1q_f64_x2(ptr))
+        }
+        #[cfg(target_arch = "loongarch64")]
+        {
+            Self(transmute(lasx_xvld::<0>(transmute(ptr))))
+        }
+    }
+
+    unsafe fn store(&self, ptr: *mut f64) {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            _mm256_store_pd(ptr, self.0);
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            vst1q_f64_x2(ptr, self.0);
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            lasx_xvst::<0>(transmute(self.0), transmute(ptr));
+        }
+    }
+
+    unsafe fn store_unaligned(&self, ptr: *mut f64) {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            _mm256_storeu_pd(ptr, self.0);
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            vst1q_f64_x2(ptr, self.0);
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            lasx_xvst::<0>(transmute(self.0), transmute(ptr));
+        }
+    }
+
+    #[inline]
+    fn reduce_sum(&self) -> f64 {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            // [a, b, c, d] -> hadd -> [a+b, a+b, c+d, c+d]
+            let sum = _mm256_hadd_pd(self.0, self.0);
+            // Extract low 128 and high 128, add them
+            let lo = _mm256_castpd256_pd128(sum);
+            let hi = _mm256_extractf128_pd(sum, 1);
+            let r = _mm_add_pd(lo, hi);
+            _mm_cvtsd_f64(r)
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            let sum = vaddq_f64(self.0.0, self.0.1);
+            vaddvq_f64(sum)
+        }
+        #[cfg(target_arch = "loongarch64")]
+        {
+            self.as_array().iter().sum()
+        }
+    }
+
+    fn reduce_min(&self) -> f64 {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            // Swap high/low 128-bit lanes and min
+            let hi = _mm256_permute2f128_pd(self.0, self.0, 1);
+            let m = _mm256_min_pd(self.0, hi);
+            // Swap within 128-bit lane and min
+            let shuf = _mm256_permute_pd(m, 0b0101);
+            let m = _mm256_min_pd(m, shuf);
+            _mm256_cvtsd_f64(m)
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            let m = vminq_f64(self.0.0, self.0.1);
+            vminvq_f64(m)
+        }
+        #[cfg(target_arch = "loongarch64")]
+        {
+            self.as_array()
+                .iter()
+                .copied()
+                .fold(f64::INFINITY, f64::min)
+        }
+    }
+
+    fn min(&self, rhs: &Self) -> Self {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            Self(_mm256_min_pd(self.0, rhs.0))
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            Self(float64x2x2_t(
+                vminq_f64(self.0.0, rhs.0.0),
+                vminq_f64(self.0.1, rhs.0.1),
+            ))
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            Self(lasx_xvfmin_d(self.0, rhs.0))
+        }
+    }
+
+    fn find(&self, val: f64) -> Option<i32> {
+        unsafe {
+            for i in 0..4 {
+                if self.as_array().get_unchecked(i) == &val {
+                    return Some(i as i32);
+                }
+            }
+        }
+        None
+    }
+}
+
+impl FloatSimd<f64, 4> for f64x4 {
+    fn multiply_add(&mut self, a: Self, b: Self) {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            self.0 = _mm256_fmadd_pd(a.0, b.0, self.0);
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            self.0.0 = vfmaq_f64(self.0.0, a.0.0, b.0.0);
+            self.0.1 = vfmaq_f64(self.0.1, a.0.1, b.0.1);
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            self.0 = lasx_xvfmadd_d(a.0, b.0, self.0);
+        }
+    }
+}
+
+impl Add for f64x4 {
+    type Output = Self;
+
+    #[inline]
+    fn add(self, rhs: Self) -> Self::Output {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            Self(_mm256_add_pd(self.0, rhs.0))
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            Self(float64x2x2_t(
+                vaddq_f64(self.0.0, rhs.0.0),
+                vaddq_f64(self.0.1, rhs.0.1),
+            ))
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            Self(lasx_xvfadd_d(self.0, rhs.0))
+        }
+    }
+}
+
+impl AddAssign for f64x4 {
+    #[inline]
+    fn add_assign(&mut self, rhs: Self) {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            self.0 = _mm256_add_pd(self.0, rhs.0)
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            self.0.0 = vaddq_f64(self.0.0, rhs.0.0);
+            self.0.1 = vaddq_f64(self.0.1, rhs.0.1);
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            self.0 = lasx_xvfadd_d(self.0, rhs.0);
+        }
+    }
+}
+
+impl Sub for f64x4 {
+    type Output = Self;
+
+    #[inline]
+    fn sub(self, rhs: Self) -> Self::Output {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            Self(_mm256_sub_pd(self.0, rhs.0))
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            Self(float64x2x2_t(
+                vsubq_f64(self.0.0, rhs.0.0),
+                vsubq_f64(self.0.1, rhs.0.1),
+            ))
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            Self(lasx_xvfsub_d(self.0, rhs.0))
+        }
+    }
+}
+
+impl SubAssign for f64x4 {
+    #[inline]
+    fn sub_assign(&mut self, rhs: Self) {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            self.0 = _mm256_sub_pd(self.0, rhs.0)
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            self.0.0 = vsubq_f64(self.0.0, rhs.0.0);
+            self.0.1 = vsubq_f64(self.0.1, rhs.0.1);
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            self.0 = lasx_xvfsub_d(self.0, rhs.0);
+        }
+    }
+}
+
+impl Mul for f64x4 {
+    type Output = Self;
+
+    #[inline]
+    fn mul(self, rhs: Self) -> Self::Output {
+        #[cfg(target_arch = "x86_64")]
+        unsafe {
+            Self(_mm256_mul_pd(self.0, rhs.0))
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            Self(float64x2x2_t(
+                vmulq_f64(self.0.0, rhs.0.0),
+                vmulq_f64(self.0.1, rhs.0.1),
+            ))
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            Self(lasx_xvfmul_d(self.0, rhs.0))
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// f64x8: 8 × f64 values (512-bit SIMD or 2 × 256-bit)
+// ---------------------------------------------------------------------------
+
+/// 8 of 64-bit `f64` values. Uses 512-bit SIMD if possible.
+#[allow(non_camel_case_types)]
+#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+#[derive(Clone, Copy)]
+pub struct f64x8(__m512d);
+
+#[allow(non_camel_case_types)]
+#[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+#[derive(Clone, Copy)]
+pub struct f64x8(__m256d, __m256d);
+
+#[allow(non_camel_case_types)]
+#[cfg(target_arch = "aarch64")]
+#[derive(Clone, Copy)]
+pub struct f64x8(float64x2x2_t, float64x2x2_t);
+
+#[allow(non_camel_case_types)]
+#[cfg(target_arch = "loongarch64")]
+#[derive(Clone, Copy)]
+pub struct f64x8(v4f64, v4f64);
+
+impl std::fmt::Debug for f64x8 {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut arr = [0.0_f64; 8];
+        unsafe {
+            self.store_unaligned(arr.as_mut_ptr());
+        }
+        write!(f, "f64x8({:?})", arr)
+    }
+}
+
+impl From<&[f64]> for f64x8 {
+    fn from(value: &[f64]) -> Self {
+        unsafe { Self::load_unaligned(value.as_ptr()) }
+    }
+}
+
+impl<'a> From<&'a [f64; 8]> for f64x8 {
+    fn from(value: &'a [f64; 8]) -> Self {
+        unsafe { Self::load_unaligned(value.as_ptr()) }
+    }
+}
+
+impl SIMD<f64, 8> for f64x8 {
+    #[inline]
+    fn splat(val: f64) -> Self {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            Self(_mm512_set1_pd(val))
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            Self(_mm256_set1_pd(val), _mm256_set1_pd(val))
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            let v = vdupq_n_f64(val);
+            Self(float64x2x2_t(v, v), float64x2x2_t(v, v))
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            let v = transmute(lasx_xvreplgr2vr_d(transmute(val)));
+            Self(v, v)
+        }
+    }
+
+    #[inline]
+    fn zeros() -> Self {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            Self(_mm512_setzero_pd())
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            Self(_mm256_setzero_pd(), _mm256_setzero_pd())
+        }
+        #[cfg(target_arch = "aarch64")]
+        {
+            Self::splat(0.0)
+        }
+        #[cfg(target_arch = "loongarch64")]
+        {
+            Self::splat(0.0)
+        }
+    }
+
+    #[inline]
+    unsafe fn load(ptr: *const f64) -> Self {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            Self(_mm512_load_pd(ptr))
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            Self(_mm256_load_pd(ptr), _mm256_load_pd(ptr.add(4)))
+        }
+        #[cfg(target_arch = "aarch64")]
+        {
+            Self::load_unaligned(ptr)
+        }
+        #[cfg(target_arch = "loongarch64")]
+        {
+            Self(
+                transmute(lasx_xvld::<0>(transmute(ptr))),
+                transmute(lasx_xvld::<32>(transmute(ptr))),
+            )
+        }
+    }
+
+    #[inline]
+    unsafe fn load_unaligned(ptr: *const f64) -> Self {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            Self(_mm512_loadu_pd(ptr))
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            Self(_mm256_loadu_pd(ptr), _mm256_loadu_pd(ptr.add(4)))
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            Self(vld1q_f64_x2(ptr), vld1q_f64_x2(ptr.add(4)))
+        }
+        #[cfg(target_arch = "loongarch64")]
+        {
+            Self(
+                transmute(lasx_xvld::<0>(transmute(ptr))),
+                transmute(lasx_xvld::<32>(transmute(ptr))),
+            )
+        }
+    }
+
+    #[inline]
+    unsafe fn store(&self, ptr: *mut f64) {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            _mm512_store_pd(ptr, self.0)
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            _mm256_store_pd(ptr, self.0);
+            _mm256_store_pd(ptr.add(4), self.1);
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            vst1q_f64_x2(ptr, self.0);
+            vst1q_f64_x2(ptr.add(4), self.1);
+        }
+        #[cfg(target_arch = "loongarch64")]
+        {
+            lasx_xvst::<0>(transmute(self.0), transmute(ptr));
+            lasx_xvst::<32>(transmute(self.1), transmute(ptr));
+        }
+    }
+
+    #[inline]
+    unsafe fn store_unaligned(&self, ptr: *mut f64) {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            _mm512_storeu_pd(ptr, self.0)
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            _mm256_storeu_pd(ptr, self.0);
+            _mm256_storeu_pd(ptr.add(4), self.1);
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            vst1q_f64_x2(ptr, self.0);
+            vst1q_f64_x2(ptr.add(4), self.1);
+        }
+        #[cfg(target_arch = "loongarch64")]
+        {
+            lasx_xvst::<0>(transmute(self.0), transmute(ptr));
+            lasx_xvst::<32>(transmute(self.1), transmute(ptr));
+        }
+    }
+
+    fn reduce_sum(&self) -> f64 {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            _mm512_mask_reduce_add_pd(0xFF, self.0)
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            let sum = _mm256_add_pd(self.0, self.1);
+            let hi = _mm256_permute2f128_pd(sum, sum, 1);
+            let sum = _mm256_add_pd(sum, hi);
+            let sum = _mm256_hadd_pd(sum, sum);
+            _mm256_cvtsd_f64(sum)
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            let sum0 = vaddq_f64(self.0.0, self.0.1);
+            let sum1 = vaddq_f64(self.1.0, self.1.1);
+            let sum = vaddq_f64(sum0, sum1);
+            vaddvq_f64(sum)
+        }
+        #[cfg(target_arch = "loongarch64")]
+        {
+            self.as_array().iter().sum()
+        }
+    }
+
+    #[inline]
+    fn reduce_min(&self) -> f64 {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            _mm512_mask_reduce_min_pd(0xFF, self.0)
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            let m = _mm256_min_pd(self.0, self.1);
+            let hi = _mm256_permute2f128_pd(m, m, 1);
+            let m = _mm256_min_pd(m, hi);
+            let shuf = _mm256_permute_pd(m, 0b0101);
+            let m = _mm256_min_pd(m, shuf);
+            _mm256_cvtsd_f64(m)
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            let m0 = vminq_f64(self.0.0, self.0.1);
+            let m1 = vminq_f64(self.1.0, self.1.1);
+            let m = vminq_f64(m0, m1);
+            vminvq_f64(m)
+        }
+        #[cfg(target_arch = "loongarch64")]
+        {
+            self.as_array()
+                .iter()
+                .copied()
+                .fold(f64::INFINITY, f64::min)
+        }
+    }
+
+    #[inline]
+    fn min(&self, rhs: &Self) -> Self {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            Self(_mm512_min_pd(self.0, rhs.0))
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            Self(_mm256_min_pd(self.0, rhs.0), _mm256_min_pd(self.1, rhs.1))
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            Self(
+                float64x2x2_t(vminq_f64(self.0.0, rhs.0.0), vminq_f64(self.0.1, rhs.0.1)),
+                float64x2x2_t(vminq_f64(self.1.0, rhs.1.0), vminq_f64(self.1.1, rhs.1.1)),
+            )
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            Self(lasx_xvfmin_d(self.0, rhs.0), lasx_xvfmin_d(self.1, rhs.1))
+        }
+    }
+
+    fn find(&self, val: f64) -> Option<i32> {
+        unsafe {
+            for i in 0..8 {
+                if self.as_array().get_unchecked(i) == &val {
+                    return Some(i as i32);
+                }
+            }
+        }
+        None
+    }
+}
+
+impl FloatSimd<f64, 8> for f64x8 {
+    #[inline]
+    fn multiply_add(&mut self, a: Self, b: Self) {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            self.0 = _mm512_fmadd_pd(a.0, b.0, self.0)
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            self.0 = _mm256_fmadd_pd(a.0, b.0, self.0);
+            self.1 = _mm256_fmadd_pd(a.1, b.1, self.1);
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            self.0.0 = vfmaq_f64(self.0.0, a.0.0, b.0.0);
+            self.0.1 = vfmaq_f64(self.0.1, a.0.1, b.0.1);
+            self.1.0 = vfmaq_f64(self.1.0, a.1.0, b.1.0);
+            self.1.1 = vfmaq_f64(self.1.1, a.1.1, b.1.1);
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            self.0 = lasx_xvfmadd_d(a.0, b.0, self.0);
+            self.1 = lasx_xvfmadd_d(a.1, b.1, self.1);
+        }
+    }
+}
+
+impl Add for f64x8 {
+    type Output = Self;
+
+    #[inline]
+    fn add(self, rhs: Self) -> Self::Output {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            Self(_mm512_add_pd(self.0, rhs.0))
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            Self(_mm256_add_pd(self.0, rhs.0), _mm256_add_pd(self.1, rhs.1))
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            Self(
+                float64x2x2_t(vaddq_f64(self.0.0, rhs.0.0), vaddq_f64(self.0.1, rhs.0.1)),
+                float64x2x2_t(vaddq_f64(self.1.0, rhs.1.0), vaddq_f64(self.1.1, rhs.1.1)),
+            )
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            Self(lasx_xvfadd_d(self.0, rhs.0), lasx_xvfadd_d(self.1, rhs.1))
+        }
+    }
+}
+
+impl AddAssign for f64x8 {
+    #[inline]
+    fn add_assign(&mut self, rhs: Self) {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            self.0 = _mm512_add_pd(self.0, rhs.0)
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            self.0 = _mm256_add_pd(self.0, rhs.0);
+            self.1 = _mm256_add_pd(self.1, rhs.1);
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            self.0.0 = vaddq_f64(self.0.0, rhs.0.0);
+            self.0.1 = vaddq_f64(self.0.1, rhs.0.1);
+            self.1.0 = vaddq_f64(self.1.0, rhs.1.0);
+            self.1.1 = vaddq_f64(self.1.1, rhs.1.1);
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            self.0 = lasx_xvfadd_d(self.0, rhs.0);
+            self.1 = lasx_xvfadd_d(self.1, rhs.1);
+        }
+    }
+}
+
+impl Mul for f64x8 {
+    type Output = Self;
+
+    #[inline]
+    fn mul(self, rhs: Self) -> Self::Output {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            Self(_mm512_mul_pd(self.0, rhs.0))
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            Self(_mm256_mul_pd(self.0, rhs.0), _mm256_mul_pd(self.1, rhs.1))
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            Self(
+                float64x2x2_t(vmulq_f64(self.0.0, rhs.0.0), vmulq_f64(self.0.1, rhs.0.1)),
+                float64x2x2_t(vmulq_f64(self.1.0, rhs.1.0), vmulq_f64(self.1.1, rhs.1.1)),
+            )
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            Self(lasx_xvfmul_d(self.0, rhs.0), lasx_xvfmul_d(self.1, rhs.1))
+        }
+    }
+}
+
+impl Sub for f64x8 {
+    type Output = Self;
+
+    #[inline]
+    fn sub(self, rhs: Self) -> Self::Output {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            Self(_mm512_sub_pd(self.0, rhs.0))
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            Self(_mm256_sub_pd(self.0, rhs.0), _mm256_sub_pd(self.1, rhs.1))
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            Self(
+                float64x2x2_t(vsubq_f64(self.0.0, rhs.0.0), vsubq_f64(self.0.1, rhs.0.1)),
+                float64x2x2_t(vsubq_f64(self.1.0, rhs.1.0), vsubq_f64(self.1.1, rhs.1.1)),
+            )
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            Self(lasx_xvfsub_d(self.0, rhs.0), lasx_xvfsub_d(self.1, rhs.1))
+        }
+    }
+}
+
+impl SubAssign for f64x8 {
+    #[inline]
+    fn sub_assign(&mut self, rhs: Self) {
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
+        unsafe {
+            self.0 = _mm512_sub_pd(self.0, rhs.0)
+        }
+        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        unsafe {
+            self.0 = _mm256_sub_pd(self.0, rhs.0);
+            self.1 = _mm256_sub_pd(self.1, rhs.1);
+        }
+        #[cfg(target_arch = "aarch64")]
+        unsafe {
+            self.0.0 = vsubq_f64(self.0.0, rhs.0.0);
+            self.0.1 = vsubq_f64(self.0.1, rhs.0.1);
+            self.1.0 = vsubq_f64(self.1.0, rhs.1.0);
+            self.1.1 = vsubq_f64(self.1.1, rhs.1.1);
+        }
+        #[cfg(target_arch = "loongarch64")]
+        unsafe {
+            self.0 = lasx_xvfsub_d(self.0, rhs.0);
+            self.1 = lasx_xvfsub_d(self.1, rhs.1);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_f64x4_basic_ops() {
+        let a = [1.0_f64, 2.0, 3.0, 4.0];
+        let b = [5.0_f64, 6.0, 7.0, 8.0];
+
+        let simd_a: f64x4 = (&a).into();
+        let simd_b: f64x4 = (&b).into();
+
+        let sum = simd_a + simd_b;
+        assert_eq!(sum.as_array(), [6.0, 8.0, 10.0, 12.0]);
+
+        let product = simd_a * simd_b;
+        assert_eq!(product.as_array(), [5.0, 12.0, 21.0, 32.0]);
+
+        let diff = simd_b - simd_a;
+        assert_eq!(diff.as_array(), [4.0, 4.0, 4.0, 4.0]);
+
+        assert_eq!(simd_a.reduce_sum(), 10.0);
+        assert_eq!(simd_a.reduce_min(), 1.0);
+    }
+
+    #[test]
+    fn test_f64x4_fma() {
+        let a = [1.0_f64, 2.0, 3.0, 4.0];
+        let b = [2.0_f64, 3.0, 4.0, 5.0];
+
+        let simd_a: f64x4 = (&a).into();
+        let simd_b: f64x4 = (&b).into();
+        let mut acc = f64x4::zeros();
+        acc.multiply_add(simd_a, simd_b);
+        assert_eq!(acc.as_array(), [2.0, 6.0, 12.0, 20.0]);
+    }
+
+    #[test]
+    fn test_f64x4_min() {
+        let a = [1.0_f64, 5.0, 2.0, 8.0];
+        let b = [3.0_f64, 2.0, 4.0, 1.0];
+        let simd_a: f64x4 = (&a).into();
+        let simd_b: f64x4 = (&b).into();
+
+        let m = simd_a.min(&simd_b);
+        assert_eq!(m.as_array(), [1.0, 2.0, 2.0, 1.0]);
+        assert_eq!(m.reduce_min(), 1.0);
+    }
+
+    #[test]
+    fn test_f64x8_basic_ops() {
+        let a: [f64; 8] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
+        let b: [f64; 8] = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0];
+
+        let simd_a: f64x8 = (&a).into();
+        let simd_b: f64x8 = (&b).into();
+
+        let sum = simd_a + simd_b;
+        assert_eq!(
+            sum.as_array(),
+            [11.0, 22.0, 33.0, 44.0, 55.0, 66.0, 77.0, 88.0]
+        );
+
+        assert_eq!(simd_a.reduce_sum(), 36.0);
+        assert_eq!(simd_a.reduce_min(), 1.0);
+    }
+
+    #[test]
+    fn test_f64x8_fma() {
+        let a: [f64; 8] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
+        let b: [f64; 8] = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0];
+
+        let simd_a: f64x8 = (&a).into();
+        let simd_b: f64x8 = (&b).into();
+        let mut acc = f64x8::zeros();
+        acc.multiply_add(simd_a, simd_b);
+        assert_eq!(acc.as_array(), a);
+        assert_eq!(acc.reduce_sum(), 36.0);
+    }
+
+    #[test]
+    fn test_f64x8_min() {
+        let a: [f64; 8] = [5.0, 1.0, 8.0, 3.0, 9.0, 2.0, 7.0, 4.0];
+        let b: [f64; 8] = [2.0, 6.0, 3.0, 7.0, 1.0, 8.0, 4.0, 9.0];
+        let simd_a: f64x8 = (&a).into();
+        let simd_b: f64x8 = (&b).into();
+
+        let m = simd_a.min(&simd_b);
+        assert_eq!(m.as_array(), [2.0, 1.0, 3.0, 3.0, 1.0, 2.0, 4.0, 4.0]);
+        assert_eq!(m.reduce_min(), 1.0);
+
+        // Test with negative values
+        let c: [f64; 8] = [-1.0, 5.0, 3.0, 7.0, 2.0, 4.0, 6.0, -3.0];
+        let simd_c: f64x8 = (&c).into();
+        assert_eq!(simd_c.reduce_min(), -3.0);
+    }
+}
diff --git a/rust/lance-namespace-impls/Cargo.toml b/rust/lance-namespace-impls/Cargo.toml
index 8c84e1bbe8b..80c6ec4cb4d 100644
--- a/rust/lance-namespace-impls/Cargo.toml
+++ b/rust/lance-namespace-impls/Cargo.toml
@@ -43,6 +43,7 @@ reqwest = { version = "0.12", optional = true, default-features = false, feature
 url = { workspace = true }
 lance = { workspace = true }
 lance-index = { workspace = true }
+lance-linalg = { workspace = true }
 lance-io = { workspace = true }
 lance-table = { workspace = true }
 object_store = { workspace = true }
@@ -68,7 +69,7 @@ rand.workspace = true
 chrono.workspace = true
 
 # AWS credential vending dependencies (optional, enabled by "credential-vendor-aws" feature)
-aws-sdk-sts = { version = "1.38.0", optional = true }
+aws-sdk-sts = { version = "1.38.0", optional = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
 aws-config = { workspace = true, optional = true }
 aws-credential-types = { workspace = true, optional = true }
 sha2 = { version = "0.10", optional = true }
diff --git a/rust/lance-namespace-impls/src/connect.rs b/rust/lance-namespace-impls/src/connect.rs
index a9bb6dff8e7..c44eb2de219 100644
--- a/rust/lance-namespace-impls/src/connect.rs
+++ b/rust/lance-namespace-impls/src/connect.rs
@@ -7,8 +7,9 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use lance::session::Session;
-use lance_core::{Error, Result};
+use lance_core::Result;
 use lance_namespace::LanceNamespace;
+use lance_namespace::error::NamespaceError;
 
 use crate::context::DynamicContextProvider;
 
@@ -186,9 +187,11 @@ impl ConnectBuilder {
                 Ok(Arc::new(builder.build()) as Arc<dyn LanceNamespace>)
             }
             #[cfg(not(feature = "rest"))]
-            "rest" => Err(Error::namespace_source(
-                "REST namespace implementation requires 'rest' feature to be enabled".into(),
-            )),
+            "rest" => Err(NamespaceError::Unsupported {
+                message: "REST namespace implementation requires 'rest' feature to be enabled"
+                    .to_string(),
+            }
+            .into()),
             "dir" => {
                 // Create directory implementation (always available)
                 let mut builder = crate::dir::DirectoryNamespaceBuilder::from_properties(
@@ -203,14 +206,14 @@ impl ConnectBuilder {
                     .await
                     .map(|ns| Arc::new(ns) as Arc<dyn LanceNamespace>)
             }
-            _ => Err(Error::namespace_source(
-                format!(
+            _ => Err(NamespaceError::Unsupported {
+                message: format!(
                     "Implementation '{}' is not available. Supported: dir{}",
                     self.impl_name,
                     if cfg!(feature = "rest") { ", rest" } else { "" }
-                )
-                .into(),
-            )),
+                ),
+            }
+            .into()),
         }
     }
 }
diff --git a/rust/lance-namespace-impls/src/credentials.rs b/rust/lance-namespace-impls/src/credentials.rs
index eeb83efd6b7..a66489ac441 100644
--- a/rust/lance-namespace-impls/src/credentials.rs
+++ b/rust/lance-namespace-impls/src/credentials.rs
@@ -353,7 +353,7 @@ pub fn detect_provider_from_uri(uri: &str) -> &'static str {
     match url.scheme() {
         "s3" => "aws",
         "gs" => "gcp",
-        "az" => "azure",
+        "az" | "abfss" => "azure",
         _ => "unknown",
     }
 }
@@ -440,7 +440,12 @@ pub async fn create_credential_vendor_for_location(
 }
 
 /// Parse permission from properties, defaulting to Read
-#[allow(dead_code)]
+#[cfg(any(
+    test,
+    feature = "credential-vendor-aws",
+    feature = "credential-vendor-azure",
+    feature = "credential-vendor-gcp"
+))]
 fn parse_permission(properties: &HashMap<String, String>) -> VendedPermission {
     properties
         .get(PERMISSION)
@@ -449,7 +454,12 @@ fn parse_permission(properties: &HashMap<String, String>) -> VendedPermission {
 }
 
 /// Parse duration from properties using a vendor-specific key, defaulting to DEFAULT_CREDENTIAL_DURATION_MILLIS
-#[allow(dead_code)]
+#[cfg(any(
+    test,
+    feature = "credential-vendor-aws",
+    feature = "credential-vendor-azure",
+    feature = "credential-vendor-gcp"
+))]
 fn parse_duration_millis(properties: &HashMap<String, String>, key: &str) -> u64 {
     properties
         .get(key)
@@ -462,13 +472,14 @@ async fn create_aws_vendor(
     properties: &HashMap<String, String>,
 ) -> Result<Option<Box<dyn CredentialVendor>>> {
     use aws::{AwsCredentialVendor, AwsCredentialVendorConfig};
-    use lance_core::Error;
+    use lance_namespace::error::NamespaceError;
 
     // AWS requires role_arn to be configured
     let role_arn = properties.get(aws_props::ROLE_ARN).ok_or_else(|| {
-        Error::invalid_input_source(
-            "AWS credential vending requires 'credential_vendor.aws_role_arn' to be set".into(),
-        )
+        lance_core::Error::from(NamespaceError::InvalidInput {
+            message: "AWS credential vending requires 'credential_vendor.aws_role_arn' to be set"
+                .to_string(),
+        })
     })?;
 
     let duration_millis = parse_duration_millis(properties, aws_props::DURATION_MILLIS);
@@ -516,14 +527,15 @@ fn create_azure_vendor(
     properties: &HashMap<String, String>,
 ) -> Result<Option<Box<dyn CredentialVendor>>> {
     use azure::{AzureCredentialVendor, AzureCredentialVendorConfig};
-    use lance_core::Error;
+    use lance_namespace::error::NamespaceError;
 
     // Azure requires account_name to be configured
     let account_name = properties.get(azure_props::ACCOUNT_NAME).ok_or_else(|| {
-        Error::invalid_input_source(
-            "Azure credential vending requires 'credential_vendor.azure_account_name' to be set"
-                .into(),
-        )
+        lance_core::Error::from(NamespaceError::InvalidInput {
+            message:
+                "Azure credential vending requires 'credential_vendor.azure_account_name' to be set"
+                    .to_string(),
+        })
     })?;
 
     let duration_millis = parse_duration_millis(properties, azure_props::DURATION_MILLIS);
@@ -556,8 +568,16 @@ mod tests {
         assert_eq!(detect_provider_from_uri("gs://bucket/path"), "gcp");
         assert_eq!(detect_provider_from_uri("GS://bucket/path"), "gcp");
 
-        // Azure (supported scheme: az://)
+        // Azure (supported schemes: az:// and abfss://)
         assert_eq!(detect_provider_from_uri("az://container/path"), "azure");
+        assert_eq!(
+            detect_provider_from_uri("az://container@account.blob.core.windows.net/path"),
+            "azure"
+        );
+        assert_eq!(
+            detect_provider_from_uri("abfss://container@account.dfs.core.windows.net/path"),
+            "azure"
+        );
 
         // Unknown (unsupported schemes)
         assert_eq!(detect_provider_from_uri("/local/path"), "unknown");
@@ -565,10 +585,6 @@ mod tests {
         assert_eq!(detect_provider_from_uri("memory://test"), "unknown");
         // Hadoop-style schemes not supported by lance-io
         assert_eq!(detect_provider_from_uri("s3a://bucket/path"), "unknown");
-        assert_eq!(
-            detect_provider_from_uri("abfss://container@account.dfs.core.windows.net/path"),
-            "unknown"
-        );
         assert_eq!(
             detect_provider_from_uri("wasbs://container@account.blob.core.windows.net/path"),
             "unknown"
diff --git a/rust/lance-namespace-impls/src/credentials/aws.rs b/rust/lance-namespace-impls/src/credentials/aws.rs
index dda738d505b..10035306969 100644
--- a/rust/lance-namespace-impls/src/credentials/aws.rs
+++ b/rust/lance-namespace-impls/src/credentials/aws.rs
@@ -12,8 +12,9 @@ use async_trait::async_trait;
 use aws_config::BehaviorVersion;
 use aws_sdk_sts::Client as StsClient;
 use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
-use lance_core::{Error, Result};
+use lance_core::Result;
 use lance_io::object_store::uri_to_url;
+use lance_namespace::error::NamespaceError;
 use lance_namespace::models::Identity;
 use log::{debug, info, warn};
 use sha2::{Digest, Sha256};
@@ -167,7 +168,9 @@ impl AwsCredentialVendor {
         let bucket = url
             .host_str()
             .ok_or_else(|| {
-                Error::invalid_input_source(format!("S3 URI '{}' missing bucket", uri).into())
+                lance_core::Error::from(NamespaceError::InvalidInput {
+                    message: format!("S3 URI '{}' missing bucket", uri),
+                })
             })?
             .to_string();
 
@@ -325,9 +328,9 @@ impl AwsCredentialVendor {
         permission: VendedPermission,
     ) -> Result<VendedCredentials> {
         let credentials = credentials.ok_or_else(|| {
-            Error::io_source(Box::new(std::io::Error::other(
-                "STS response missing credentials",
-            )))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: "STS response missing credentials".to_string(),
+            })
         })?;
 
         let access_key_id = credentials.access_key_id().to_string();
@@ -391,10 +394,12 @@ impl AwsCredentialVendor {
             .send()
             .await
             .map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "AssumeRoleWithWebIdentity failed for role '{}': {}",
-                    self.config.role_arn, e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "AssumeRoleWithWebIdentity failed for role '{}': {}",
+                        self.config.role_arn, e
+                    ),
+                })
             })?;
 
         self.extract_credentials(
@@ -413,9 +418,10 @@ impl AwsCredentialVendor {
         api_key: &str,
     ) -> Result<VendedCredentials> {
         let salt = self.config.api_key_salt.as_ref().ok_or_else(|| {
-            Error::invalid_input_source(
-                "api_key_salt must be configured to use API key authentication".into(),
-            )
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "api_key_salt must be configured to use API key authentication"
+                    .to_string(),
+            })
         })?;
 
         let key_hash = Self::hash_api_key(api_key, salt);
@@ -431,7 +437,9 @@ impl AwsCredentialVendor {
                     "Invalid API key: hash {} not found in permissions map",
                     &key_hash[..8]
                 );
-                Error::invalid_input_source("Invalid API key".into())
+                lance_core::Error::from(NamespaceError::InvalidInput {
+                    message: "Invalid API key".to_string(),
+                })
             })?;
 
         let policy = Self::build_policy(bucket, prefix, permission);
@@ -453,10 +461,12 @@ impl AwsCredentialVendor {
             .external_id(&key_hash); // Use hash as external_id
 
         let response = request.send().await.map_err(|e| {
-            Error::io_source(Box::new(std::io::Error::other(format!(
-                "AssumeRole with API key failed for role '{}': {}",
-                self.config.role_arn, e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!(
+                    "AssumeRole with API key failed for role '{}': {}",
+                    self.config.role_arn, e
+                ),
+            })
         })?;
 
         self.extract_credentials(response.credentials(), bucket, prefix, permission)
@@ -496,10 +506,12 @@ impl AwsCredentialVendor {
         }
 
         let response = request.send().await.map_err(|e| {
-            Error::io_source(Box::new(std::io::Error::other(format!(
-                "AssumeRole failed for role '{}': {}",
-                self.config.role_arn, e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!(
+                    "AssumeRole failed for role '{}': {}",
+                    self.config.role_arn, e
+                ),
+            })
         })?;
 
         self.extract_credentials(
@@ -546,9 +558,11 @@ impl CredentialVendor for AwsCredentialVendor {
             }
             Some(_) => {
                 // Identity provided but neither api_key nor auth_token set
-                Err(Error::invalid_input_source(
-                    "Identity provided but neither api_key nor auth_token is set".into(),
-                ))
+                Err(NamespaceError::InvalidInput {
+                    message: "Identity provided but neither api_key nor auth_token is set"
+                        .to_string(),
+                }
+                .into())
             }
             None => {
                 // Use AssumeRole with static configuration
diff --git a/rust/lance-namespace-impls/src/credentials/azure.rs b/rust/lance-namespace-impls/src/credentials/azure.rs
index 2d1b1b5eab8..22620ec98b0 100644
--- a/rust/lance-namespace-impls/src/credentials/azure.rs
+++ b/rust/lance-namespace-impls/src/credentials/azure.rs
@@ -16,8 +16,9 @@ use azure_storage::prelude::*;
 use azure_storage::shared_access_signature::service_sas::{BlobSharedAccessSignature, SasKey};
 use azure_storage_blobs::prelude::*;
 use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
-use lance_core::{Error, Result};
+use lance_core::Result;
 use lance_io::object_store::uri_to_url;
+use lance_namespace::error::NamespaceError;
 use lance_namespace::models::Identity;
 use log::{debug, info, warn};
 use sha2::{Digest, Sha256};
@@ -237,10 +238,9 @@ impl AzureCredentialVendor {
         let credential =
             DefaultAzureCredential::create(azure_identity::TokenCredentialOptions::default())
                 .map_err(|e| {
-                    Error::io_source(Box::new(std::io::Error::other(format!(
-                        "Failed to create Azure credentials: {}",
-                        e
-                    ))))
+                    lance_core::Error::from(NamespaceError::Internal {
+                        message: format!("Failed to create Azure credentials: {}", e),
+                    })
                 })?;
 
         let credential: Arc<dyn TokenCredential> = Arc::new(credential);
@@ -265,10 +265,12 @@ impl AzureCredentialVendor {
             .get_user_deligation_key(now, key_end_time)
             .await
             .map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to get user delegation key for account '{}': {}",
-                    account, e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to get user delegation key for account '{}': {}",
+                        account, e
+                    ),
+                })
             })?;
 
         let permissions = Self::build_sas_permissions(self.config.permission);
@@ -283,20 +285,21 @@ impl AzureCredentialVendor {
             )
             .await
             .map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to generate SAS token for container '{}': {}",
-                    container, e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to generate SAS token for container '{}': {}",
+                        container, e
+                    ),
+                })
             })?;
 
         let expires_at_millis =
             (end_time.unix_timestamp() * 1000 + end_time.millisecond() as i64) as u64;
 
         let token = sas_token.token().map_err(|e| {
-            Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to get SAS token: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to get SAS token: {}", e),
+            })
         })?;
 
         Ok((token, expires_at_millis))
@@ -312,10 +315,9 @@ impl AzureCredentialVendor {
         let credential =
             DefaultAzureCredential::create(azure_identity::TokenCredentialOptions::default())
                 .map_err(|e| {
-                    Error::io_source(Box::new(std::io::Error::other(format!(
-                        "Failed to create Azure credentials: {}",
-                        e
-                    ))))
+                    lance_core::Error::from(NamespaceError::Internal {
+                        message: format!("Failed to create Azure credentials: {}", e),
+                    })
                 })?;
 
         let credential: Arc<dyn TokenCredential> = Arc::new(credential);
@@ -336,10 +338,12 @@ impl AzureCredentialVendor {
             .get_user_deligation_key(now, key_end_time)
             .await
             .map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to get user delegation key for account '{}': {}",
-                    account, e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to get user delegation key for account '{}': {}",
+                        account, e
+                    ),
+                })
             })?;
 
         let permissions = Self::build_sas_permissions(permission);
@@ -352,20 +356,21 @@ impl AzureCredentialVendor {
             )
             .await
             .map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to generate SAS token for container '{}': {}",
-                    container, e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to generate SAS token for container '{}': {}",
+                        container, e
+                    ),
+                })
             })?;
 
         let expires_at_millis =
             (end_time.unix_timestamp() * 1000 + end_time.millisecond() as i64) as u64;
 
         let token = sas_token.token().map_err(|e| {
-            Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to get SAS token: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to get SAS token: {}", e),
+            })
         })?;
 
         Ok((token, expires_at_millis))
@@ -391,10 +396,9 @@ impl AzureCredentialVendor {
         let credential =
             DefaultAzureCredential::create(azure_identity::TokenCredentialOptions::default())
                 .map_err(|e| {
-                    Error::io_source(Box::new(std::io::Error::other(format!(
-                        "Failed to create Azure credentials: {}",
-                        e
-                    ))))
+                    lance_core::Error::from(NamespaceError::Internal {
+                        message: format!("Failed to create Azure credentials: {}", e),
+                    })
                 })?;
 
         let credential: Arc<dyn TokenCredential> = Arc::new(credential);
@@ -415,10 +419,12 @@ impl AzureCredentialVendor {
             .get_user_deligation_key(now, key_end_time)
             .await
             .map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to get user delegation key for account '{}': {}",
-                    account, e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to get user delegation key for account '{}': {}",
+                        account, e
+                    ),
+                })
             })?;
 
         // Normalize path: remove leading/trailing slashes
@@ -448,10 +454,9 @@ impl AzureCredentialVendor {
         .signed_directory_depth(depth as u8);
 
         let token = sas.token().map_err(|e| {
-            Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to generate directory SAS token: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to generate directory SAS token: {}", e),
+            })
         })?;
 
         let expires_at_millis =
@@ -472,15 +477,16 @@ impl AzureCredentialVendor {
     /// 2. The OIDC token's issuer and subject to match the Federated Credential configuration
     async fn exchange_oidc_for_azure_token(&self, oidc_token: &str) -> Result<String> {
         let tenant_id = self.config.tenant_id.as_ref().ok_or_else(|| {
-            Error::invalid_input_source(
-                "azure_tenant_id must be configured for OIDC token exchange".into(),
-            )
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "azure_tenant_id must be configured for OIDC token exchange".to_string(),
+            })
         })?;
 
         let client_id = self.config.federated_client_id.as_ref().ok_or_else(|| {
-            Error::invalid_input_source(
-                "azure_federated_client_id must be configured for OIDC token exchange".into(),
-            )
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "azure_federated_client_id must be configured for OIDC token exchange"
+                    .to_string(),
+            })
         })?;
 
         let token_url = format!(
@@ -506,26 +512,27 @@ impl AzureCredentialVendor {
             .send()
             .await
             .map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to exchange OIDC token for Azure AD token: {}",
-                    e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to exchange OIDC token for Azure AD token: {}", e),
+                })
             })?;
 
         if !response.status().is_success() {
             let status = response.status();
             let body = response.text().await.unwrap_or_default();
-            return Err(Error::io_source(Box::new(std::io::Error::other(format!(
-                "Azure AD token exchange failed with status {}: {}",
-                status, body
-            )))));
+            return Err(NamespaceError::Internal {
+                message: format!(
+                    "Azure AD token exchange failed with status {}: {}",
+                    status, body
+                ),
+            }
+            .into());
         }
 
         let token_response: serde_json::Value = response.json().await.map_err(|e| {
-            Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to parse Azure AD token response: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to parse Azure AD token response: {}", e),
+            })
         })?;
 
         token_response
@@ -533,9 +540,9 @@ impl AzureCredentialVendor {
             .and_then(|v| v.as_str())
             .map(|s| s.to_string())
             .ok_or_else(|| {
-                Error::io_source(Box::new(std::io::Error::other(
-                    "Azure AD token response missing access_token",
-                )))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: "Azure AD token response missing access_token".to_string(),
+                })
             })
     }
 
@@ -571,10 +578,12 @@ impl AzureCredentialVendor {
             .get_user_deligation_key(now, key_end_time)
             .await
             .map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to get user delegation key with federated token: {}",
-                    e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to get user delegation key with federated token: {}",
+                        e
+                    ),
+                })
             })?;
 
         let permissions = Self::build_sas_permissions(permission);
@@ -594,17 +603,18 @@ impl AzureCredentialVendor {
                 )
                 .await
                 .map_err(|e| {
-                    Error::io_source(Box::new(std::io::Error::other(format!(
-                        "Failed to generate SAS token with federated token: {}",
-                        e
-                    ))))
+                    lance_core::Error::from(NamespaceError::Internal {
+                        message: format!(
+                            "Failed to generate SAS token with federated token: {}",
+                            e
+                        ),
+                    })
                 })?;
 
             sas_token.token().map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to get SAS token: {}",
-                    e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to get SAS token: {}", e),
+                })
             })?
         } else {
             // Directory-scoped SAS
@@ -622,10 +632,12 @@ impl AzureCredentialVendor {
             .signed_directory_depth(depth as u8);
 
             sas.token().map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to generate directory SAS token with federated token: {}",
-                    e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to generate directory SAS token with federated token: {}",
+                        e
+                    ),
+                })
             })?
         };
 
@@ -694,9 +706,10 @@ impl AzureCredentialVendor {
         api_key: &str,
     ) -> Result<VendedCredentials> {
         let salt = self.config.api_key_salt.as_ref().ok_or_else(|| {
-            Error::invalid_input_source(
-                "api_key_salt must be configured to use API key authentication".into(),
-            )
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "api_key_salt must be configured to use API key authentication"
+                    .to_string(),
+            })
         })?;
 
         let key_hash = Self::hash_api_key(api_key, salt);
@@ -712,7 +725,9 @@ impl AzureCredentialVendor {
                     "Invalid API key: hash {} not found in permissions map",
                     &key_hash[..8]
                 );
-                Error::invalid_input_source("Invalid API key".into())
+                lance_core::Error::from(NamespaceError::InvalidInput {
+                    message: "Invalid API key".to_string(),
+                })
             })?;
 
         debug!(
@@ -806,9 +821,9 @@ impl CredentialVendor for AzureCredentialVendor {
         let url = uri_to_url(table_location)?;
 
         let container = url.host_str().ok_or_else(|| {
-            Error::invalid_input_source(
-                format!("Azure URI '{}' missing container", table_location).into(),
-            )
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: format!("Azure URI '{}' missing container", table_location),
+            })
         })?;
 
         // Extract path for directory-scoped SAS
@@ -818,7 +833,7 @@ impl CredentialVendor for AzureCredentialVendor {
             self.config
                 .account_name
                 .as_ref()
-                .ok_or_else(|| Error::invalid_input_source("Azure credential vending requires 'credential_vendor.azure_account_name' to be set in configuration".into()))?;
+                .ok_or_else(|| lance_core::Error::from(NamespaceError::InvalidInput { message: "Azure credential vending requires 'credential_vendor.azure_account_name' to be set in configuration".to_string() }))?;
 
         // Dispatch based on identity
         match identity {
@@ -832,9 +847,10 @@ impl CredentialVendor for AzureCredentialVendor {
                 self.vend_with_api_key(account, container, path, api_key)
                     .await
             }
-            Some(_) => Err(Error::invalid_input_source(
-                "Identity provided but neither auth_token nor api_key is set".into(),
-            )),
+            Some(_) => Err(NamespaceError::InvalidInput {
+                message: "Identity provided but neither auth_token nor api_key is set".to_string(),
+            }
+            .into()),
             None => {
                 // Static credential vending using DefaultAzureCredential
                 // Use directory-scoped SAS when path is provided, container-level otherwise
diff --git a/rust/lance-namespace-impls/src/credentials/gcp.rs b/rust/lance-namespace-impls/src/credentials/gcp.rs
index 7f33e4b663d..d4fe3d9a9d8 100644
--- a/rust/lance-namespace-impls/src/credentials/gcp.rs
+++ b/rust/lance-namespace-impls/src/credentials/gcp.rs
@@ -46,8 +46,9 @@ use std::collections::HashMap;
 use async_trait::async_trait;
 use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
 use google_cloud_auth::credentials;
-use lance_core::{Error, Result};
+use lance_core::Result;
 use lance_io::object_store::uri_to_url;
+use lance_namespace::error::NamespaceError;
 use lance_namespace::models::Identity;
 use log::{debug, info, warn};
 use reqwest::Client;
@@ -249,10 +250,9 @@ impl GcpCredentialVendor {
         let credential = credentials::create_access_token_credential()
             .await
             .map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to create GCP credentials: {}",
-                    e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to create GCP credentials: {}", e),
+                })
             })?;
 
         Ok(Self {
@@ -267,19 +267,21 @@ impl GcpCredentialVendor {
         let url = uri_to_url(uri)?;
 
         if url.scheme() != "gs" {
-            return Err(Error::invalid_input_source(
-                format!(
+            return Err(NamespaceError::InvalidInput {
+                message: format!(
                     "Unsupported GCS URI scheme '{}', expected 'gs'",
                     url.scheme()
-                )
-                .into(),
-            ));
+                ),
+            }
+            .into());
         }
 
         let bucket = url
             .host_str()
             .ok_or_else(|| {
-                Error::invalid_input_source(format!("GCS URI '{}' missing bucket", uri).into())
+                lance_core::Error::from(NamespaceError::InvalidInput {
+                    message: format!("GCS URI '{}' missing bucket", uri),
+                })
             })?
             .to_string();
 
@@ -295,10 +297,9 @@ impl GcpCredentialVendor {
     /// directly.
     async fn get_source_token(&self) -> Result<String> {
         let base_token = self.credential.get_token().await.map_err(|e| {
-            Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to get GCP token: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to get GCP token: {}", e),
+            })
         })?;
 
         // If service account impersonation is configured, use generateAccessToken API
@@ -337,10 +338,9 @@ impl GcpCredentialVendor {
             .send()
             .await
             .map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to call IAM generateAccessToken: {}",
-                    e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to call IAM generateAccessToken: {}", e),
+                })
             })?;
 
         if !response.status().is_success() {
@@ -349,17 +349,19 @@ impl GcpCredentialVendor {
                 .text()
                 .await
                 .unwrap_or_else(|_| "unknown error".to_string());
-            return Err(Error::io_source(Box::new(std::io::Error::other(format!(
-                "IAM generateAccessToken failed for '{}' with status {}: {}",
-                service_account, status, body
-            )))));
+            return Err(NamespaceError::Internal {
+                message: format!(
+                    "IAM generateAccessToken failed for '{}' with status {}: {}",
+                    service_account, status, body
+                ),
+            }
+            .into());
         }
 
         let token_response: GenerateAccessTokenResponse = response.json().await.map_err(|e| {
-            Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to parse generateAccessToken response: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to parse generateAccessToken response: {}", e),
+            })
         })?;
 
         Ok(token_response.access_token)
@@ -445,10 +447,9 @@ impl GcpCredentialVendor {
         access_boundary: &CredentialAccessBoundary,
     ) -> Result<(String, u64)> {
         let options_json = serde_json::to_string(access_boundary).map_err(|e| {
-            Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to serialize access boundary: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to serialize access boundary: {}", e),
+            })
         })?;
 
         let params = [
@@ -475,10 +476,9 @@ impl GcpCredentialVendor {
             .send()
             .await
             .map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to call STS token exchange: {}",
-                    e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to call STS token exchange: {}", e),
+                })
             })?;
 
         if !response.status().is_success() {
@@ -487,17 +487,16 @@ impl GcpCredentialVendor {
                 .text()
                 .await
                 .unwrap_or_else(|_| "unknown error".to_string());
-            return Err(Error::io_source(Box::new(std::io::Error::other(format!(
-                "STS token exchange failed with status {}: {}",
-                status, body
-            )))));
+            return Err(NamespaceError::Internal {
+                message: format!("STS token exchange failed with status {}: {}", status, body),
+            }
+            .into());
         }
 
         let token_response: TokenExchangeResponse = response.json().await.map_err(|e| {
-            Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to parse STS response: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to parse STS response: {}", e),
+            })
         })?;
 
         // Calculate expiration time
@@ -592,10 +591,11 @@ impl GcpCredentialVendor {
             .workload_identity_provider
             .as_ref()
             .ok_or_else(|| {
-                Error::invalid_input_source(
-                    "gcp_workload_identity_provider must be configured for OIDC token exchange"
-                        .into(),
-                )
+                lance_core::Error::from(NamespaceError::InvalidInput {
+                    message:
+                        "gcp_workload_identity_provider must be configured for OIDC token exchange"
+                            .to_string(),
+                })
             })?;
 
         // Normalize audience to full format expected by GCP STS
@@ -624,26 +624,27 @@ impl GcpCredentialVendor {
             .send()
             .await
             .map_err(|e| {
-                Error::io_source(Box::new(std::io::Error::other(format!(
-                    "Failed to exchange OIDC token for GCP token: {}",
-                    e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to exchange OIDC token for GCP token: {}", e),
+                })
             })?;
 
         if !response.status().is_success() {
             let status = response.status();
             let body = response.text().await.unwrap_or_default();
-            return Err(Error::io_source(Box::new(std::io::Error::other(format!(
-                "GCP STS token exchange failed with status {}: {}",
-                status, body
-            )))));
+            return Err(NamespaceError::Internal {
+                message: format!(
+                    "GCP STS token exchange failed with status {}: {}",
+                    status, body
+                ),
+            }
+            .into());
         }
 
         let token_response: TokenExchangeResponse = response.json().await.map_err(|e| {
-            Error::io_source(Box::new(std::io::Error::other(format!(
-                "Failed to parse GCP STS token response: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to parse GCP STS token response: {}", e),
+            })
         })?;
 
         let federated_token = token_response.access_token;
@@ -706,9 +707,10 @@ impl GcpCredentialVendor {
         api_key: &str,
     ) -> Result<VendedCredentials> {
         let salt = self.config.api_key_salt.as_ref().ok_or_else(|| {
-            Error::invalid_input_source(
-                "api_key_salt must be configured to use API key authentication".into(),
-            )
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "api_key_salt must be configured to use API key authentication"
+                    .to_string(),
+            })
         })?;
 
         let key_hash = Self::hash_api_key(api_key, salt);
@@ -724,7 +726,9 @@ impl GcpCredentialVendor {
                     "Invalid API key: hash {} not found in permissions map",
                     &key_hash[..8]
                 );
-                Error::invalid_input_source("Invalid API key".into())
+                lance_core::Error::from(NamespaceError::InvalidInput {
+                    message: "Invalid API key".to_string(),
+                })
             })?;
 
         debug!(
@@ -790,9 +794,10 @@ impl CredentialVendor for GcpCredentialVendor {
                 let api_key = id.api_key.as_ref().unwrap();
                 self.vend_with_api_key(&bucket, &prefix, api_key).await
             }
-            Some(_) => Err(Error::invalid_input_source(
-                "Identity provided but neither auth_token nor api_key is set".into(),
-            )),
+            Some(_) => Err(NamespaceError::InvalidInput {
+                message: "Identity provided but neither auth_token nor api_key is set".to_string(),
+            }
+            .into()),
             None => {
                 // Static credential vending using ADC
                 let source_token = self.get_source_token().await?;
diff --git a/rust/lance-namespace-impls/src/dir.rs b/rust/lance-namespace-impls/src/dir.rs
index a9e5b4cc112..1dbfe2b6cfd 100644
--- a/rust/lance-namespace-impls/src/dir.rs
+++ b/rust/lance-namespace-impls/src/dir.rs
@@ -8,43 +8,97 @@
 
 pub mod manifest;
 
+use arrow::array::Float32Array;
 use arrow::record_batch::RecordBatchIterator;
 use arrow_ipc::reader::StreamReader;
 use async_trait::async_trait;
 use bytes::Bytes;
 use futures::TryStreamExt;
 use lance::dataset::builder::DatasetBuilder;
-use lance::dataset::{Dataset, WriteParams};
+use lance::dataset::scanner::Scanner;
+use lance::dataset::statistics::DatasetStatisticsExt;
+use lance::dataset::transaction::{Operation, Transaction};
+use lance::dataset::{Dataset, WriteMode, WriteParams};
+use lance::index::{DatasetIndexExt, IndexParams, vector::VectorIndexParams};
 use lance::session::Session;
+use lance_index::scalar::{
+    BuiltinIndexType, FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams,
+};
+use lance_index::vector::{
+    bq::RQBuildParams, hnsw::builder::HnswBuildParams, ivf::IvfBuildParams, pq::PQBuildParams,
+    sq::builder::SQBuildParams,
+};
+use lance_index::{IndexType, is_system_index};
 use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry};
+use lance_linalg::distance::MetricType;
 use lance_table::io::commit::ManifestNamingScheme;
 use object_store::path::Path;
 use object_store::{Error as ObjectStoreError, ObjectStore as OSObjectStore, PutMode, PutOptions};
 use std::collections::HashMap;
 use std::io::Cursor;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 
 use crate::context::DynamicContextProvider;
 use lance_namespace::models::{
-    BatchDeleteTableVersionsRequest, BatchDeleteTableVersionsResponse, CreateNamespaceRequest,
-    CreateNamespaceResponse, CreateTableRequest, CreateTableResponse, CreateTableVersionRequest,
+    AnalyzeTableQueryPlanRequest, BatchDeleteTableVersionsRequest,
+    BatchDeleteTableVersionsResponse, CountTableRowsRequest, CreateNamespaceRequest,
+    CreateNamespaceResponse, CreateTableIndexRequest, CreateTableIndexResponse, CreateTableRequest,
+    CreateTableResponse, CreateTableScalarIndexResponse, CreateTableVersionRequest,
     CreateTableVersionResponse, DeclareTableRequest, DeclareTableResponse,
-    DescribeNamespaceRequest, DescribeNamespaceResponse, DescribeTableRequest,
-    DescribeTableResponse, DescribeTableVersionRequest, DescribeTableVersionResponse,
-    DropNamespaceRequest, DropNamespaceResponse, DropTableRequest, DropTableResponse, Identity,
-    ListNamespacesRequest, ListNamespacesResponse, ListTableVersionsRequest,
-    ListTableVersionsResponse, ListTablesRequest, ListTablesResponse, NamespaceExistsRequest,
-    TableExistsRequest, TableVersion,
+    DescribeNamespaceRequest, DescribeNamespaceResponse, DescribeTableIndexStatsRequest,
+    DescribeTableIndexStatsResponse, DescribeTableRequest, DescribeTableResponse,
+    DescribeTableVersionRequest, DescribeTableVersionResponse, DescribeTransactionRequest,
+    DescribeTransactionResponse, DropNamespaceRequest, DropNamespaceResponse,
+    DropTableIndexRequest, DropTableIndexResponse, DropTableRequest, DropTableResponse,
+    ExplainTableQueryPlanRequest, FragmentStats, FragmentSummary, GetTableStatsRequest,
+    GetTableStatsResponse, Identity, IndexContent, InsertIntoTableRequest, InsertIntoTableResponse,
+    ListNamespacesRequest, ListNamespacesResponse, ListTableIndicesRequest,
+    ListTableIndicesResponse, ListTableVersionsRequest, ListTableVersionsResponse,
+    ListTablesRequest, ListTablesResponse, NamespaceExistsRequest, QueryTableRequest,
+    QueryTableRequestColumns, QueryTableRequestVector, RestoreTableRequest, RestoreTableResponse,
+    TableExistsRequest, TableVersion, UpdateTableSchemaMetadataRequest,
+    UpdateTableSchemaMetadataResponse,
 };
 
-use lance_core::{Error, Result, box_error};
+use lance_core::{Error, Result};
 use lance_namespace::LanceNamespace;
+use lance_namespace::error::NamespaceError;
 use lance_namespace::schema::arrow_schema_to_json;
 
 use crate::credentials::{
     CredentialVendor, create_credential_vendor_for_location, has_credential_vendor_config,
 };
 
+/// Thread-safe metrics tracker for namespace operations.
+///
+/// Tracks the count of each API operation when `ops_metrics_enabled` is true.
+/// Use `retrieve()` to get a snapshot of all operation counts.
+#[derive(Debug, Default)]
+pub struct OpsMetrics {
+    counters: Mutex<HashMap<String, u64>>,
+}
+
+impl OpsMetrics {
+    /// Increment the counter for an operation.
+    pub fn increment(&self, operation: &str) {
+        if let Ok(mut counters) = self.counters.lock() {
+            *counters.entry(operation.to_string()).or_insert(0) += 1;
+        }
+    }
+
+    /// Get a snapshot of all operation counts.
+    pub fn retrieve(&self) -> HashMap<String, u64> {
+        self.counters.lock().map(|c| c.clone()).unwrap_or_default()
+    }
+
+    /// Reset all counters to zero.
+    pub fn reset(&self) {
+        if let Ok(mut counters) = self.counters.lock() {
+            counters.clear();
+        }
+    }
+}
+
 /// Result of checking table status atomically.
 ///
 /// This struct captures the state of a table directory in a single snapshot,
@@ -58,6 +112,35 @@ pub(crate) struct TableStatus {
     pub(crate) has_reserved_file: bool,
 }
 
+enum DirectoryIndexParams {
+    Scalar {
+        index_type: IndexType,
+        params: ScalarIndexParams,
+    },
+    Inverted(InvertedIndexParams),
+    Vector {
+        index_type: IndexType,
+        params: VectorIndexParams,
+    },
+}
+
+impl DirectoryIndexParams {
+    fn index_type(&self) -> IndexType {
+        match self {
+            Self::Scalar { index_type, .. } | Self::Vector { index_type, .. } => *index_type,
+            Self::Inverted(_) => IndexType::Inverted,
+        }
+    }
+
+    fn params(&self) -> &dyn IndexParams {
+        match self {
+            Self::Scalar { params, .. } => params,
+            Self::Inverted(params) => params,
+            Self::Vector { params, .. } => params,
+        }
+    }
+}
+
 /// Builder for creating a DirectoryNamespace.
 ///
 /// This builder provides a fluent API for configuring and establishing
@@ -101,9 +184,27 @@ pub struct DirectoryNamespaceBuilder {
     dir_listing_enabled: bool,
     inline_optimization_enabled: bool,
     table_version_tracking_enabled: bool,
+    /// When true, table versions are stored in the `__manifest` table instead of
+    /// relying on Lance's native version management.
+    table_version_storage_enabled: bool,
+    /// When true, enables migration mode where the namespace checks the manifest first
+    /// before falling back to directory listing for root-level tables. When false (default),
+    /// root-level tables use directory listing directly without checking the manifest,
+    /// avoiding extra object store calls.
+    dir_listing_to_manifest_migration_enabled: bool,
     credential_vendor_properties: HashMap<String, String>,
     context_provider: Option<Arc<dyn DynamicContextProvider>>,
     commit_retries: Option<u32>,
+    /// When true, returns input storage options in describe_table/declare_table responses
+    /// when no credential vendor is configured. Useful for testing. Default: false.
+    vend_input_storage_options: bool,
+    /// When set, adds expires_at_millis to vended storage options. The value is calculated
+    /// as current_time_millis + this interval. This allows clients to know when to refresh
+    /// credentials by calling describe_table again. Only effective when vend_input_storage_options
+    /// is true.
+    vend_input_storage_options_refresh_interval_millis: Option<u64>,
+    /// When true, tracks operation metrics. Default: false.
+    ops_metrics_enabled: bool,
 }
 
 impl std::fmt::Debug for DirectoryNamespaceBuilder {
@@ -121,10 +222,27 @@ impl std::fmt::Debug for DirectoryNamespaceBuilder {
                 "table_version_tracking_enabled",
                 &self.table_version_tracking_enabled,
             )
+            .field(
+                "table_version_storage_enabled",
+                &self.table_version_storage_enabled,
+            )
+            .field(
+                "dir_listing_to_manifest_migration_enabled",
+                &self.dir_listing_to_manifest_migration_enabled,
+            )
             .field(
                 "context_provider",
                 &self.context_provider.as_ref().map(|_| "Some(...)"),
             )
+            .field(
+                "vend_input_storage_options",
+                &self.vend_input_storage_options,
+            )
+            .field(
+                "vend_input_storage_options_refresh_interval_millis",
+                &self.vend_input_storage_options_refresh_interval_millis,
+            )
+            .field("ops_metrics_enabled", &self.ops_metrics_enabled)
             .finish()
     }
 }
@@ -144,9 +262,14 @@ impl DirectoryNamespaceBuilder {
             dir_listing_enabled: true, // Default to enabled for backwards compatibility
             inline_optimization_enabled: true,
             table_version_tracking_enabled: false, // Default to disabled
+            table_version_storage_enabled: false,  // Default to disabled
+            dir_listing_to_manifest_migration_enabled: false, // Default to disabled
             credential_vendor_properties: HashMap::new(),
             context_provider: None,
             commit_retries: None,
+            vend_input_storage_options: false,
+            vend_input_storage_options_refresh_interval_millis: None,
+            ops_metrics_enabled: false,
         }
     }
 
@@ -168,6 +291,17 @@ impl DirectoryNamespaceBuilder {
         self
     }
 
+    /// Enable or disable migration mode from directory listing to manifest.
+    ///
+    /// When enabled, root-level table operations check the manifest first before
+    /// falling back to directory listing. When disabled (default), root-level tables
+    /// use directory listing directly, avoiding extra object store calls.
+    /// Only relevant when both `manifest_enabled` and `dir_listing_enabled` are true.
+    pub fn dir_listing_to_manifest_migration_enabled(mut self, enabled: bool) -> Self {
+        self.dir_listing_to_manifest_migration_enabled = enabled;
+        self
+    }
+
     /// Enable or disable inline optimization of the __manifest table.
     ///
     /// When enabled (default), performs compaction and indexing on the __manifest table
@@ -190,6 +324,19 @@ impl DirectoryNamespaceBuilder {
         self
     }
 
+    /// Enable or disable table version management through the `__manifest` table.
+    ///
+    /// When enabled, table versions are tracked as `table_version` entries in the
+    /// `__manifest` Lance table. This enables:
+    /// - Centralized version tracking instead of per-table `_versions/` directories
+    ///
+    /// Requires `manifest_enabled` to be true.
+    /// When disabled (default), version storage uses per-table storage operations.
+    pub fn table_version_storage_enabled(mut self, enabled: bool) -> Self {
+        self.table_version_storage_enabled = enabled;
+        self
+    }
+
     /// Create a DirectoryNamespaceBuilder from properties HashMap.
     ///
     /// This method parses a properties map into builder configuration.
@@ -260,9 +407,9 @@ impl DirectoryNamespaceBuilder {
     ) -> Result<Self> {
         // Extract root from properties (required)
         let root = properties.get("root").cloned().ok_or_else(|| {
-            Error::namespace_source(
-                "Missing required property 'root' for directory namespace".into(),
-            )
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Missing required property 'root' for directory namespace".to_string(),
+            })
         })?;
 
         // Extract storage options (properties prefixed with "storage.")
@@ -304,6 +451,18 @@ impl DirectoryNamespaceBuilder {
             .and_then(|v| v.parse::<bool>().ok())
             .unwrap_or(false);
 
+        // Extract table_version_storage_enabled (default: false)
+        let table_version_storage_enabled = properties
+            .get("table_version_storage_enabled")
+            .and_then(|v| v.parse::<bool>().ok())
+            .unwrap_or(false);
+
+        // Extract dir_listing_to_manifest_migration_enabled (default: false)
+        let dir_listing_to_manifest_migration_enabled = properties
+            .get("dir_listing_to_manifest_migration_enabled")
+            .and_then(|v| v.parse::<bool>().ok())
+            .unwrap_or(false);
+
         // Extract credential vendor properties (properties prefixed with "credential_vendor.")
         // The prefix is stripped to get short property names
         // The build() method will check if enabled=true before creating the vendor
@@ -319,6 +478,23 @@ impl DirectoryNamespaceBuilder {
             .get("commit_retries")
             .and_then(|v| v.parse::<u32>().ok());
 
+        // Extract vend_input_storage_options (default: false)
+        let vend_input_storage_options = properties
+            .get("vend_input_storage_options")
+            .and_then(|v| v.parse::<bool>().ok())
+            .unwrap_or(false);
+
+        // Extract vend_input_storage_options_refresh_interval_millis (optional)
+        let vend_input_storage_options_refresh_interval_millis = properties
+            .get("vend_input_storage_options_refresh_interval_millis")
+            .and_then(|v| v.parse::<u64>().ok());
+
+        // Extract ops_metrics_enabled (default: false)
+        let ops_metrics_enabled = properties
+            .get("ops_metrics_enabled")
+            .and_then(|v| v.parse::<bool>().ok())
+            .unwrap_or(false);
+
         Ok(Self {
             root: root.trim_end_matches('/').to_string(),
             storage_options,
@@ -327,9 +503,14 @@ impl DirectoryNamespaceBuilder {
             dir_listing_enabled,
             inline_optimization_enabled,
             table_version_tracking_enabled,
+            table_version_storage_enabled,
+            dir_listing_to_manifest_migration_enabled,
             credential_vendor_properties,
             context_provider: None,
             commit_retries,
+            vend_input_storage_options,
+            vend_input_storage_options_refresh_interval_millis,
+            ops_metrics_enabled,
         })
     }
 
@@ -442,6 +623,50 @@ impl DirectoryNamespaceBuilder {
         self
     }
 
+    /// Enable or disable returning input storage options in responses.
+    ///
+    /// When enabled, `describe_table` and `declare_table` will return the storage
+    /// options passed to the builder when no credential vendor is configured.
+    /// This is useful for testing scenarios where you want to pass storage options
+    /// through to clients.
+    ///
+    /// Default is false (storage options are not returned unless credential vending is configured).
+    pub fn vend_input_storage_options(mut self, enabled: bool) -> Self {
+        self.vend_input_storage_options = enabled;
+        self
+    }
+
+    /// Set the refresh interval for vended input storage options.
+    ///
+    /// When set, vended storage options will include an `expires_at_millis` field
+    /// calculated as `current_time_millis + interval_millis`. This allows clients
+    /// to know when to refresh credentials by calling `describe_table` again.
+    ///
+    /// This only has effect when `vend_input_storage_options` is enabled.
+    ///
+    /// # Arguments
+    ///
+    /// * `interval_millis` - The refresh interval in milliseconds
+    pub fn vend_input_storage_options_refresh_interval_millis(
+        mut self,
+        interval_millis: u64,
+    ) -> Self {
+        self.vend_input_storage_options_refresh_interval_millis = Some(interval_millis);
+        self
+    }
+
+    /// Enable or disable operation metrics tracking.
+    ///
+    /// When enabled, the namespace will track how many times each API operation
+    /// is called. Use `retrieve_ops_metrics()` on the built namespace to get
+    /// the current counts.
+    ///
+    /// Default is false.
+    pub fn ops_metrics_enabled(mut self, enabled: bool) -> Self {
+        self.ops_metrics_enabled = enabled;
+        self
+    }
+
     /// Build the DirectoryNamespace.
     ///
     /// # Returns
@@ -455,6 +680,14 @@ impl DirectoryNamespaceBuilder {
     /// - Connection to the storage backend fails
     /// - Storage options are invalid
     pub async fn build(self) -> Result<DirectoryNamespace> {
+        // Validate: table_version_storage_enabled requires manifest_enabled
+        if self.table_version_storage_enabled && !self.manifest_enabled {
+            return Err(NamespaceError::InvalidInput {
+                message: "table_version_storage_enabled requires manifest_enabled=true".to_string(),
+            }
+            .into());
+        }
+
         let (object_store, base_path) =
             Self::initialize_object_store(&self.root, &self.storage_options, &self.session).await?;
 
@@ -468,6 +701,7 @@ impl DirectoryNamespaceBuilder {
                 self.dir_listing_enabled,
                 self.inline_optimization_enabled,
                 self.commit_retries,
+                self.table_version_storage_enabled,
             )
             .await
             {
@@ -495,6 +729,12 @@ impl DirectoryNamespaceBuilder {
             None
         };
 
+        let ops_metrics = if self.ops_metrics_enabled {
+            Some(Arc::new(OpsMetrics::default()))
+        } else {
+            None
+        };
+
         Ok(DirectoryNamespace {
             root: self.root,
             storage_options: self.storage_options,
@@ -503,9 +743,16 @@ impl DirectoryNamespaceBuilder {
             base_path,
             manifest_ns,
             dir_listing_enabled: self.dir_listing_enabled,
+            dir_listing_to_manifest_migration_enabled: self
+                .dir_listing_to_manifest_migration_enabled,
             table_version_tracking_enabled: self.table_version_tracking_enabled,
+            table_version_storage_enabled: self.table_version_storage_enabled,
             credential_vendor,
             context_provider: self.context_provider,
+            vend_input_storage_options: self.vend_input_storage_options,
+            vend_input_storage_options_refresh_interval_millis: self
+                .vend_input_storage_options_refresh_interval_millis,
+            ops_metrics,
         })
     }
 
@@ -535,7 +782,9 @@ impl DirectoryNamespaceBuilder {
         let (object_store, base_path) = ObjectStore::from_uri_and_params(registry, root, &params)
             .await
             .map_err(|e| {
-                Error::namespace_source(format!("Failed to create object store: {}", e).into())
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to create object store: {:?}", e),
+                })
             })?;
 
         Ok((object_store, base_path))
@@ -568,15 +817,20 @@ impl DirectoryNamespaceBuilder {
 pub struct DirectoryNamespace {
     root: String,
     storage_options: Option<HashMap<String, String>>,
-    #[allow(dead_code)]
     session: Option<Arc<Session>>,
     object_store: Arc<ObjectStore>,
     base_path: Path,
     manifest_ns: Option<Arc<manifest::ManifestNamespace>>,
     dir_listing_enabled: bool,
+    /// When true, root-level table operations check the manifest first before
+    /// falling back to directory listing. When false, root-level tables skip
+    /// the manifest check and use directory listing directly.
+    dir_listing_to_manifest_migration_enabled: bool,
     /// When true, `describe_table` returns `managed_versioning: true` to indicate
     /// commits should go through namespace table version APIs.
     table_version_tracking_enabled: bool,
+    /// When true, table versions are stored in the `__manifest` table.
+    table_version_storage_enabled: bool,
     /// Credential vendor created once during initialization.
     /// Used to vend temporary credentials for table access.
     credential_vendor: Option<Arc<dyn CredentialVendor>>,
@@ -584,6 +838,13 @@ pub struct DirectoryNamespace {
     /// Stored but not directly used in operations (available for future extensions).
     #[allow(dead_code)]
     context_provider: Option<Arc<dyn DynamicContextProvider>>,
+    /// When true, returns input storage options in responses when no credential vendor is configured.
+    vend_input_storage_options: bool,
+    /// Refresh interval in milliseconds for vended input storage options.
+    /// When set, expires_at_millis is added to storage options.
+    vend_input_storage_options_refresh_interval_millis: Option<u64>,
+    /// Operation metrics tracker, created when ops_metrics_enabled is true.
+    ops_metrics: Option<Arc<OpsMetrics>>,
 }
 
 impl std::fmt::Debug for DirectoryNamespace {
@@ -598,6 +859,13 @@ impl std::fmt::Display for DirectoryNamespace {
     }
 }
 
+/// Describes the version ranges to delete for a single table.
+/// Used by `batch_delete_table_versions` and `delete_physical_version_files`.
+struct TableDeleteEntry {
+    table_id: Option<Vec<String>>,
+    ranges: Vec<(i64, i64)>,
+}
+
 impl DirectoryNamespace {
     /// Apply pagination to a list of table names
     ///
@@ -607,7 +875,15 @@ impl DirectoryNamespace {
     /// * `names` - The vector of table names to paginate
     /// * `page_token` - Skip items until finding one greater than this value (start_after semantics)
     /// * `limit` - Maximum number of items to keep
-    fn apply_pagination(names: &mut Vec<String>, page_token: Option<String>, limit: Option<i32>) {
+    ///
+    /// # Returns
+    /// The next page token (last item in this page) if more results exist beyond the limit,
+    /// or `None` if this is the last page.
+    fn apply_pagination(
+        names: &mut Vec<String>,
+        page_token: Option<String>,
+        limit: Option<i32>,
+    ) -> Option<String> {
         // Sort alphabetically for consistent ordering
         names.sort();
 
@@ -623,12 +899,23 @@ impl DirectoryNamespace {
             }
         }
 
-        // Apply limit
+        // Apply limit and compute next page token
         if let Some(limit) = limit
             && limit >= 0
         {
-            names.truncate(limit as usize);
+            let limit = limit as usize;
+            if names.len() > limit {
+                let next_page_token = if limit > 0 {
+                    Some(names[limit - 1].clone())
+                } else {
+                    None
+                };
+                names.truncate(limit);
+                return next_page_token;
+            }
         }
+
+        None
     }
 
     /// List tables using directory scanning (fallback method)
@@ -639,10 +926,9 @@ impl DirectoryNamespace {
             .read_dir(self.base_path.clone())
             .await
             .map_err(|e| {
-                Error::io_source(box_error(std::io::Error::other(format!(
-                    "Failed to list directory: {}",
-                    e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to list directory: {:?}", e),
+                })
             })?;
 
         for entry in entries {
@@ -670,10 +956,13 @@ impl DirectoryNamespace {
         if let Some(id) = id
             && !id.is_empty()
         {
-            return Err(Error::namespace_source(format!(
-                "Directory namespace only supports root namespace operations, but got namespace ID: {:?}. Expected empty ID.",
-                id
-            ).into()));
+            return Err(NamespaceError::Unsupported {
+                message: format!(
+                    "Directory namespace only supports root namespace operations, but got namespace ID: {:?}. Expected empty ID.",
+                    id
+                ),
+            }
+            .into());
         }
         Ok(())
     }
@@ -681,432 +970,108 @@ impl DirectoryNamespace {
     /// Extract table name from table ID
     fn table_name_from_id(id: &Option<Vec<String>>) -> Result<String> {
         let id = id.as_ref().ok_or_else(|| {
-            Error::namespace_source("Directory namespace table ID cannot be empty".into())
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Directory namespace table ID cannot be empty".to_string(),
+            })
         })?;
 
         if id.len() != 1 {
-            return Err(Error::namespace_source(format!(
-                "Multi-level table IDs are only supported when manifest mode is enabled, but got: {:?}",
-                id
-            )
-            .into()));
+            return Err(NamespaceError::Unsupported {
+                message: format!(
+                    "Multi-level table IDs are only supported when manifest mode is enabled, but got: {:?}",
+                    id
+                ),
+            }
+            .into());
         }
 
         Ok(id[0].clone())
     }
 
+    fn format_table_id(table_id: &[String]) -> String {
+        format!(
+            "table id '{}'",
+            manifest::ManifestNamespace::str_object_id(table_id)
+        )
+    }
+
+    fn format_table_id_from_request(id: &Option<Vec<String>>) -> String {
+        id.as_ref()
+            .map(|table_id| Self::format_table_id(table_id))
+            .unwrap_or_else(|| "table id '<unknown>'".to_string())
+    }
+
     async fn resolve_table_location(&self, id: &Option<Vec<String>>) -> Result<String> {
         let mut describe_req = DescribeTableRequest::new();
         describe_req.id = id.clone();
         describe_req.load_detailed_metadata = Some(false);
 
-        let describe_resp = self.describe_table(describe_req).await?;
+        // Use internal impl to avoid counting this as an external API call
+        let describe_resp = self.describe_table_impl(describe_req).await?;
 
         describe_resp.location.ok_or_else(|| {
-            Error::namespace_source(format!("Table location not found for: {:?}", id).into())
+            lance_core::Error::from(NamespaceError::TableNotFound {
+                message: format!("Table location not found for: {:?}", id),
+            })
         })
     }
 
-    fn table_full_uri(&self, table_name: &str) -> String {
-        format!("{}/{}.lance", &self.root, table_name)
-    }
-
-    fn uri_to_object_store_path(uri: &str) -> Path {
-        let path_str = if let Some(rest) = uri.strip_prefix("file://") {
-            rest
-        } else if let Some(rest) = uri.strip_prefix("s3://") {
-            rest.split_once('/').map(|(_, p)| p).unwrap_or(rest)
-        } else if let Some(rest) = uri.strip_prefix("gs://") {
-            rest.split_once('/').map(|(_, p)| p).unwrap_or(rest)
-        } else if let Some(rest) = uri.strip_prefix("az://") {
-            rest.split_once('/').map(|(_, p)| p).unwrap_or(rest)
-        } else {
-            uri
-        };
-        Path::from(path_str)
-    }
-
-    /// Get the object store path for a table (relative to base_path)
-    fn table_path(&self, table_name: &str) -> Path {
-        self.base_path
-            .child(format!("{}.lance", table_name).as_str())
-    }
-
-    /// Get the reserved file path for a table
-    fn table_reserved_file_path(&self, table_name: &str) -> Path {
-        self.base_path
-            .child(format!("{}.lance", table_name).as_str())
-            .child(".lance-reserved")
-    }
-
-    /// Get the deregistered marker file path for a table
-    fn table_deregistered_file_path(&self, table_name: &str) -> Path {
-        self.base_path
-            .child(format!("{}.lance", table_name).as_str())
-            .child(".lance-deregistered")
-    }
-
-    /// Atomically check table existence and deregistration status.
-    ///
-    /// This performs a single directory listing to get a consistent snapshot of the
-    /// table's state, avoiding race conditions between checking existence and
-    /// checking deregistration status.
-    pub(crate) async fn check_table_status(&self, table_name: &str) -> TableStatus {
-        let table_path = self.table_path(table_name);
-        match self.object_store.read_dir(table_path).await {
-            Ok(entries) => {
-                let exists = !entries.is_empty();
-                let is_deregistered = entries.iter().any(|e| e.ends_with(".lance-deregistered"));
-                let has_reserved_file = entries.iter().any(|e| e.ends_with(".lance-reserved"));
-                TableStatus {
-                    exists,
-                    is_deregistered,
-                    has_reserved_file,
+    /// Internal describe_table implementation that doesn't record metrics.
+    /// Used by both the public describe_table (which records metrics) and
+    /// internal callers like resolve_table_location (which shouldn't).
+    async fn describe_table_impl(
+        &self,
+        request: DescribeTableRequest,
+    ) -> Result<DescribeTableResponse> {
+        let is_root_level = request.id.as_ref().is_some_and(|id| id.len() == 1);
+        let skip_manifest_for_root = self.dir_listing_enabled
+            && is_root_level
+            && !self.dir_listing_to_manifest_migration_enabled;
+        if let Some(ref manifest_ns) = self.manifest_ns
+            && !skip_manifest_for_root
+        {
+            match manifest_ns.describe_table(request.clone()).await {
+                Ok(mut response) => {
+                    if let Some(ref table_uri) = response.table_uri {
+                        // For backwards compatibility, only skip vending credentials when explicitly set to false
+                        let vend = request.vend_credentials.unwrap_or(true);
+                        let identity = request.identity.as_deref();
+                        response.storage_options = self
+                            .get_storage_options_for_table(table_uri, vend, identity)
+                            .await?;
+                    }
+                    // Set managed_versioning flag when table_version_tracking_enabled
+                    if self.table_version_tracking_enabled {
+                        response.managed_versioning = Some(true);
+                    }
+                    return Ok(response);
+                }
+                Err(_) if self.dir_listing_enabled && is_root_level => {
+                    // Fall through to directory check only for single-level IDs
                 }
+                Err(e) => return Err(e),
             }
-            Err(_) => TableStatus {
-                exists: false,
-                is_deregistered: false,
-                has_reserved_file: false,
-            },
         }
-    }
 
-    async fn put_marker_file_atomic(
-        &self,
-        path: &Path,
-        file_description: &str,
-    ) -> std::result::Result<(), String> {
-        let put_opts = PutOptions {
-            mode: PutMode::Create,
-            ..Default::default()
-        };
+        let table_name = Self::table_name_from_id(&request.id)?;
+        let table_id = Self::format_table_id_from_request(&request.id);
+        let table_uri = self.table_full_uri(&table_name);
 
-        match self
-            .object_store
-            .inner
-            .put_opts(path, bytes::Bytes::new().into(), put_opts)
-            .await
-        {
-            Ok(_) => Ok(()),
-            Err(ObjectStoreError::AlreadyExists { .. })
-            | Err(ObjectStoreError::Precondition { .. }) => {
-                Err(format!("{} already exists", file_description))
+        // Atomically check table existence and deregistration status
+        let status = self.check_table_status(&table_name).await;
+
+        if !status.exists {
+            return Err(NamespaceError::TableNotFound {
+                message: table_id.clone(),
             }
-            Err(e) => Err(format!("Failed to create {}: {}", file_description, e)),
+            .into());
         }
-    }
 
-    /// Get storage options for a table, using credential vending if configured.
-    ///
-    /// If credential vendor properties are configured and the table location matches
-    /// a supported cloud provider, this will create an appropriate vendor and vend
-    /// temporary credentials scoped to the table location. Otherwise, returns the
-    /// static storage options.
-    ///
-    /// The vendor type is auto-selected based on the table URI:
-    /// - `s3://` locations use AWS STS AssumeRole
-    /// - `gs://` locations use GCP OAuth2 tokens
-    /// - `az://` locations use Azure SAS tokens
-    ///
-    /// The permission level (Read, Write, Admin) is configured at namespace
-    /// initialization time via the `credential_vendor_permission` property.
-    ///
-    /// # Arguments
-    ///
-    /// * `table_uri` - The full URI of the table
-    /// * `identity` - Optional identity from the request for identity-based credential vending
-    async fn get_storage_options_for_table(
-        &self,
-        table_uri: &str,
-        identity: Option<&Identity>,
-    ) -> Result<Option<HashMap<String, String>>> {
-        if let Some(ref vendor) = self.credential_vendor {
-            let vended = vendor.vend_credentials(table_uri, identity).await?;
-            return Ok(Some(vended.storage_options));
-        }
-        Ok(self.storage_options.clone())
-    }
-
-    /// Migrate directory-based tables to the manifest.
-    ///
-    /// This is a one-time migration operation that:
-    /// 1. Scans the directory for existing `.lance` tables
-    /// 2. Registers any unmigrated tables in the manifest
-    /// 3. Returns the count of tables that were migrated
-    ///
-    /// This method is safe to run multiple times - it will skip tables that are already
-    /// registered in the manifest.
-    ///
-    /// # Usage
-    ///
-    /// After creating tables in directory-only mode or dual mode, you can migrate them
-    /// to the manifest to enable manifest-only mode:
-    ///
-    /// ```no_run
-    /// # use lance_namespace_impls::DirectoryNamespaceBuilder;
-    /// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
-    /// // Create namespace with dual mode (manifest + directory listing)
-    /// let namespace = DirectoryNamespaceBuilder::new("/path/to/data")
-    ///     .manifest_enabled(true)
-    ///     .dir_listing_enabled(true)
-    ///     .build()
-    ///     .await?;
-    ///
-    /// // ... tables are created and used ...
-    ///
-    /// // Migrate existing directory tables to manifest
-    /// let migrated_count = namespace.migrate().await?;
-    /// println!("Migrated {} tables", migrated_count);
-    ///
-    /// // Now you can disable directory listing for better performance:
-    /// // (requires rebuilding the namespace)
-    /// let namespace = DirectoryNamespaceBuilder::new("/path/to/data")
-    ///     .manifest_enabled(true)
-    ///     .dir_listing_enabled(false)  // All tables now in manifest
-    ///     .build()
-    ///     .await?;
-    /// # Ok(())
-    /// # }
-    /// ```
-    ///
-    /// # Returns
-    ///
-    /// Returns the number of tables that were migrated to the manifest.
-    ///
-    /// # Errors
-    ///
-    /// Returns an error if:
-    /// - Manifest is not enabled
-    /// - Directory listing fails
-    /// - Manifest registration fails
-    pub async fn migrate(&self) -> Result<usize> {
-        // We only care about tables in the root namespace
-        let Some(ref manifest_ns) = self.manifest_ns else {
-            return Ok(0); // No manifest, nothing to migrate
-        };
-
-        // Get all table locations already in the manifest
-        let manifest_locations = manifest_ns.list_manifest_table_locations().await?;
-
-        // Get all tables from directory
-        let dir_tables = self.list_directory_tables().await?;
-
-        // Register each directory table that doesn't have an overlapping location
-        // If a directory name already exists in the manifest,
-        // that means the table must have already been migrated or created
-        // in the manifest, so we can skip it.
-        let mut migrated_count = 0;
-        for table_name in dir_tables {
-            // For root namespace tables, the directory name is "table_name.lance"
-            let dir_name = format!("{}.lance", table_name);
-            if !manifest_locations.contains(&dir_name) {
-                manifest_ns.register_table(&table_name, dir_name).await?;
-                migrated_count += 1;
-            }
-        }
-
-        Ok(migrated_count)
-    }
-}
-
-#[async_trait]
-impl LanceNamespace for DirectoryNamespace {
-    async fn list_namespaces(
-        &self,
-        request: ListNamespacesRequest,
-    ) -> Result<ListNamespacesResponse> {
-        if let Some(ref manifest_ns) = self.manifest_ns {
-            return manifest_ns.list_namespaces(request).await;
-        }
-
-        Self::validate_root_namespace_id(&request.id)?;
-        Ok(ListNamespacesResponse::new(vec![]))
-    }
-
-    async fn describe_namespace(
-        &self,
-        request: DescribeNamespaceRequest,
-    ) -> Result<DescribeNamespaceResponse> {
-        if let Some(ref manifest_ns) = self.manifest_ns {
-            return manifest_ns.describe_namespace(request).await;
-        }
-
-        Self::validate_root_namespace_id(&request.id)?;
-        #[allow(clippy::needless_update)]
-        Ok(DescribeNamespaceResponse {
-            properties: Some(HashMap::new()),
-            ..Default::default()
-        })
-    }
-
-    async fn create_namespace(
-        &self,
-        request: CreateNamespaceRequest,
-    ) -> Result<CreateNamespaceResponse> {
-        if let Some(ref manifest_ns) = self.manifest_ns {
-            return manifest_ns.create_namespace(request).await;
-        }
-
-        if request.id.is_none() || request.id.as_ref().unwrap().is_empty() {
-            return Err(Error::namespace_source(
-                "Root namespace already exists and cannot be created".into(),
-            ));
-        }
-
-        Err(Error::not_supported_source(
-            "Child namespaces are only supported when manifest mode is enabled".into(),
-        ))
-    }
-
-    async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result<DropNamespaceResponse> {
-        if let Some(ref manifest_ns) = self.manifest_ns {
-            return manifest_ns.drop_namespace(request).await;
-        }
-
-        if request.id.is_none() || request.id.as_ref().unwrap().is_empty() {
-            return Err(Error::namespace_source(
-                "Root namespace cannot be dropped".into(),
-            ));
-        }
-
-        Err(Error::not_supported_source(
-            "Child namespaces are only supported when manifest mode is enabled".into(),
-        ))
-    }
-
-    async fn namespace_exists(&self, request: NamespaceExistsRequest) -> Result<()> {
-        if let Some(ref manifest_ns) = self.manifest_ns {
-            return manifest_ns.namespace_exists(request).await;
-        }
-
-        if request.id.is_none() || request.id.as_ref().unwrap().is_empty() {
-            return Ok(());
-        }
-
-        Err(Error::namespace_source(
-            "Child namespaces are only supported when manifest mode is enabled".into(),
-        ))
-    }
-
-    async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
-        // Validate that namespace ID is provided
-        let namespace_id = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Namespace ID is required".into()))?;
-
-        // For child namespaces, always delegate to manifest (if enabled)
-        if !namespace_id.is_empty() {
-            if let Some(ref manifest_ns) = self.manifest_ns {
-                return manifest_ns.list_tables(request).await;
-            }
-            return Err(Error::not_supported_source(
-                "Child namespaces are only supported when manifest mode is enabled".into(),
-            ));
-        }
-
-        // When only manifest is enabled (no directory listing), delegate directly to manifest
-        if let Some(ref manifest_ns) = self.manifest_ns
-            && !self.dir_listing_enabled
-        {
-            return manifest_ns.list_tables(request).await;
-        }
-
-        // When both manifest and directory listing are enabled, we need to merge and deduplicate
-        let mut tables = if self.manifest_ns.is_some() && self.dir_listing_enabled {
-            // Get all manifest table locations (for deduplication)
-            let manifest_locations = if let Some(ref manifest_ns) = self.manifest_ns {
-                manifest_ns.list_manifest_table_locations().await?
-            } else {
-                std::collections::HashSet::new()
-            };
-
-            // Get all manifest tables (without pagination for merging)
-            let mut manifest_request = request.clone();
-            manifest_request.limit = None;
-            manifest_request.page_token = None;
-            let manifest_tables = if let Some(ref manifest_ns) = self.manifest_ns {
-                let manifest_response = manifest_ns.list_tables(manifest_request).await?;
-                manifest_response.tables
-            } else {
-                vec![]
-            };
-
-            // Start with all manifest table names
-            // Add directory tables that aren't already in the manifest (by location)
-            let mut all_tables: Vec<String> = manifest_tables;
-            let dir_tables = self.list_directory_tables().await?;
-            for table_name in dir_tables {
-                // Check if this table's location is already in the manifest
-                // Manifest stores full URIs, so we need to check both formats
-                let full_location = format!("{}/{}.lance", self.root, table_name);
-                let relative_location = format!("{}.lance", table_name);
-                if !manifest_locations.contains(&full_location)
-                    && !manifest_locations.contains(&relative_location)
-                {
-                    all_tables.push(table_name);
-                }
-            }
-
-            all_tables
-        } else {
-            self.list_directory_tables().await?
-        };
-
-        // Apply sorting and pagination
-        Self::apply_pagination(&mut tables, request.page_token, request.limit);
-        let response = ListTablesResponse::new(tables);
-        Ok(response)
-    }
-
-    async fn describe_table(&self, request: DescribeTableRequest) -> Result<DescribeTableResponse> {
-        if let Some(ref manifest_ns) = self.manifest_ns {
-            match manifest_ns.describe_table(request.clone()).await {
-                Ok(mut response) => {
-                    // Only apply identity-based credential vending when explicitly requested
-                    if request.vend_credentials == Some(true) && self.credential_vendor.is_some() {
-                        if let Some(ref table_uri) = response.table_uri {
-                            let identity = request.identity.as_deref();
-                            response.storage_options = self
-                                .get_storage_options_for_table(table_uri, identity)
-                                .await?;
-                        }
-                    } else if request.vend_credentials == Some(false) {
-                        response.storage_options = None;
-                    }
-                    // Set managed_versioning flag when table_version_tracking_enabled
-                    if self.table_version_tracking_enabled {
-                        response.managed_versioning = Some(true);
-                    }
-                    return Ok(response);
-                }
-                Err(_)
-                    if self.dir_listing_enabled
-                        && request.id.as_ref().is_some_and(|id| id.len() == 1) =>
-                {
-                    // Fall through to directory check only for single-level IDs
-                }
-                Err(e) => return Err(e),
-            }
-        }
-
-        let table_name = Self::table_name_from_id(&request.id)?;
-        let table_uri = self.table_full_uri(&table_name);
-
-        // Atomically check table existence and deregistration status
-        let status = self.check_table_status(&table_name).await;
-
-        if !status.exists {
-            return Err(Error::namespace_source(
-                format!("Table does not exist: {}", table_name).into(),
-            ));
-        }
-
-        if status.is_deregistered {
-            return Err(Error::namespace_source(
-                format!("Table is deregistered: {}", table_name).into(),
-            ));
+        if status.is_deregistered {
+            return Err(NamespaceError::TableNotFound {
+                message: format!("Table is deregistered: {}", table_id),
+            }
+            .into());
         }
 
         let load_detailed_metadata = request.load_detailed_metadata.unwrap_or(false);
@@ -1116,12 +1081,9 @@ impl LanceNamespace for DirectoryNamespace {
 
         // If not loading detailed metadata, return minimal response with just location
         if !load_detailed_metadata {
-            let storage_options = if vend_credentials {
-                self.get_storage_options_for_table(&table_uri, identity)
-                    .await?
-            } else {
-                None
-            };
+            let storage_options = self
+                .get_storage_options_for_table(&table_uri, vend_credentials, identity)
+                .await?;
             return Ok(DescribeTableResponse {
                 table: Some(table_name),
                 namespace: request.id.as_ref().map(|id| {
@@ -1156,19 +1118,26 @@ impl LanceNamespace for DirectoryNamespace {
             Ok(mut dataset) => {
                 // If a specific version is requested, checkout that version
                 if let Some(requested_version) = request.version {
-                    dataset = dataset.checkout_version(requested_version as u64).await?;
+                    dataset = dataset
+                        .checkout_version(requested_version as u64)
+                        .await
+                        .map_err(|e| {
+                            lance_core::Error::from(NamespaceError::TableVersionNotFound {
+                                message: format!(
+                                    "Version {} not found for table '{}': {}",
+                                    requested_version, table_name, e
+                                ),
+                            })
+                        })?;
                 }
 
                 let version_info = dataset.version();
                 let lance_schema = dataset.schema();
                 let arrow_schema: arrow_schema::Schema = lance_schema.into();
                 let json_schema = arrow_schema_to_json(&arrow_schema)?;
-                let storage_options = if vend_credentials {
-                    self.get_storage_options_for_table(&table_uri, identity)
-                        .await?
-                } else {
-                    None
-                };
+                let storage_options = self
+                    .get_storage_options_for_table(&table_uri, vend_credentials, identity)
+                    .await?;
 
                 // Convert BTreeMap to HashMap for the response
                 let metadata: std::collections::HashMap<String, String> =
@@ -1200,12 +1169,9 @@ impl LanceNamespace for DirectoryNamespace {
             Err(err) => {
                 // Use the reserved file status from the atomic check
                 if status.has_reserved_file {
-                    let storage_options = if vend_credentials {
-                        self.get_storage_options_for_table(&table_uri, identity)
-                            .await?
-                    } else {
-                        None
-                    };
+                    let storage_options = self
+                        .get_storage_options_for_table(&table_uri, vend_credentials, identity)
+                        .await?;
                     Ok(DescribeTableResponse {
                         table: Some(table_name),
                         namespace: request.id.as_ref().map(|id| {
@@ -1226,3588 +1192,7529 @@ impl LanceNamespace for DirectoryNamespace {
                         ..Default::default()
                     })
                 } else {
-                    Err(Error::namespace_source(
-                        format!(
+                    Err(NamespaceError::Internal {
+                        message: format!(
                             "Table directory exists but cannot load dataset {}: {:?}",
                             table_name, err
-                        )
-                        .into(),
-                    ))
+                        ),
+                    }
+                    .into())
                 }
             }
         }
     }
 
-    async fn table_exists(&self, request: TableExistsRequest) -> Result<()> {
-        if let Some(ref manifest_ns) = self.manifest_ns {
-            match manifest_ns.table_exists(request.clone()).await {
-                Ok(()) => return Ok(()),
-                Err(_) if self.dir_listing_enabled => {
-                    // Fall through to directory check
-                }
-                Err(e) => return Err(e),
+    async fn load_dataset(
+        &self,
+        table_uri: &str,
+        version: Option<i64>,
+        operation: &str,
+    ) -> Result<Dataset> {
+        if let Some(version) = version
+            && version < 0
+        {
+            return Err(NamespaceError::InvalidInput {
+                message: format!(
+                    "Table version for {} must be non-negative, got {}",
+                    operation, version
+                ),
             }
+            .into());
         }
 
-        let table_name = Self::table_name_from_id(&request.id)?;
-
-        // Atomically check table existence and deregistration status
-        let status = self.check_table_status(&table_name).await;
-
-        if !status.exists {
-            return Err(Error::namespace_source(
-                format!("Table does not exist: {}", table_name).into(),
-            ));
+        let mut builder = DatasetBuilder::from_uri(table_uri);
+        if let Some(opts) = &self.storage_options {
+            builder = builder.with_storage_options(opts.clone());
+        }
+        if let Some(sess) = &self.session {
+            builder = builder.with_session(sess.clone());
         }
 
-        if status.is_deregistered {
-            return Err(Error::namespace_source(
-                format!("Table is deregistered: {}", table_name).into(),
-            ));
+        let dataset = builder.load().await.map_err(|e| {
+            lance_core::Error::from(NamespaceError::TableNotFound {
+                message: format!(
+                    "Failed to open table at '{}' for {}: {}",
+                    table_uri, operation, e
+                ),
+            })
+        })?;
+
+        if let Some(version) = version {
+            return dataset.checkout_version(version as u64).await.map_err(|e| {
+                lance_core::Error::from(NamespaceError::TableVersionNotFound {
+                    message: format!(
+                        "Failed to checkout version {} for table at '{}' during {}: {}",
+                        version, table_uri, operation, e
+                    ),
+                })
+            });
         }
 
-        Ok(())
+        Ok(dataset)
     }
 
-    async fn drop_table(&self, request: DropTableRequest) -> Result<DropTableResponse> {
-        if let Some(ref manifest_ns) = self.manifest_ns {
-            return manifest_ns.drop_table(request).await;
+    fn parse_index_type(index_type: &str) -> Result<IndexType> {
+        match index_type.trim().to_ascii_uppercase().as_str() {
+            "SCALAR" | "BTREE" => Ok(IndexType::BTree),
+            "BITMAP" => Ok(IndexType::Bitmap),
+            "LABEL_LIST" | "LABELLIST" => Ok(IndexType::LabelList),
+            "INVERTED" | "FTS" => Ok(IndexType::Inverted),
+            "NGRAM" => Ok(IndexType::NGram),
+            "ZONEMAP" | "ZONE_MAP" => Ok(IndexType::ZoneMap),
+            "BLOOMFILTER" | "BLOOM_FILTER" => Ok(IndexType::BloomFilter),
+            "RTREE" | "R_TREE" => Ok(IndexType::RTree),
+            "VECTOR" | "IVF_PQ" => Ok(IndexType::IvfPq),
+            "IVF_FLAT" => Ok(IndexType::IvfFlat),
+            "IVF_SQ" => Ok(IndexType::IvfSq),
+            "IVF_RQ" => Ok(IndexType::IvfRq),
+            "IVF_HNSW_FLAT" => Ok(IndexType::IvfHnswFlat),
+            "IVF_HNSW_SQ" => Ok(IndexType::IvfHnswSq),
+            "IVF_HNSW_PQ" => Ok(IndexType::IvfHnswPq),
+            other => Err(NamespaceError::InvalidInput {
+                message: format!("Unsupported index_type '{}'", other),
+            }
+            .into()),
         }
+    }
 
-        let table_name = Self::table_name_from_id(&request.id)?;
-        let table_uri = self.table_full_uri(&table_name);
-        let table_path = self.table_path(&table_name);
-
-        self.object_store
-            .remove_dir_all(table_path)
-            .await
-            .map_err(|e| {
-                Error::namespace_source(
-                    format!("Failed to drop table {}: {}", table_name, e).into(),
-                )
-            })?;
+    fn parse_metric_type(distance_type: Option<&str>) -> Result<MetricType> {
+        let distance_type = distance_type.unwrap_or("l2");
+        MetricType::try_from(distance_type).map_err(|e| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: format!(
+                    "Unsupported distance_type '{}' for vector index: {}",
+                    distance_type, e
+                ),
+            })
+        })
+    }
 
-        Ok(DropTableResponse {
-            id: request.id,
-            location: Some(table_uri),
-            ..Default::default()
+    fn build_index_params(request: &CreateTableIndexRequest) -> Result<DirectoryIndexParams> {
+        let index_type = Self::parse_index_type(&request.index_type)?;
+        Ok(match index_type {
+            IndexType::BTree => DirectoryIndexParams::Scalar {
+                index_type,
+                params: ScalarIndexParams::for_builtin(BuiltinIndexType::BTree),
+            },
+            IndexType::Bitmap => DirectoryIndexParams::Scalar {
+                index_type,
+                params: ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap),
+            },
+            IndexType::LabelList => DirectoryIndexParams::Scalar {
+                index_type,
+                params: ScalarIndexParams::for_builtin(BuiltinIndexType::LabelList),
+            },
+            IndexType::NGram => DirectoryIndexParams::Scalar {
+                index_type,
+                params: ScalarIndexParams::for_builtin(BuiltinIndexType::NGram),
+            },
+            IndexType::ZoneMap => DirectoryIndexParams::Scalar {
+                index_type,
+                params: ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap),
+            },
+            IndexType::BloomFilter => DirectoryIndexParams::Scalar {
+                index_type,
+                params: ScalarIndexParams::for_builtin(BuiltinIndexType::BloomFilter),
+            },
+            IndexType::RTree => DirectoryIndexParams::Scalar {
+                index_type,
+                params: ScalarIndexParams::for_builtin(BuiltinIndexType::RTree),
+            },
+            IndexType::Inverted => {
+                let mut params = InvertedIndexParams::default();
+                if let Some(with_position) = request.with_position {
+                    params = params.with_position(with_position);
+                }
+                if let Some(base_tokenizer) = &request.base_tokenizer {
+                    params = params.base_tokenizer(base_tokenizer.clone());
+                }
+                if let Some(language) = &request.language {
+                    params = params.language(language)?;
+                }
+                if let Some(max_token_length) = request.max_token_length {
+                    if max_token_length < 0 {
+                        return Err(NamespaceError::InvalidInput {
+                            message: format!(
+                                "FTS max_token_length must be non-negative, got {}",
+                                max_token_length
+                            ),
+                        }
+                        .into());
+                    }
+                    params = params.max_token_length(Some(max_token_length as usize));
+                }
+                if let Some(lower_case) = request.lower_case {
+                    params = params.lower_case(lower_case);
+                }
+                if let Some(stem) = request.stem {
+                    params = params.stem(stem);
+                }
+                if let Some(remove_stop_words) = request.remove_stop_words {
+                    params = params.remove_stop_words(remove_stop_words);
+                }
+                if let Some(ascii_folding) = request.ascii_folding {
+                    params = params.ascii_folding(ascii_folding);
+                }
+                DirectoryIndexParams::Inverted(params)
+            }
+            IndexType::IvfFlat => DirectoryIndexParams::Vector {
+                index_type,
+                params: VectorIndexParams::with_ivf_flat_params(
+                    Self::parse_metric_type(request.distance_type.as_deref())?,
+                    IvfBuildParams::default(),
+                ),
+            },
+            IndexType::IvfPq => DirectoryIndexParams::Vector {
+                index_type,
+                params: VectorIndexParams::with_ivf_pq_params(
+                    Self::parse_metric_type(request.distance_type.as_deref())?,
+                    IvfBuildParams::default(),
+                    PQBuildParams::default(),
+                ),
+            },
+            IndexType::IvfSq => DirectoryIndexParams::Vector {
+                index_type,
+                params: VectorIndexParams::with_ivf_sq_params(
+                    Self::parse_metric_type(request.distance_type.as_deref())?,
+                    IvfBuildParams::default(),
+                    SQBuildParams::default(),
+                ),
+            },
+            IndexType::IvfRq => DirectoryIndexParams::Vector {
+                index_type,
+                params: VectorIndexParams::with_ivf_rq_params(
+                    Self::parse_metric_type(request.distance_type.as_deref())?,
+                    IvfBuildParams::default(),
+                    RQBuildParams::default(),
+                ),
+            },
+            IndexType::IvfHnswFlat => DirectoryIndexParams::Vector {
+                index_type,
+                params: VectorIndexParams::ivf_hnsw(
+                    Self::parse_metric_type(request.distance_type.as_deref())?,
+                    IvfBuildParams::default(),
+                    HnswBuildParams::default(),
+                ),
+            },
+            IndexType::IvfHnswSq => DirectoryIndexParams::Vector {
+                index_type,
+                params: VectorIndexParams::with_ivf_hnsw_sq_params(
+                    Self::parse_metric_type(request.distance_type.as_deref())?,
+                    IvfBuildParams::default(),
+                    HnswBuildParams::default(),
+                    SQBuildParams::default(),
+                ),
+            },
+            IndexType::IvfHnswPq => DirectoryIndexParams::Vector {
+                index_type,
+                params: VectorIndexParams::with_ivf_hnsw_pq_params(
+                    Self::parse_metric_type(request.distance_type.as_deref())?,
+                    IvfBuildParams::default(),
+                    HnswBuildParams::default(),
+                    PQBuildParams::default(),
+                ),
+            },
+            other => {
+                return Err(NamespaceError::InvalidInput {
+                    message: format!("Unsupported index type for namespace API: {}", other),
+                }
+                .into());
+            }
         })
     }
 
-    async fn create_table(
-        &self,
-        request: CreateTableRequest,
-        request_data: Bytes,
-    ) -> Result<CreateTableResponse> {
-        if let Some(ref manifest_ns) = self.manifest_ns {
-            return manifest_ns.create_table(request, request_data).await;
-        }
+    fn paginate_indices(
+        indices: &mut Vec<IndexContent>,
+        page_token: Option<String>,
+        limit: Option<i32>,
+    ) -> Option<String> {
+        indices.sort_by(|a, b| a.index_name.cmp(&b.index_name));
 
-        let table_name = Self::table_name_from_id(&request.id)?;
-        let table_uri = self.table_full_uri(&table_name);
-        if request_data.is_empty() {
-            return Err(Error::namespace_source(
-                "Request data (Arrow IPC stream) is required for create_table".into(),
-            ));
+        if let Some(start_after) = page_token {
+            if let Some(index) = indices
+                .iter()
+                .position(|index| index.index_name.as_str() > start_after.as_str())
+            {
+                indices.drain(0..index);
+            } else {
+                indices.clear();
+            }
         }
 
-        // Parse the Arrow IPC stream from request_data
-        let cursor = Cursor::new(request_data.to_vec());
-        let stream_reader = StreamReader::try_new(cursor, None).map_err(|e| {
-            Error::namespace_source(format!("Invalid Arrow IPC stream: {}", e).into())
-        })?;
-        let arrow_schema = stream_reader.schema();
+        let mut next_page_token = None;
+        if let Some(limit) = limit
+            && limit >= 0
+        {
+            let limit = limit as usize;
+            if limit > 0 && indices.len() > limit {
+                next_page_token = Some(indices[limit - 1].index_name.clone());
+            }
+            indices.truncate(limit);
+        }
+        if indices.is_empty() {
+            None
+        } else {
+            next_page_token
+        }
+    }
 
-        // Collect all batches from the stream
-        let mut batches = Vec::new();
-        for batch_result in stream_reader {
-            batches.push(batch_result.map_err(|e| {
-                Error::namespace_source(
-                    format!("Failed to read batch from IPC stream: {}", e).into(),
-                )
-            })?);
+    fn transaction_operation_name(transaction: &Transaction) -> String {
+        match &transaction.operation {
+            Operation::CreateIndex {
+                new_indices,
+                removed_indices,
+            } if new_indices.is_empty() && !removed_indices.is_empty() => "DropIndex".to_string(),
+            _ => transaction.operation.to_string(),
         }
+    }
 
-        // Create RecordBatchReader from the batches
-        let reader = if batches.is_empty() {
-            let batch = arrow::record_batch::RecordBatch::new_empty(arrow_schema.clone());
-            let batches = vec![Ok(batch)];
-            RecordBatchIterator::new(batches, arrow_schema.clone())
-        } else {
-            let batch_results: Vec<_> = batches.into_iter().map(Ok).collect();
-            RecordBatchIterator::new(batch_results, arrow_schema)
-        };
+    fn transaction_response(
+        version: u64,
+        transaction: &Transaction,
+    ) -> DescribeTransactionResponse {
+        let mut properties = transaction
+            .transaction_properties
+            .as_ref()
+            .map(|properties| (**properties).clone())
+            .unwrap_or_default();
+        properties.insert("uuid".to_string(), transaction.uuid.clone());
+        properties.insert("version".to_string(), version.to_string());
+        properties.insert(
+            "read_version".to_string(),
+            transaction.read_version.to_string(),
+        );
+        properties.insert(
+            "operation".to_string(),
+            Self::transaction_operation_name(transaction),
+        );
+        if let Some(tag) = &transaction.tag {
+            properties.insert("tag".to_string(), tag.clone());
+        }
 
-        let store_params = self.storage_options.as_ref().map(|opts| ObjectStoreParams {
-            storage_options_accessor: Some(Arc::new(
-                lance_io::object_store::StorageOptionsAccessor::with_static_options(opts.clone()),
-            )),
-            ..Default::default()
-        });
+        DescribeTransactionResponse {
+            status: "SUCCEEDED".to_string(),
+            properties: Some(properties),
+        }
+    }
 
-        let write_params = WriteParams {
-            mode: lance::dataset::WriteMode::Create,
-            store_params,
-            ..Default::default()
+    fn describe_table_index_stats_response(
+        stats: &serde_json::Value,
+    ) -> DescribeTableIndexStatsResponse {
+        let get_i64 = |key: &str| {
+            stats.get(key).and_then(|value| {
+                value
+                    .as_i64()
+                    .or_else(|| value.as_u64().and_then(|v| i64::try_from(v).ok()))
+            })
         };
 
-        // Create the Lance dataset using the actual Lance API
-        Dataset::write(reader, &table_uri, Some(write_params))
-            .await
-            .map_err(|e| {
-                Error::namespace_source(format!("Failed to create Lance dataset: {}", e).into())
-            })?;
-
-        Ok(CreateTableResponse {
-            version: Some(1),
-            location: Some(table_uri),
-            storage_options: self.storage_options.clone(),
-            ..Default::default()
-        })
+        DescribeTableIndexStatsResponse {
+            distance_type: stats
+                .get("distance_type")
+                .and_then(|value| value.as_str())
+                .map(str::to_string),
+            index_type: stats
+                .get("index_type")
+                .and_then(|value| value.as_str())
+                .map(str::to_string),
+            num_indexed_rows: get_i64("num_indexed_rows"),
+            num_unindexed_rows: get_i64("num_unindexed_rows"),
+            num_indices: get_i64("num_indices").and_then(|value| i32::try_from(value).ok()),
+        }
     }
 
-    async fn declare_table(&self, request: DeclareTableRequest) -> Result<DeclareTableResponse> {
-        if let Some(ref manifest_ns) = self.manifest_ns {
-            let mut response = manifest_ns.declare_table(request.clone()).await?;
-            // Only apply identity-based credential vending when explicitly requested
-            if request.vend_credentials == Some(true) && self.credential_vendor.is_some() {
-                if let Some(ref location) = response.location {
-                    let identity = request.identity.as_deref();
-                    response.storage_options = self
-                        .get_storage_options_for_table(location, identity)
-                        .await?;
-                }
-            } else if request.vend_credentials == Some(false) {
-                response.storage_options = None;
-            }
-            // Set managed_versioning when table_version_tracking_enabled
-            if self.table_version_tracking_enabled {
-                response.managed_versioning = Some(true);
-            }
-            return Ok(response);
+    /// When transaction_id is not parseable as a version number (i.e. it's a UUID),
+    /// find_transaction iterates through every version in reverse, reading each
+    /// transaction file from storage. For tables with many versions this will
+    /// be extremely slow — each iteration is a separate I/O call.
+    async fn find_transaction(&self, dataset: &Dataset, id: &str) -> Result<(u64, Transaction)> {
+        if let Ok(version) = id.parse::<u64>() {
+            let transaction = dataset
+                .read_transaction_by_version(version)
+                .await
+                .map_err(|e| {
+                    lance_core::Error::from(NamespaceError::TransactionNotFound {
+                        message: format!(
+                            "Failed to read transaction for version {}: {}",
+                            version, e
+                        ),
+                    })
+                })?
+                .ok_or_else(|| {
+                    lance_core::Error::from(NamespaceError::TransactionNotFound {
+                        message: format!("version {}", version),
+                    })
+                })?;
+            return Ok((version, transaction));
         }
 
-        let table_name = Self::table_name_from_id(&request.id)?;
-        let table_uri = self.table_full_uri(&table_name);
+        let versions = dataset.versions().await.map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!(
+                    "Failed to list table versions while resolving transaction '{}': {}",
+                    id, e
+                ),
+            })
+        })?;
 
-        // Validate location if provided
-        if let Some(location) = &request.location {
-            let location = location.trim_end_matches('/');
-            if location != table_uri {
-                return Err(Error::namespace_source(
-                    format!(
-                        "Cannot declare table {} at location {}, must be at location {}",
-                        table_name, location, table_uri
-                    )
-                    .into(),
-                ));
+        for version in versions.into_iter().rev() {
+            if let Some(transaction) = dataset
+                .read_transaction_by_version(version.version)
+                .await
+                .map_err(|e| {
+                    lance_core::Error::from(NamespaceError::Internal {
+                        message: format!(
+                            "Failed to read transaction for version {} while resolving '{}': {}",
+                            version.version, id, e
+                        ),
+                    })
+                })?
+                && transaction.uuid == id
+            {
+                return Ok((version.version, transaction));
             }
         }
 
-        // Check if table already has data (created via create_table).
-        // The atomic put only prevents races between concurrent declare_table calls,
-        // not between declare_table and existing data.
-        let status = self.check_table_status(&table_name).await;
-        if status.exists && !status.has_reserved_file {
-            // Table has data but no reserved file - it was created with data
-            return Err(Error::namespace_source(
-                format!("Table already exists: {}", table_name).into(),
-            ));
+        Err(NamespaceError::TransactionNotFound {
+            message: id.to_string(),
         }
+        .into())
+    }
 
-        // Atomically create the .lance-reserved file to mark the table as declared.
-        // This uses put_if_not_exists semantics to avoid race conditions between
-        // concurrent declare_table calls.
-        let reserved_file_path = self.table_reserved_file_path(&table_name);
-
-        self.put_marker_file_atomic(&reserved_file_path, &format!("table {}", table_name))
-            .await
-            .map_err(|e| Error::namespace_source(e.into()))?;
+    fn table_full_uri(&self, table_name: &str) -> String {
+        format!("{}/{}.lance", &self.root, table_name)
+    }
 
-        // For backwards compatibility, only skip vending credentials when explicitly set to false
-        let vend_credentials = request.vend_credentials.unwrap_or(true);
-        let identity = request.identity.as_deref();
-        let storage_options = if vend_credentials {
-            self.get_storage_options_for_table(&table_uri, identity)
-                .await?
+    fn uri_to_object_store_path(uri: &str) -> Path {
+        let path_str = if let Some(rest) = uri.strip_prefix("file://") {
+            rest
+        } else if let Some(rest) = uri.strip_prefix("s3://") {
+            rest.split_once('/').map(|(_, p)| p).unwrap_or(rest)
+        } else if let Some(rest) = uri.strip_prefix("gs://") {
+            rest.split_once('/').map(|(_, p)| p).unwrap_or(rest)
+        } else if let Some(rest) = uri.strip_prefix("az://") {
+            rest.split_once('/').map(|(_, p)| p).unwrap_or(rest)
         } else {
-            None
+            uri
         };
+        Path::from(path_str)
+    }
 
-        Ok(DeclareTableResponse {
-            location: Some(table_uri),
-            storage_options,
-            managed_versioning: if self.table_version_tracking_enabled {
-                Some(true)
-            } else {
-                None
-            },
-            ..Default::default()
-        })
+    /// Get the object store path for a table (relative to base_path)
+    fn table_path(&self, table_name: &str) -> Path {
+        self.base_path
+            .child(format!("{}.lance", table_name).as_str())
     }
 
-    async fn register_table(
-        &self,
-        request: lance_namespace::models::RegisterTableRequest,
-    ) -> Result<lance_namespace::models::RegisterTableResponse> {
-        // If manifest is enabled, delegate to manifest namespace
-        if let Some(ref manifest_ns) = self.manifest_ns {
-            return LanceNamespace::register_table(manifest_ns.as_ref(), request).await;
-        }
+    /// Get the reserved file path for a table
+    fn table_reserved_file_path(&self, table_name: &str) -> Path {
+        self.base_path
+            .child(format!("{}.lance", table_name).as_str())
+            .child(".lance-reserved")
+    }
 
-        // Without manifest, register_table is not supported
-        Err(Error::not_supported_source(
-            "register_table is only supported when manifest mode is enabled".into(),
-        ))
+    /// Get the deregistered marker file path for a table
+    fn table_deregistered_file_path(&self, table_name: &str) -> Path {
+        self.base_path
+            .child(format!("{}.lance", table_name).as_str())
+            .child(".lance-deregistered")
     }
 
-    async fn deregister_table(
-        &self,
-        request: lance_namespace::models::DeregisterTableRequest,
-    ) -> Result<lance_namespace::models::DeregisterTableResponse> {
-        // If manifest is enabled, delegate to manifest namespace
-        if let Some(ref manifest_ns) = self.manifest_ns {
-            return LanceNamespace::deregister_table(manifest_ns.as_ref(), request).await;
+    /// Atomically check table existence and deregistration status.
+    ///
+    /// This performs a single directory listing to get a consistent snapshot of the
+    /// table's state, avoiding race conditions between checking existence and
+    /// checking deregistration status.
+    pub(crate) async fn check_table_status(&self, table_name: &str) -> TableStatus {
+        let table_path = self.table_path(table_name);
+        match self.object_store.read_dir(table_path).await {
+            Ok(entries) => {
+                let exists = !entries.is_empty();
+                let is_deregistered = entries.iter().any(|e| e.ends_with(".lance-deregistered"));
+                let has_reserved_file = entries.iter().any(|e| e.ends_with(".lance-reserved"));
+                TableStatus {
+                    exists,
+                    is_deregistered,
+                    has_reserved_file,
+                }
+            }
+            Err(_) => TableStatus {
+                exists: false,
+                is_deregistered: false,
+                has_reserved_file: false,
+            },
         }
+    }
 
-        // V1 mode: create a .lance-deregistered marker file in the table directory
-        let table_name = Self::table_name_from_id(&request.id)?;
-        let table_uri = self.table_full_uri(&table_name);
-
-        // Check table existence and deregistration status.
-        // This provides better error messages for common cases.
-        let status = self.check_table_status(&table_name).await;
+    async fn put_marker_file_atomic(
+        &self,
+        path: &Path,
+        file_description: &str,
+    ) -> std::result::Result<(), String> {
+        let put_opts = PutOptions {
+            mode: PutMode::Create,
+            ..Default::default()
+        };
 
-        if !status.exists {
-            return Err(Error::namespace_source(
-                format!("Table does not exist: {}", table_name).into(),
-            ));
+        match self
+            .object_store
+            .inner
+            .put_opts(path, bytes::Bytes::new().into(), put_opts)
+            .await
+        {
+            Ok(_) => Ok(()),
+            Err(ObjectStoreError::AlreadyExists { .. })
+            | Err(ObjectStoreError::Precondition { .. }) => {
+                Err(format!("{} already exists", file_description))
+            }
+            Err(e) => Err(format!("Failed to create {}: {:?}", file_description, e)),
         }
+    }
 
-        if status.is_deregistered {
-            return Err(Error::namespace_source(
-                format!("Table is already deregistered: {}", table_name).into(),
-            ));
+    /// Get storage options for a table, using credential vending if configured.
+    ///
+    /// If credential vendor properties are configured and the table location matches
+    /// a supported cloud provider, this will create an appropriate vendor and vend
+    /// temporary credentials scoped to the table location. Otherwise, returns the
+    /// static storage options.
+    ///
+    /// The vendor type is auto-selected based on the table URI:
+    /// - `s3://` locations use AWS STS AssumeRole
+    /// - `gs://` locations use GCP OAuth2 tokens
+    /// - `az://` locations use Azure SAS tokens
+    ///
+    /// The permission level (Read, Write, Admin) is configured at namespace
+    /// initialization time via the `credential_vendor_permission` property.
+    ///
+    /// # Arguments
+    ///
+    /// * `table_uri` - The full URI of the table
+    /// * `identity` - Optional identity from the request for identity-based credential vending
+    async fn get_storage_options_for_table(
+        &self,
+        table_uri: &str,
+        vend_credentials: bool,
+        identity: Option<&Identity>,
+    ) -> Result<Option<HashMap<String, String>>> {
+        if vend_credentials && let Some(ref vendor) = self.credential_vendor {
+            let vended = vendor.vend_credentials(table_uri, identity).await?;
+            return Ok(Some(vended.storage_options));
+        }
+        // When vend_input_storage_options is enabled and no credential vendor is configured,
+        // return the input storage options. This is useful for testing.
+        if self.vend_input_storage_options {
+            let mut options = self.storage_options.clone().unwrap_or_default();
+            // Add expires_at_millis if refresh interval is configured
+            if let Some(refresh_interval_millis) =
+                self.vend_input_storage_options_refresh_interval_millis
+            {
+                let now_millis = std::time::SystemTime::now()
+                    .duration_since(std::time::UNIX_EPOCH)
+                    .unwrap()
+                    .as_millis() as u64;
+                let expires_at_millis = now_millis + refresh_interval_millis;
+                options.insert(
+                    "expires_at_millis".to_string(),
+                    expires_at_millis.to_string(),
+                );
+            }
+            return Ok(Some(options));
         }
+        // When no credential vendor is configured, return None to avoid
+        // leaking the namespace's own static credentials to clients.
+        Ok(None)
+    }
 
-        // Atomically create the .lance-deregistered marker file.
-        // This uses put_if_not_exists semantics to prevent race conditions
-        // when multiple processes try to deregister the same table concurrently.
-        // If a race occurs and another process already created the file,
-        // we'll get an AlreadyExists error which we convert to a proper message.
-        let deregistered_path = self.table_deregistered_file_path(&table_name);
-        self.put_marker_file_atomic(
-            &deregistered_path,
-            &format!("deregistration marker for table {}", table_name),
-        )
-        .await
-        .map_err(|e| {
-            // Convert "already exists" to "already deregistered" for better UX
-            let message = if e.contains("already exists") {
-                format!("Table is already deregistered: {}", table_name)
-            } else {
-                e
-            };
-            Error::namespace_source(message.into())
-        })?;
+    /// Migrate directory-based tables to the manifest.
+    ///
+    /// This is a one-time migration operation that:
+    /// 1. Scans the directory for existing `.lance` tables
+    /// 2. Registers any unmigrated tables in the manifest
+    /// 3. Returns the count of tables that were migrated
+    ///
+    /// This method is safe to run multiple times - it will skip tables that are already
+    /// registered in the manifest.
+    ///
+    /// # Usage
+    ///
+    /// After creating tables in directory-only mode or dual mode, you can migrate them
+    /// to the manifest to enable manifest-only mode:
+    ///
+    /// ```no_run
+    /// # use lance_namespace_impls::DirectoryNamespaceBuilder;
+    /// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
+    /// // Create namespace with dual mode (manifest + directory listing)
+    /// let namespace = DirectoryNamespaceBuilder::new("/path/to/data")
+    ///     .manifest_enabled(true)
+    ///     .dir_listing_enabled(true)
+    ///     .build()
+    ///     .await?;
+    ///
+    /// // ... tables are created and used ...
+    ///
+    /// // Migrate existing directory tables to manifest
+    /// let migrated_count = namespace.migrate().await?;
+    /// println!("Migrated {} tables", migrated_count);
+    ///
+    /// // Now you can disable directory listing for better performance:
+    /// // (requires rebuilding the namespace)
+    /// let namespace = DirectoryNamespaceBuilder::new("/path/to/data")
+    ///     .manifest_enabled(true)
+    ///     .dir_listing_enabled(false)  // All tables now in manifest
+    ///     .build()
+    ///     .await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    ///
+    /// # Returns
+    ///
+    /// Returns the number of tables that were migrated to the manifest.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if:
+    /// - Manifest is not enabled
+    /// - Directory listing fails
+    /// - Manifest registration fails
+    pub async fn migrate(&self) -> Result<usize> {
+        // We only care about tables in the root namespace
+        let Some(ref manifest_ns) = self.manifest_ns else {
+            return Ok(0); // No manifest, nothing to migrate
+        };
 
-        Ok(lance_namespace::models::DeregisterTableResponse {
-            id: request.id,
-            location: Some(table_uri),
-            ..Default::default()
-        })
+        // Get all table locations already in the manifest
+        let manifest_locations = manifest_ns.list_manifest_table_locations().await?;
+
+        // Get all tables from directory
+        let dir_tables = self.list_directory_tables().await?;
+
+        // Register each directory table that doesn't have an overlapping location
+        // If a directory name already exists in the manifest,
+        // that means the table must have already been migrated or created
+        // in the manifest, so we can skip it.
+        let mut migrated_count = 0;
+        for table_name in dir_tables {
+            // For root namespace tables, the directory name is "table_name.lance"
+            let dir_name = format!("{}.lance", table_name);
+            if !manifest_locations.contains(&dir_name) {
+                manifest_ns.register_table(&table_name, dir_name).await?;
+                migrated_count += 1;
+            }
+        }
+
+        Ok(migrated_count)
     }
 
-    async fn list_table_versions(
+    /// Delete physical manifest files for the given table version ranges (best-effort).
+    ///
+    /// This helper is used by `batch_delete_table_versions` in both the manifest-enabled
+    /// and non-manifest paths. It resolves each table's storage location, computes the
+    /// version file paths, and attempts to delete them. Errors are logged (best-effort)
+    /// when `best_effort` is true, or returned immediately when false.
+    ///
+    /// Returns the number of files successfully deleted.
+    async fn delete_physical_version_files(
         &self,
-        request: ListTableVersionsRequest,
-    ) -> Result<ListTableVersionsResponse> {
-        let table_uri = self.resolve_table_location(&request.id).await?;
+        table_entries: &[TableDeleteEntry],
+        best_effort: bool,
+    ) -> Result<i64> {
+        let mut deleted_count = 0i64;
+        for te in table_entries {
+            let table_uri = self.resolve_table_location(&te.table_id).await?;
+            let table_path = Self::uri_to_object_store_path(&table_uri);
+            let table_path_str = table_path.as_ref();
+            let versions_dir_path = Path::from(format!("{}_versions", table_path_str));
+
+            for (start, end) in &te.ranges {
+                for version in *start..=*end {
+                    let version_path =
+                        versions_dir_path.child(format!("{}.manifest", version as u64));
+                    match self.object_store.inner.delete(&version_path).await {
+                        Ok(_) => {
+                            deleted_count += 1;
+                        }
+                        Err(object_store::Error::NotFound { .. }) => {}
+                        Err(e) => {
+                            if best_effort {
+                                log::warn!(
+                                    "Failed to delete manifest file for version {} of table {:?}: {:?}",
+                                    version,
+                                    te.table_id,
+                                    e
+                                );
+                            } else {
+                                return Err(NamespaceError::Internal {
+                                    message: format!(
+                                        "Failed to delete version {} for table at '{}': {}",
+                                        version, table_uri, e
+                                    ),
+                                }
+                                .into());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        Ok(deleted_count)
+    }
 
-        let table_path = Self::uri_to_object_store_path(&table_uri);
-        let versions_dir = table_path.child("_versions");
-        let manifest_metas: Vec<_> = self
-            .object_store
-            .read_dir_all(&versions_dir, None)
-            .try_collect()
-            .await
-            .map_err(|e| {
-                Error::namespace_source(
-                    format!(
-                        "Failed to list manifest files for table at '{}': {}",
-                        table_uri, e
-                    )
-                    .into(),
+    /// Apply all query parameters from a `QueryTableRequest`-like source onto a `Scanner`.
+    ///
+    /// This covers vector search, filters, column projection, limits, and ANN tuning knobs so
+    /// that `explain_table_query_plan` / `analyze_table_query_plan` produce an accurate plan.
+    #[allow(clippy::too_many_arguments)]
+    fn apply_query_params_to_scanner(
+        scanner: &mut Scanner,
+        filter: Option<&str>,
+        columns: Option<&QueryTableRequestColumns>,
+        vector_column: Option<&str>,
+        vector: &QueryTableRequestVector,
+        k: i32,
+        offset: Option<i32>,
+        prefilter: Option<bool>,
+        bypass_vector_index: Option<bool>,
+        nprobes: Option<i32>,
+        ef: Option<i32>,
+        refine_factor: Option<i32>,
+        distance_type: Option<&str>,
+        fast_search_flag: Option<bool>,
+        with_row_id: Option<bool>,
+        lower_bound: Option<f32>,
+        upper_bound: Option<f32>,
+        operation: &str,
+    ) -> Result<()> {
+        // prefilter must be set before nearest() so the fragment-scan guard sees it.
+        if let Some(pf) = prefilter {
+            scanner.prefilter(pf);
+        }
+
+        if let Some(filter) = filter {
+            scanner.filter(filter).map_err(|e| {
+                Error::invalid_input_source(
+                    format!("Invalid filter expression for {}: {}", operation, e).into(),
                 )
             })?;
+        }
 
-        let is_v2_naming = manifest_metas
-            .first()
-            .is_some_and(|meta| meta.location.filename().is_some_and(|f| f.len() == 29));
+        if let Some(cols) = columns {
+            if let Some(ref names) = cols.column_names {
+                scanner.project(names.as_slice()).map_err(|e| {
+                    Error::invalid_input_source(
+                        format!("Invalid column projection for {}: {}", operation, e).into(),
+                    )
+                })?;
+            } else if let Some(ref aliases) = cols.column_aliases {
+                // aliases maps output_alias -> source_column
+                let pairs: Vec<(&str, &str)> = aliases
+                    .iter()
+                    .map(|(alias, src)| (alias.as_str(), src.as_str()))
+                    .collect();
+                scanner.project_with_transform(&pairs).map_err(|e| {
+                    Error::invalid_input_source(
+                        format!("Invalid column aliases for {}: {}", operation, e).into(),
+                    )
+                })?;
+            }
+        }
+
+        // Resolve query vector: prefer single_vector, fall back to first row of multi_vector.
+        let query_vec: Option<Vec<f32>> = vector
+            .single_vector
+            .as_ref()
+            .filter(|v| !v.is_empty())
+            .cloned()
+            .or_else(|| {
+                vector
+                    .multi_vector
+                    .as_ref()
+                    .and_then(|mv| mv.first())
+                    .filter(|v| !v.is_empty())
+                    .cloned()
+            });
+
+        if let Some(q_vec) = query_vec {
+            let col = vector_column.unwrap_or("vector");
+            let q = Arc::new(Float32Array::from(q_vec));
+            scanner
+                .nearest(col, q.as_ref(), k.max(1) as usize)
+                .map_err(|e| {
+                    Error::invalid_input_source(
+                        format!("Invalid vector query for {}: {}", operation, e).into(),
+                    )
+                })?;
+
+            // ANN parameters — must be applied after nearest().
+            if let Some(n) = nprobes {
+                scanner.nprobes(n.max(1) as usize);
+            }
+            if let Some(e) = ef {
+                scanner.ef(e.max(1) as usize);
+            }
+            if let Some(rf) = refine_factor {
+                scanner.refine(rf.max(0) as u32);
+            }
+            // bypass_vector_index and fast_search are mutually exclusive; apply in order.
+            if let Some(true) = bypass_vector_index {
+                scanner.use_index(false);
+            }
+            if let Some(true) = fast_search_flag {
+                scanner.fast_search();
+            }
+            if lower_bound.is_some() || upper_bound.is_some() {
+                scanner.distance_range(lower_bound, upper_bound);
+            }
+            if let Some(dt) = distance_type {
+                let metric = Self::parse_metric_type(Some(dt))?;
+                scanner.distance_metric(metric);
+            }
+            // Apply offset on top of the k nearest results.
+            if let Some(off) = offset.filter(|&o| o > 0) {
+                scanner.limit(None, Some(off as i64)).map_err(|e| {
+                    Error::invalid_input_source(
+                        format!("Invalid offset for {}: {}", operation, e).into(),
+                    )
+                })?;
+            }
+        } else {
+            // Scalar (non-vector) query: treat k as a row LIMIT.
+            let limit = if k > 0 { Some(k as i64) } else { None };
+            scanner
+                .limit(limit, offset.map(|o| o as i64))
+                .map_err(|e| {
+                    Error::invalid_input_source(
+                        format!("Invalid limit/offset for {}: {}", operation, e).into(),
+                    )
+                })?;
+        }
+
+        if let Some(true) = with_row_id {
+            scanner.with_row_id();
+        }
+
+        Ok(())
+    }
+
+    /// Retrieve a snapshot of operation metrics.
+    ///
+    /// Returns a HashMap where keys are operation names (e.g., "list_tables", "describe_table")
+    /// and values are the number of times each operation was called.
+    ///
+    /// Returns an empty HashMap if `ops_metrics_enabled` was false when building the namespace.
+    pub fn retrieve_ops_metrics(&self) -> HashMap<String, u64> {
+        self.ops_metrics
+            .as_ref()
+            .map(|m| m.retrieve())
+            .unwrap_or_default()
+    }
+
+    /// Reset all operation metrics counters to zero.
+    ///
+    /// Does nothing if `ops_metrics_enabled` was false when building the namespace.
+    pub fn reset_ops_metrics(&self) {
+        if let Some(ref metrics) = self.ops_metrics {
+            metrics.reset();
+        }
+    }
+
+    /// Increment the counter for an operation.
+    fn record_op(&self, operation: &str) {
+        if let Some(ref metrics) = self.ops_metrics {
+            metrics.increment(operation);
+        }
+    }
+}
+
+#[async_trait]
+impl LanceNamespace for DirectoryNamespace {
+    async fn list_namespaces(
+        &self,
+        request: ListNamespacesRequest,
+    ) -> Result<ListNamespacesResponse> {
+        self.record_op("list_namespaces");
+        if let Some(ref manifest_ns) = self.manifest_ns {
+            return manifest_ns.list_namespaces(request).await;
+        }
+
+        Self::validate_root_namespace_id(&request.id)?;
+        Ok(ListNamespacesResponse::new(vec![]))
+    }
+
+    async fn describe_namespace(
+        &self,
+        request: DescribeNamespaceRequest,
+    ) -> Result<DescribeNamespaceResponse> {
+        self.record_op("describe_namespace");
+        if let Some(ref manifest_ns) = self.manifest_ns {
+            return manifest_ns.describe_namespace(request).await;
+        }
+
+        Self::validate_root_namespace_id(&request.id)?;
+        #[allow(clippy::needless_update)]
+        Ok(DescribeNamespaceResponse {
+            properties: Some(HashMap::new()),
+            ..Default::default()
+        })
+    }
+
+    async fn create_namespace(
+        &self,
+        request: CreateNamespaceRequest,
+    ) -> Result<CreateNamespaceResponse> {
+        self.record_op("create_namespace");
+        if let Some(ref manifest_ns) = self.manifest_ns {
+            return manifest_ns.create_namespace(request).await;
+        }
+
+        if request.id.is_none() || request.id.as_ref().unwrap().is_empty() {
+            return Err(NamespaceError::NamespaceAlreadyExists {
+                message: "root namespace".to_string(),
+            }
+            .into());
+        }
+
+        Err(NamespaceError::Unsupported {
+            message: "Child namespaces are only supported when manifest mode is enabled"
+                .to_string(),
+        }
+        .into())
+    }
+
+    async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result<DropNamespaceResponse> {
+        self.record_op("drop_namespace");
+        if let Some(ref manifest_ns) = self.manifest_ns {
+            return manifest_ns.drop_namespace(request).await;
+        }
+
+        if request.id.is_none() || request.id.as_ref().unwrap().is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "Root namespace cannot be dropped".to_string(),
+            }
+            .into());
+        }
+
+        Err(NamespaceError::Unsupported {
+            message: "Child namespaces are only supported when manifest mode is enabled"
+                .to_string(),
+        }
+        .into())
+    }
+
+    async fn namespace_exists(&self, request: NamespaceExistsRequest) -> Result<()> {
+        self.record_op("namespace_exists");
+        if let Some(ref manifest_ns) = self.manifest_ns {
+            return manifest_ns.namespace_exists(request).await;
+        }
+
+        if request.id.is_none() || request.id.as_ref().unwrap().is_empty() {
+            return Ok(());
+        }
+
+        Err(NamespaceError::NamespaceNotFound {
+            message: "Child namespaces are only supported when manifest mode is enabled"
+                .to_string(),
+        }
+        .into())
+    }
+
+    async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
+        self.record_op("list_tables");
+        // Validate that namespace ID is provided
+        let namespace_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Namespace ID is required".to_string(),
+            })
+        })?;
+
+        // For child namespaces, always delegate to manifest (if enabled)
+        if !namespace_id.is_empty() {
+            if let Some(ref manifest_ns) = self.manifest_ns {
+                return manifest_ns.list_tables(request).await;
+            }
+            return Err(NamespaceError::Unsupported {
+                message: "Child namespaces are only supported when manifest mode is enabled"
+                    .to_string(),
+            }
+            .into());
+        }
+
+        // When only manifest is enabled (no directory listing), delegate directly to manifest
+        if let Some(ref manifest_ns) = self.manifest_ns
+            && !self.dir_listing_enabled
+        {
+            return manifest_ns.list_tables(request).await;
+        }
+
+        // When both manifest and directory listing are enabled with migration mode,
+        // we need to merge and deduplicate
+        let mut tables = if self.manifest_ns.is_some()
+            && self.dir_listing_enabled
+            && self.dir_listing_to_manifest_migration_enabled
+        {
+            // Get all manifest table locations (for deduplication)
+            let manifest_locations = if let Some(ref manifest_ns) = self.manifest_ns {
+                manifest_ns.list_manifest_table_locations().await?
+            } else {
+                std::collections::HashSet::new()
+            };
+
+            // Get all manifest tables (without pagination for merging)
+            let mut manifest_request = request.clone();
+            manifest_request.limit = None;
+            manifest_request.page_token = None;
+            let manifest_tables = if let Some(ref manifest_ns) = self.manifest_ns {
+                let manifest_response = manifest_ns.list_tables(manifest_request).await?;
+                manifest_response.tables
+            } else {
+                vec![]
+            };
+
+            // Start with all manifest table names
+            // Add directory tables that aren't already in the manifest (by location)
+            let mut all_tables: Vec<String> = manifest_tables;
+            let dir_tables = self.list_directory_tables().await?;
+            for table_name in dir_tables {
+                // Check if this table's location is already in the manifest
+                // Manifest stores full URIs, so we need to check both formats
+                let full_location = format!("{}/{}.lance", self.root, table_name);
+                let relative_location = format!("{}.lance", table_name);
+                if !manifest_locations.contains(&full_location)
+                    && !manifest_locations.contains(&relative_location)
+                {
+                    all_tables.push(table_name);
+                }
+            }
+
+            all_tables
+        } else {
+            self.list_directory_tables().await?
+        };
+
+        // Apply sorting and pagination
+        let next_page_token =
+            Self::apply_pagination(&mut tables, request.page_token, request.limit);
+        let mut response = ListTablesResponse::new(tables);
+        response.page_token = next_page_token;
+        Ok(response)
+    }
+
+    async fn describe_table(&self, request: DescribeTableRequest) -> Result<DescribeTableResponse> {
+        self.record_op("describe_table");
+        self.describe_table_impl(request).await
+    }
+
+    async fn table_exists(&self, request: TableExistsRequest) -> Result<()> {
+        self.record_op("table_exists");
+        let is_root_level = request.id.as_ref().is_some_and(|id| id.len() == 1);
+        let skip_manifest_for_root = self.dir_listing_enabled
+            && is_root_level
+            && !self.dir_listing_to_manifest_migration_enabled;
+        if let Some(ref manifest_ns) = self.manifest_ns
+            && !skip_manifest_for_root
+        {
+            match manifest_ns.table_exists(request.clone()).await {
+                Ok(()) => return Ok(()),
+                Err(_) if self.dir_listing_enabled && is_root_level => {
+                    // Fall through to directory check only for single-level IDs
+                }
+                Err(e) => return Err(e),
+            }
+        }
+
+        let table_name = Self::table_name_from_id(&request.id)?;
+        let table_id = Self::format_table_id_from_request(&request.id);
+
+        // Atomically check table existence and deregistration status
+        let status = self.check_table_status(&table_name).await;
+
+        if !status.exists {
+            return Err(NamespaceError::TableNotFound {
+                message: table_id.clone(),
+            }
+            .into());
+        }
+
+        if status.is_deregistered {
+            return Err(NamespaceError::TableNotFound {
+                message: format!("Table is deregistered: {}", table_id),
+            }
+            .into());
+        }
+
+        Ok(())
+    }
+
+    async fn drop_table(&self, request: DropTableRequest) -> Result<DropTableResponse> {
+        self.record_op("drop_table");
+        if let Some(ref manifest_ns) = self.manifest_ns {
+            return manifest_ns.drop_table(request).await;
+        }
+
+        let table_name = Self::table_name_from_id(&request.id)?;
+        let table_uri = self.table_full_uri(&table_name);
+        let table_path = self.table_path(&table_name);
+
+        self.object_store
+            .remove_dir_all(table_path)
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to drop table {}: {:?}", table_name, e),
+                })
+            })?;
+
+        Ok(DropTableResponse {
+            id: request.id,
+            location: Some(table_uri),
+            ..Default::default()
+        })
+    }
+
+    async fn create_table(
+        &self,
+        request: CreateTableRequest,
+        request_data: Bytes,
+    ) -> Result<CreateTableResponse> {
+        self.record_op("create_table");
+        if let Some(ref manifest_ns) = self.manifest_ns {
+            return manifest_ns.create_table(request, request_data).await;
+        }
+
+        let table_name = Self::table_name_from_id(&request.id)?;
+        let table_uri = self.table_full_uri(&table_name);
+        if request_data.is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "Request data (Arrow IPC stream) is required for create_table".to_string(),
+            }
+            .into());
+        }
+
+        // Parse the Arrow IPC stream from request_data
+        let cursor = Cursor::new(request_data.to_vec());
+        let stream_reader = StreamReader::try_new(cursor, None).map_err(|e| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: format!("Invalid Arrow IPC stream: {:?}", e),
+            })
+        })?;
+        let arrow_schema = stream_reader.schema();
+
+        // Collect all batches from the stream
+        let mut batches = Vec::new();
+        for batch_result in stream_reader {
+            batches.push(batch_result.map_err(|e| {
+                lance_core::Error::from(NamespaceError::InvalidInput {
+                    message: format!("Failed to read batch from IPC stream: {:?}", e),
+                })
+            })?);
+        }
+
+        // Create RecordBatchReader from the batches
+        let reader = if batches.is_empty() {
+            let batch = arrow::record_batch::RecordBatch::new_empty(arrow_schema.clone());
+            let batches = vec![Ok(batch)];
+            RecordBatchIterator::new(batches, arrow_schema.clone())
+        } else {
+            let batch_results: Vec<_> = batches.into_iter().map(Ok).collect();
+            RecordBatchIterator::new(batch_results, arrow_schema)
+        };
+
+        let store_params = self.storage_options.as_ref().map(|opts| ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(
+                lance_io::object_store::StorageOptionsAccessor::with_static_options(opts.clone()),
+            )),
+            ..Default::default()
+        });
+
+        let write_params = WriteParams {
+            mode: WriteMode::Create,
+            store_params,
+            ..Default::default()
+        };
+
+        // Create the Lance dataset using the actual Lance API
+        Dataset::write(reader, &table_uri, Some(write_params))
+            .await
+            .map_err(|e| {
+                let err_msg = format!("{}", e);
+                let ns_err = if err_msg.contains("already exists") {
+                    NamespaceError::TableAlreadyExists {
+                        message: format!("Table already exists at '{}': {:?}", table_uri, e),
+                    }
+                } else {
+                    NamespaceError::Internal {
+                        message: format!("Failed to create Lance dataset: {:?}", e),
+                    }
+                };
+                lance_core::Error::from(ns_err)
+            })?;
+
+        Ok(CreateTableResponse {
+            version: Some(1),
+            location: Some(table_uri),
+            storage_options: self.storage_options.clone(),
+            ..Default::default()
+        })
+    }
+
+    async fn declare_table(&self, request: DeclareTableRequest) -> Result<DeclareTableResponse> {
+        self.record_op("declare_table");
+        if let Some(ref manifest_ns) = self.manifest_ns {
+            let mut response = manifest_ns.declare_table(request.clone()).await?;
+            if let Some(ref location) = response.location {
+                // For backwards compatibility, only skip vending credentials when explicitly set to false
+                let vend = request.vend_credentials.unwrap_or(true);
+                let identity = request.identity.as_deref();
+                response.storage_options = self
+                    .get_storage_options_for_table(location, vend, identity)
+                    .await?;
+            }
+            // Set managed_versioning when table_version_tracking_enabled
+            if self.table_version_tracking_enabled {
+                response.managed_versioning = Some(true);
+            }
+            return Ok(response);
+        }
+
+        let table_name = Self::table_name_from_id(&request.id)?;
+        let table_uri = self.table_full_uri(&table_name);
+
+        // Validate location if provided
+        if let Some(location) = &request.location {
+            let location = location.trim_end_matches('/');
+            if location != table_uri {
+                return Err(NamespaceError::InvalidInput {
+                    message: format!(
+                        "Cannot declare table {} at location {}, must be at location {}",
+                        table_name, location, table_uri
+                    ),
+                }
+                .into());
+            }
+        }
+
+        // Check if table already has data (created via create_table).
+        // The atomic put only prevents races between concurrent declare_table calls,
+        // not between declare_table and existing data.
+        let status = self.check_table_status(&table_name).await;
+        if status.exists && !status.has_reserved_file {
+            // Table has data but no reserved file - it was created with data
+            return Err(NamespaceError::TableAlreadyExists {
+                message: table_name.to_string(),
+            }
+            .into());
+        }
+
+        // Atomically create the .lance-reserved file to mark the table as declared.
+        // This uses put_if_not_exists semantics to avoid race conditions between
+        // concurrent declare_table calls.
+        let reserved_file_path = self.table_reserved_file_path(&table_name);
+
+        self.put_marker_file_atomic(&reserved_file_path, &format!("table {}", table_name))
+            .await
+            .map_err(|e| {
+                if e.contains("already exists") {
+                    lance_core::Error::from(NamespaceError::TableAlreadyExists {
+                        message: table_name.to_string(),
+                    })
+                } else {
+                    lance_core::Error::from(NamespaceError::Internal { message: e })
+                }
+            })?;
+
+        // For backwards compatibility, only skip vending credentials when explicitly set to false
+        let vend_credentials = request.vend_credentials.unwrap_or(true);
+        let identity = request.identity.as_deref();
+        let storage_options = self
+            .get_storage_options_for_table(&table_uri, vend_credentials, identity)
+            .await?;
+
+        Ok(DeclareTableResponse {
+            location: Some(table_uri),
+            storage_options,
+            managed_versioning: if self.table_version_tracking_enabled {
+                Some(true)
+            } else {
+                None
+            },
+            ..Default::default()
+        })
+    }
+
+    async fn register_table(
+        &self,
+        request: lance_namespace::models::RegisterTableRequest,
+    ) -> Result<lance_namespace::models::RegisterTableResponse> {
+        self.record_op("register_table");
+        // If manifest is enabled, delegate to manifest namespace
+        if let Some(ref manifest_ns) = self.manifest_ns {
+            return LanceNamespace::register_table(manifest_ns.as_ref(), request).await;
+        }
+
+        // Without manifest, register_table is not supported
+        Err(NamespaceError::Unsupported {
+            message: "register_table is only supported when manifest mode is enabled".to_string(),
+        }
+        .into())
+    }
+
+    async fn deregister_table(
+        &self,
+        request: lance_namespace::models::DeregisterTableRequest,
+    ) -> Result<lance_namespace::models::DeregisterTableResponse> {
+        self.record_op("deregister_table");
+        // If manifest is enabled, delegate to manifest namespace
+        if let Some(ref manifest_ns) = self.manifest_ns {
+            return LanceNamespace::deregister_table(manifest_ns.as_ref(), request).await;
+        }
+
+        // V1 mode: create a .lance-deregistered marker file in the table directory
+        let table_name = Self::table_name_from_id(&request.id)?;
+        let table_uri = self.table_full_uri(&table_name);
+
+        // Check table existence and deregistration status.
+        // This provides better error messages for common cases.
+        let status = self.check_table_status(&table_name).await;
+
+        if !status.exists {
+            return Err(NamespaceError::TableNotFound {
+                message: table_name.to_string(),
+            }
+            .into());
+        }
+
+        if status.is_deregistered {
+            return Err(NamespaceError::TableNotFound {
+                message: format!("Table is already deregistered: {}", table_name),
+            }
+            .into());
+        }
+
+        // Atomically create the .lance-deregistered marker file.
+        // This uses put_if_not_exists semantics to prevent race conditions
+        // when multiple processes try to deregister the same table concurrently.
+        // If a race occurs and another process already created the file,
+        // we'll get an AlreadyExists error which we convert to a proper message.
+        let deregistered_path = self.table_deregistered_file_path(&table_name);
+        self.put_marker_file_atomic(
+            &deregistered_path,
+            &format!("deregistration marker for table {}", table_name),
+        )
+        .await
+        .map_err(|e| {
+            if e.contains("already exists") {
+                lance_core::Error::from(NamespaceError::InvalidTableState {
+                    message: format!("Table is already deregistered: {}", table_name),
+                })
+            } else {
+                lance_core::Error::from(NamespaceError::Internal { message: e })
+            }
+        })?;
+
+        Ok(lance_namespace::models::DeregisterTableResponse {
+            id: request.id,
+            location: Some(table_uri),
+            ..Default::default()
+        })
+    }
+
+    async fn list_table_versions(
+        &self,
+        request: ListTableVersionsRequest,
+    ) -> Result<ListTableVersionsResponse> {
+        self.record_op("list_table_versions");
+        // When table_version_storage_enabled, query from __manifest
+        if self.table_version_storage_enabled
+            && let Some(ref manifest_ns) = self.manifest_ns
+        {
+            let table_id = request.id.clone().unwrap_or_default();
+            let want_descending = request.descending == Some(true);
+            return manifest_ns
+                .list_table_versions(&table_id, want_descending, request.limit)
+                .await;
+        }
+
+        // Fallback when table_version_storage is not enabled: list from _versions/ directory
+        let table_uri = self.resolve_table_location(&request.id).await?;
+
+        let table_path = Self::uri_to_object_store_path(&table_uri);
+        let versions_dir = table_path.child("_versions");
+        let manifest_metas: Vec<_> = self
+            .object_store
+            .read_dir_all(&versions_dir, None)
+            .try_collect()
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to list manifest files for table at '{}': {}",
+                        table_uri, e
+                    ),
+                })
+            })?;
+
+        let is_v2_naming = manifest_metas
+            .first()
+            .is_some_and(|meta| meta.location.filename().is_some_and(|f| f.len() == 29));
+
+        let mut table_versions: Vec<TableVersion> = manifest_metas
+            .into_iter()
+            .filter_map(|meta| {
+                let filename = meta.location.filename()?;
+                let version_str = filename.strip_suffix(".manifest")?;
+                if version_str.starts_with('d') {
+                    return None;
+                }
+                let file_version: u64 = version_str.parse().ok()?;
+
+                let actual_version = if file_version > u64::MAX / 2 {
+                    u64::MAX - file_version
+                } else {
+                    file_version
+                };
+
+                // Use full path from object_store (relative to object store root)
+                Some(TableVersion {
+                    version: actual_version as i64,
+                    manifest_path: meta.location.to_string(),
+                    manifest_size: Some(meta.size as i64),
+                    e_tag: meta.e_tag,
+                    timestamp_millis: Some(meta.last_modified.timestamp_millis()),
+                    metadata: None,
+                })
+            })
+            .collect();
+
+        let list_is_ordered = self.object_store.list_is_lexically_ordered;
+        let want_descending = request.descending == Some(true);
+
+        let needs_sort = if list_is_ordered {
+            if is_v2_naming {
+                !want_descending
+            } else {
+                want_descending
+            }
+        } else {
+            true
+        };
+
+        if needs_sort {
+            if want_descending {
+                table_versions.sort_by(|a, b| b.version.cmp(&a.version));
+            } else {
+                table_versions.sort_by(|a, b| a.version.cmp(&b.version));
+            }
+        }
+
+        if let Some(limit) = request.limit {
+            table_versions.truncate(limit as usize);
+        }
+
+        Ok(ListTableVersionsResponse {
+            versions: table_versions,
+            page_token: None,
+        })
+    }
+
+    async fn create_table_version(
+        &self,
+        request: CreateTableVersionRequest,
+    ) -> Result<CreateTableVersionResponse> {
+        self.record_op("create_table_version");
+        let table_uri = self.resolve_table_location(&request.id).await?;
+
+        let staging_manifest_path = &request.manifest_path;
+        let version = request.version as u64;
+
+        let table_path = Self::uri_to_object_store_path(&table_uri);
+
+        // Determine naming scheme from request, default to V2
+        let naming_scheme = match request.naming_scheme.as_deref() {
+            Some("V1") => ManifestNamingScheme::V1,
+            _ => ManifestNamingScheme::V2,
+        };
+
+        // Compute final path using the naming scheme
+        let final_path = naming_scheme.manifest_path(&table_path, version);
+
+        let staging_path = Self::uri_to_object_store_path(staging_manifest_path);
+        let manifest_data = self
+            .object_store
+            .inner
+            .get(&staging_path)
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to read staging manifest at '{}': {}",
+                        staging_manifest_path, e
+                    ),
+                })
+            })?
+            .bytes()
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to read staging manifest bytes at '{}': {}",
+                        staging_manifest_path, e
+                    ),
+                })
+            })?;
+
+        let manifest_size = manifest_data.len() as i64;
+
+        let put_result = self
+            .object_store
+            .inner
+            .put_opts(
+                &final_path,
+                manifest_data.into(),
+                PutOptions {
+                    mode: PutMode::Create,
+                    ..Default::default()
+                },
+            )
+            .await
+            .map_err(|e| match e {
+                object_store::Error::AlreadyExists { .. }
+                | object_store::Error::Precondition { .. } => {
+                    lance_core::Error::from(NamespaceError::ConcurrentModification {
+                        message: format!(
+                            "Version {} already exists for table at '{}'",
+                            version, table_uri
+                        ),
+                    })
+                }
+                _ => lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to create version {} for table at '{}': {}",
+                        version, table_uri, e
+                    ),
+                }),
+            })?;
+
+        // Delete the staging manifest after successful copy
+        if let Err(e) = self.object_store.inner.delete(&staging_path).await {
+            log::warn!(
+                "Failed to delete staging manifest at '{}': {:?}",
+                staging_path,
+                e
+            );
+        }
+
+        // If table_version_storage_enabled is enabled, also record in __manifest (best-effort)
+        if self.table_version_storage_enabled
+            && let Some(ref manifest_ns) = self.manifest_ns
+        {
+            let table_id_str =
+                manifest::ManifestNamespace::str_object_id(&request.id.clone().unwrap_or_default());
+            let object_id =
+                manifest::ManifestNamespace::build_version_object_id(&table_id_str, version as i64);
+            let metadata_json = serde_json::json!({
+                "manifest_path": final_path.to_string(),
+                "manifest_size": manifest_size,
+                "e_tag": put_result.e_tag,
+                "naming_scheme": request.naming_scheme.as_deref().unwrap_or("V2"),
+            })
+            .to_string();
+
+            if let Err(e) = manifest_ns
+                .insert_into_manifest_with_metadata(
+                    vec![manifest::ManifestEntry {
+                        object_id,
+                        object_type: manifest::ObjectType::TableVersion,
+                        location: None,
+                        metadata: Some(metadata_json),
+                    }],
+                    None,
+                )
+                .await
+            {
+                log::warn!(
+                    "Failed to record table version in __manifest (best-effort): {:?}",
+                    e
+                );
+            }
+        }
+
+        Ok(CreateTableVersionResponse {
+            transaction_id: None,
+            version: Some(Box::new(TableVersion {
+                version: version as i64,
+                manifest_path: final_path.to_string(),
+                manifest_size: Some(manifest_size),
+                e_tag: put_result.e_tag,
+                timestamp_millis: None,
+                metadata: None,
+            })),
+        })
+    }
+
+    async fn describe_table_version(
+        &self,
+        request: DescribeTableVersionRequest,
+    ) -> Result<DescribeTableVersionResponse> {
+        self.record_op("describe_table_version");
+        // When table_version_storage_enabled and a specific version is requested,
+        // query from __manifest to avoid opening the entire dataset
+        if self.table_version_storage_enabled
+            && let (Some(manifest_ns), Some(version)) = (&self.manifest_ns, request.version)
+        {
+            let table_id = request.id.clone().unwrap_or_default();
+            return manifest_ns.describe_table_version(&table_id, version).await;
+        }
+
+        // Fallback when table_version_storage is not enabled: open the dataset to describe the version
+        let table_uri = self.resolve_table_location(&request.id).await?;
+
+        // Use DatasetBuilder with storage options to support S3 with custom endpoints
+        let mut builder = DatasetBuilder::from_uri(&table_uri);
+        if let Some(opts) = &self.storage_options {
+            builder = builder.with_storage_options(opts.clone());
+        }
+        if let Some(sess) = &self.session {
+            builder = builder.with_session(sess.clone());
+        }
+        let mut dataset = builder.load().await.map_err(|e| {
+            lance_core::Error::from(NamespaceError::TableNotFound {
+                message: format!("Failed to open table at '{}': {:?}", table_uri, e),
+            })
+        })?;
+
+        if let Some(version) = request.version {
+            dataset = dataset
+                .checkout_version(version as u64)
+                .await
+                .map_err(|e| {
+                    lance_core::Error::from(NamespaceError::TableVersionNotFound {
+                        message: format!(
+                            "Failed to checkout version {} for table at '{}': {}",
+                            version, table_uri, e
+                        ),
+                    })
+                })?;
+        }
+
+        let version_info = dataset.version();
+        let manifest_location = dataset.manifest_location();
+        let metadata: std::collections::HashMap<String, String> =
+            version_info.metadata.into_iter().collect();
+
+        let table_version = TableVersion {
+            version: version_info.version as i64,
+            manifest_path: manifest_location.path.to_string(),
+            manifest_size: manifest_location.size.map(|s| s as i64),
+            e_tag: manifest_location.e_tag.clone(),
+            timestamp_millis: Some(version_info.timestamp.timestamp_millis()),
+            metadata: if metadata.is_empty() {
+                None
+            } else {
+                Some(metadata)
+            },
+        };
+
+        Ok(DescribeTableVersionResponse {
+            version: Box::new(table_version),
+        })
+    }
+
+    async fn batch_delete_table_versions(
+        &self,
+        request: BatchDeleteTableVersionsRequest,
+    ) -> Result<BatchDeleteTableVersionsResponse> {
+        self.record_op("batch_delete_table_versions");
+        // Single-table mode: use `id` (from path parameter) + `ranges` to delete
+        // versions from one table.
+        let ranges: Vec<(i64, i64)> = request
+            .ranges
+            .iter()
+            .map(|r| {
+                let start = r.start_version;
+                let end = if r.end_version > 0 {
+                    r.end_version
+                } else {
+                    start
+                };
+                (start, end)
+            })
+            .collect();
+        let table_entries = vec![TableDeleteEntry {
+            table_id: request.id.clone(),
+            ranges,
+        }];
+
+        let mut total_deleted_count = 0i64;
+
+        if self.table_version_storage_enabled
+            && let Some(ref manifest_ns) = self.manifest_ns
+        {
+            // Phase 1 (atomic commit point): Delete version records from __manifest
+            // for ALL tables in a single atomic operation. This is the authoritative
+            // source of truth — once __manifest entries are removed, the versions
+            // are logically deleted across all tables atomically.
+
+            // Collect all (table_id_str, ranges) for batch deletion
+            let mut all_object_ids: Vec<String> = Vec::new();
+            for te in &table_entries {
+                let table_id_str = manifest::ManifestNamespace::str_object_id(
+                    &te.table_id.clone().unwrap_or_default(),
+                );
+                for (start, end) in &te.ranges {
+                    for version in *start..=*end {
+                        let object_id = manifest::ManifestNamespace::build_version_object_id(
+                            &table_id_str,
+                            version,
+                        );
+                        all_object_ids.push(object_id);
+                    }
+                }
+            }
+
+            if !all_object_ids.is_empty() {
+                total_deleted_count = manifest_ns
+                    .batch_delete_table_versions_by_object_ids(&all_object_ids)
+                    .await?;
+            }
+
+            // Phase 2: Delete physical manifest files (best-effort).
+            // Even if some file deletions fail, the versions are already removed from
+            // __manifest, so they won't be visible to readers. Leftover files are
+            // orphaned but harmless and can be cleaned up later.
+            let _ = self
+                .delete_physical_version_files(&table_entries, true)
+                .await;
+
+            return Ok(BatchDeleteTableVersionsResponse {
+                deleted_count: Some(total_deleted_count),
+                transaction_id: None,
+            });
+        }
+
+        // Fallback when table_version_storage is not enabled: delete physical files directly (no __manifest)
+        total_deleted_count = self
+            .delete_physical_version_files(&table_entries, false)
+            .await?;
+
+        Ok(BatchDeleteTableVersionsResponse {
+            deleted_count: Some(total_deleted_count),
+            transaction_id: None,
+        })
+    }
+
+    async fn create_table_index(
+        &self,
+        request: CreateTableIndexRequest,
+    ) -> Result<CreateTableIndexResponse> {
+        self.record_op("create_table_index");
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let mut dataset = self
+            .load_dataset(&table_uri, None, "create_table_index")
+            .await?;
+        let index_request = Self::build_index_params(&request)?;
+
+        dataset
+            .create_index(
+                &[request.column.as_str()],
+                index_request.index_type(),
+                request.name.clone(),
+                index_request.params(),
+                false,
+            )
+            .await
+            .map_err(|e| {
+                let err_msg = format!("{}", e);
+                let ns_err = if err_msg.contains("already exists") {
+                    NamespaceError::TableIndexAlreadyExists {
+                        message: format!(
+                            "Index '{}' already exists on table '{}': {:?}",
+                            request.name.as_deref().unwrap_or("<auto-generated>"),
+                            table_uri,
+                            e
+                        ),
+                    }
+                } else if err_msg.contains("not found") || err_msg.contains("does not exist") {
+                    NamespaceError::TableColumnNotFound {
+                        message: format!(
+                            "Column '{}' not found for table '{}': {:?}",
+                            request.column, table_uri, e
+                        ),
+                    }
+                } else {
+                    NamespaceError::Internal {
+                        message: format!(
+                            "Failed to create {} index '{}' on column '{}' for table '{}': {:?}",
+                            request.index_type,
+                            request.name.as_deref().unwrap_or("<auto-generated>"),
+                            request.column,
+                            table_uri,
+                            e
+                        ),
+                    }
+                };
+                lance_core::Error::from(ns_err)
+            })?;
+
+        let transaction_id = dataset
+            .read_transaction()
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to read committed transaction after creating index on '{}': {}",
+                        table_uri, e
+                    ),
+                })
+            })?
+            .map(|transaction| transaction.uuid);
+
+        Ok(CreateTableIndexResponse { transaction_id })
+    }
+
+    async fn list_table_indices(
+        &self,
+        request: ListTableIndicesRequest,
+    ) -> Result<ListTableIndicesResponse> {
+        self.record_op("list_table_indices");
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let dataset = self
+            .load_dataset(&table_uri, request.version, "list_table_indices")
+            .await?;
+        let mut indices = dataset
+            .describe_indices(None)
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to describe table indices for '{}': {:?}", table_uri, e),
+                })
+            })?
+            .into_iter()
+            .filter(|description| {
+                description
+                    .metadata()
+                    .first()
+                    .map(|metadata| !is_system_index(metadata))
+                    .unwrap_or(false)
+            })
+            .map(|description| {
+                let columns = description
+                    .field_ids()
+                    .iter()
+                        .map(|field_id| {
+                        dataset
+                            .schema()
+                            .field_path(i32::try_from(*field_id).map_err(|e| {
+                                lance_core::Error::from(NamespaceError::Internal {
+                                    message: format!(
+                                        "Field id {} does not fit in i32 for table '{}': {}",
+                                        field_id, table_uri, e
+                                    ),
+                                })
+                            })?)
+                            .map_err(|e| {
+                            lance_core::Error::from(NamespaceError::Internal {
+                                message: format!(
+                                    "Failed to resolve field path for field_id {} in table '{}': {}",
+                                    field_id, table_uri, e
+                                ),
+                            })
+                        })
+                    })
+                    .collect::<Result<Vec<_>>>()?;
+
+                Ok(IndexContent {
+                    index_name: description.name().to_string(),
+                    index_uuid: description.metadata()[0].uuid.to_string(),
+                    columns,
+                    status: "SUCCEEDED".to_string(),
+                })
+            })
+            .collect::<Result<Vec<_>>>()?;
+
+        let page_token = Self::paginate_indices(&mut indices, request.page_token, request.limit);
+        Ok(ListTableIndicesResponse {
+            indexes: indices,
+            page_token,
+        })
+    }
+
+    async fn describe_table_index_stats(
+        &self,
+        request: DescribeTableIndexStatsRequest,
+    ) -> Result<DescribeTableIndexStatsResponse> {
+        self.record_op("describe_table_index_stats");
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let dataset = self
+            .load_dataset(&table_uri, request.version, "describe_table_index_stats")
+            .await?;
+        let index_name = request.index_name.as_deref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Index name is required for describe_table_index_stats".to_string(),
+            })
+        })?;
+        let metadatas = dataset
+            .load_indices_by_name(index_name)
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::TableIndexNotFound {
+                    message: format!(
+                        "Failed to load index '{}' metadata for table '{}': {}",
+                        index_name, table_uri, e
+                    ),
+                })
+            })?;
+        if metadatas.first().is_some_and(is_system_index) {
+            return Err(NamespaceError::Unsupported {
+                message: format!("System index '{}' is not exposed by this API", index_name),
+            }
+            .into());
+        }
+
+        let stats = <Dataset as DatasetIndexExt>::index_statistics(&dataset, index_name)
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::TableIndexNotFound {
+                    message: format!(
+                        "Failed to describe index statistics for '{}' on table '{}': {}",
+                        index_name, table_uri, e
+                    ),
+                })
+            })?;
+        let stats: serde_json::Value = serde_json::from_str(&stats).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!(
+                    "Failed to parse index statistics for '{}' on table '{}': {}",
+                    index_name, table_uri, e
+                ),
+            })
+        })?;
+
+        Ok(Self::describe_table_index_stats_response(&stats))
+    }
+
+    async fn describe_transaction(
+        &self,
+        request: DescribeTransactionRequest,
+    ) -> Result<DescribeTransactionResponse> {
+        self.record_op("describe_transaction");
+        let mut request_id = request.id.ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Transaction id must include table id and transaction identifier"
+                    .to_string(),
+            })
+        })?;
+        if request_id.len() < 2 {
+            return Err(NamespaceError::InvalidInput {
+                message: format!(
+                    "Transaction request id must include table id and transaction identifier, got {:?}",
+                    request_id
+                ),
+            }
+            .into());
+        }
+
+        let id = request_id.pop().expect("request_id len checked above");
+        let table_id = Some(request_id);
+        let table_uri = self.resolve_table_location(&table_id).await?;
+        let dataset = self
+            .load_dataset(&table_uri, None, "describe_transaction")
+            .await?;
+        let (version, transaction) = self.find_transaction(&dataset, &id).await?;
+
+        Ok(Self::transaction_response(version, &transaction))
+    }
+
+    async fn create_table_scalar_index(
+        &self,
+        request: CreateTableIndexRequest,
+    ) -> Result<CreateTableScalarIndexResponse> {
+        self.record_op("create_table_scalar_index");
+        let index_type = Self::parse_index_type(&request.index_type)?;
+        if !index_type.is_scalar() {
+            return Err(NamespaceError::InvalidInput {
+                message: format!(
+                    "create_table_scalar_index only supports scalar index types, got {}",
+                    request.index_type
+                ),
+            }
+            .into());
+        }
+
+        let response = self.create_table_index(request).await?;
+        Ok(CreateTableScalarIndexResponse {
+            transaction_id: response.transaction_id,
+        })
+    }
+
+    async fn drop_table_index(
+        &self,
+        request: DropTableIndexRequest,
+    ) -> Result<DropTableIndexResponse> {
+        self.record_op("drop_table_index");
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let index_name = request.index_name.as_deref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Index name is required for drop_table_index".to_string(),
+            })
+        })?;
+        let mut dataset = self
+            .load_dataset(&table_uri, None, "drop_table_index")
+            .await?;
+        let metadatas = dataset
+            .load_indices_by_name(index_name)
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::TableIndexNotFound {
+                    message: format!(
+                        "Failed to load index '{}' before dropping it from table '{}': {}",
+                        index_name, table_uri, e
+                    ),
+                })
+            })?;
+        if metadatas.first().is_some_and(is_system_index) {
+            return Err(NamespaceError::Unsupported {
+                message: format!(
+                    "System index '{}' cannot be dropped via this API",
+                    index_name
+                ),
+            }
+            .into());
+        }
+
+        dataset.drop_index(index_name).await.map_err(|e| {
+            lance_core::Error::from(NamespaceError::TableIndexNotFound {
+                message: format!(
+                    "Failed to drop index '{}' from table '{}': {}",
+                    index_name, table_uri, e
+                ),
+            })
+        })?;
+
+        let transaction_id = dataset
+            .read_transaction()
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to read committed transaction after dropping index '{}' from '{}': {}",
+                        index_name, table_uri, e
+                    ),
+                })
+            })?
+            .map(|transaction| transaction.uuid);
+
+        Ok(DropTableIndexResponse { transaction_id })
+    }
+
+    async fn list_all_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
+        // In dir-only mode there are no child namespaces, so all tables live in the
+        // root directory. This is equivalent to listing the root namespace.
+        let mut tables = self.list_directory_tables().await?;
+        Self::apply_pagination(&mut tables, request.page_token, request.limit);
+        Ok(ListTablesResponse::new(tables))
+    }
+
+    async fn restore_table(&self, request: RestoreTableRequest) -> Result<RestoreTableResponse> {
+        let version = request.version;
+        if version < 0 {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Table version for restore_table must be non-negative, got {}",
+                    version
+                )
+                .into(),
+            ));
+        }
+
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let mut dataset = self.load_dataset(&table_uri, None, "restore_table").await?;
+
+        dataset = dataset
+            .checkout_version(version as u64)
+            .await
+            .map_err(|e| {
+                Error::namespace_source(
+                    format!(
+                        "Failed to checkout version {} for restore at '{}': {}",
+                        version, table_uri, e
+                    )
+                    .into(),
+                )
+            })?;
+
+        dataset.restore().await.map_err(|e| {
+            Error::namespace_source(
+                format!(
+                    "Failed to restore table at '{}' to version {}: {}",
+                    table_uri, version, e
+                )
+                .into(),
+            )
+        })?;
+
+        let transaction_id = dataset
+            .read_transaction()
+            .await
+            .map_err(|e| {
+                Error::namespace_source(
+                    format!(
+                        "Failed to read transaction after restoring '{}': {}",
+                        table_uri, e
+                    )
+                    .into(),
+                )
+            })?
+            .map(|t| t.uuid);
+
+        Ok(RestoreTableResponse { transaction_id })
+    }
+
+    async fn update_table_schema_metadata(
+        &self,
+        request: UpdateTableSchemaMetadataRequest,
+    ) -> Result<UpdateTableSchemaMetadataResponse> {
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let mut dataset = self
+            .load_dataset(&table_uri, None, "update_table_schema_metadata")
+            .await?;
+
+        let new_metadata = request.metadata.unwrap_or_default();
+        let updated_metadata = dataset
+            .update_schema_metadata(new_metadata.iter().map(|(k, v)| (k.as_str(), v.as_str())))
+            .await
+            .map_err(|e| {
+                Error::namespace_source(
+                    format!(
+                        "Failed to update schema metadata for table at '{}': {}",
+                        table_uri, e
+                    )
+                    .into(),
+                )
+            })?;
+
+        let transaction_id = dataset
+            .read_transaction()
+            .await
+            .map_err(|e| {
+                Error::namespace_source(
+                    format!(
+                        "Failed to read transaction after updating metadata for '{}': {}",
+                        table_uri, e
+                    )
+                    .into(),
+                )
+            })?
+            .map(|t| t.uuid);
+
+        Ok(UpdateTableSchemaMetadataResponse {
+            metadata: Some(updated_metadata),
+            transaction_id,
+        })
+    }
+
+    async fn get_table_stats(
+        &self,
+        request: GetTableStatsRequest,
+    ) -> Result<GetTableStatsResponse> {
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let dataset = Arc::new(
+            self.load_dataset(&table_uri, None, "get_table_stats")
+                .await?,
+        );
+
+        // Compute total bytes on disk using field-level statistics
+        let data_stats = dataset.calculate_data_stats().await.map_err(|e| {
+            Error::namespace_source(
+                format!(
+                    "Failed to calculate data statistics for table at '{}': {}",
+                    table_uri, e
+                )
+                .into(),
+            )
+        })?;
+        let total_bytes: i64 = data_stats
+            .fields
+            .iter()
+            .map(|f| f.bytes_on_disk as i64)
+            .sum();
+
+        // Collect per-fragment row counts
+        let fragment_row_futures: Vec<_> = dataset
+            .get_fragments()
+            .into_iter()
+            .map(|f| async move { f.physical_rows().await })
+            .collect();
+        let fragment_row_results = futures::future::join_all(fragment_row_futures).await;
+        let mut fragment_row_counts: Vec<i64> = fragment_row_results
+            .into_iter()
+            .filter_map(|r| r.ok())
+            .map(|r| r as i64)
+            .collect();
+
+        let num_fragments = fragment_row_counts.len() as i64;
+        let num_rows: i64 = fragment_row_counts.iter().sum();
+
+        // Fragments with fewer rows than the compaction target are considered "small",
+        // consistent with CompactionOptions::target_rows_per_fragment default.
+        const SMALL_FRAGMENT_THRESHOLD: i64 = 1024 * 1024;
+        let num_small_fragments = fragment_row_counts
+            .iter()
+            .filter(|&&r| r < SMALL_FRAGMENT_THRESHOLD)
+            .count() as i64;
+
+        // Compute length summary statistics
+        fragment_row_counts.sort_unstable();
+        let lengths = if fragment_row_counts.is_empty() {
+            FragmentSummary::new(0, 0, 0, 0, 0, 0, 0)
+        } else {
+            let len = fragment_row_counts.len();
+            let min = fragment_row_counts[0];
+            let max = fragment_row_counts[len - 1];
+            let mean = num_rows / num_fragments;
+            let pct = |p: f64| fragment_row_counts[((len - 1) as f64 * p) as usize];
+            FragmentSummary::new(min, max, mean, pct(0.25), pct(0.50), pct(0.75), pct(0.99))
+        };
+
+        // Count non-system indices
+        let indices = dataset.load_indices().await.map_err(|e| {
+            Error::namespace_source(
+                format!("Failed to load indices for table at '{}': {}", table_uri, e).into(),
+            )
+        })?;
+        let num_indices = indices.iter().filter(|m| !is_system_index(m)).count() as i64;
+
+        let fragment_stats = FragmentStats::new(num_fragments, num_small_fragments, lengths);
+        Ok(GetTableStatsResponse::new(
+            total_bytes,
+            num_rows,
+            num_indices,
+            fragment_stats,
+        ))
+    }
+
+    async fn explain_table_query_plan(
+        &self,
+        request: ExplainTableQueryPlanRequest,
+    ) -> Result<String> {
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let dataset = self
+            .load_dataset(
+                &table_uri,
+                request.query.version,
+                "explain_table_query_plan",
+            )
+            .await?;
+        let verbose = request.verbose.unwrap_or(false);
+
+        let mut scanner = dataset.scan();
+        Self::apply_query_params_to_scanner(
+            &mut scanner,
+            request.query.filter.as_deref(),
+            request.query.columns.as_deref(),
+            request.query.vector_column.as_deref(),
+            &request.query.vector,
+            request.query.k,
+            request.query.offset,
+            request.query.prefilter,
+            request.query.bypass_vector_index,
+            request.query.nprobes,
+            request.query.ef,
+            request.query.refine_factor,
+            request.query.distance_type.as_deref(),
+            request.query.fast_search,
+            request.query.with_row_id,
+            request.query.lower_bound,
+            request.query.upper_bound,
+            "explain_table_query_plan",
+        )?;
+
+        scanner.explain_plan(verbose).await.map_err(|e| {
+            Error::namespace_source(
+                format!(
+                    "Failed to explain query plan for table at '{}': {}",
+                    table_uri, e
+                )
+                .into(),
+            )
+        })
+    }
+
+    async fn analyze_table_query_plan(
+        &self,
+        request: AnalyzeTableQueryPlanRequest,
+    ) -> Result<String> {
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let dataset = self
+            .load_dataset(&table_uri, request.version, "analyze_table_query_plan")
+            .await?;
+
+        let mut scanner = dataset.scan();
+        Self::apply_query_params_to_scanner(
+            &mut scanner,
+            request.filter.as_deref(),
+            request.columns.as_deref(),
+            request.vector_column.as_deref(),
+            &request.vector,
+            request.k,
+            request.offset,
+            request.prefilter,
+            request.bypass_vector_index,
+            request.nprobes,
+            request.ef,
+            request.refine_factor,
+            request.distance_type.as_deref(),
+            request.fast_search,
+            request.with_row_id,
+            request.lower_bound,
+            request.upper_bound,
+            "analyze_table_query_plan",
+        )?;
+
+        scanner.analyze_plan().await.map_err(|e| {
+            Error::namespace_source(
+                format!(
+                    "Failed to analyze query plan for table at '{}': {}",
+                    table_uri, e
+                )
+                .into(),
+            )
+        })
+    }
+
+    async fn count_table_rows(&self, request: CountTableRowsRequest) -> Result<i64> {
+        self.record_op("count_table_rows");
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let dataset = self
+            .load_dataset(&table_uri, request.version, "count_table_rows")
+            .await?;
+
+        let count =
+            dataset
+                .count_rows(request.predicate)
+                .await
+                .map_err(|e| NamespaceError::Internal {
+                    message: format!("Failed to count rows for table at '{}': {:?}", table_uri, e),
+                })?;
+
+        Ok(count as i64)
+    }
+
+    async fn insert_into_table(
+        &self,
+        request: InsertIntoTableRequest,
+        request_data: Bytes,
+    ) -> Result<InsertIntoTableResponse> {
+        self.record_op("insert_into_table");
+        let table_uri = self.resolve_table_location(&request.id).await?;
+
+        if request_data.is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "Request data (Arrow IPC stream) is required for insert_into_table"
+                    .to_string(),
+            }
+            .into());
+        }
+
+        let cursor = Cursor::new(request_data.as_ref());
+        let stream_reader =
+            StreamReader::try_new(cursor, None).map_err(|e| NamespaceError::InvalidInput {
+                message: format!("Invalid Arrow IPC stream: {:?}", e),
+            })?;
+        let arrow_schema = stream_reader.schema();
+
+        let mut batches = Vec::new();
+        for batch_result in stream_reader {
+            batches.push(batch_result.map_err(|e| NamespaceError::InvalidInput {
+                message: format!("Failed to read batch from IPC stream: {:?}", e),
+            })?);
+        }
+
+        let reader = if batches.is_empty() {
+            let batch = arrow::record_batch::RecordBatch::new_empty(arrow_schema.clone());
+            let batches = vec![Ok(batch)];
+            RecordBatchIterator::new(batches, arrow_schema.clone())
+        } else {
+            let batch_results: Vec<_> = batches.into_iter().map(Ok).collect();
+            RecordBatchIterator::new(batch_results, arrow_schema)
+        };
+
+        let mode = match request.mode.as_deref() {
+            Some(m) if m.eq_ignore_ascii_case("overwrite") => WriteMode::Overwrite,
+            Some(m) if m.eq_ignore_ascii_case("append") => WriteMode::Append,
+            None => WriteMode::Append,
+            Some(m) => {
+                return Err(lance_namespace::error::NamespaceError::InvalidInput {
+                    message: format!(
+                        "Unsupported write mode '{}'. Supported modes are: 'append', 'overwrite'",
+                        m
+                    ),
+                }
+                .into());
+            }
+        };
+
+        let store_params = self.storage_options.as_ref().map(|opts| ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(
+                lance_io::object_store::StorageOptionsAccessor::with_static_options(opts.clone()),
+            )),
+            ..Default::default()
+        });
+
+        let write_params = WriteParams {
+            mode,
+            store_params,
+            session: self.session.clone(),
+            ..Default::default()
+        };
+
+        Dataset::write(reader, &table_uri, Some(write_params))
+            .await
+            .map_err(|e| {
+                let err_msg = format!("{}", e);
+                if err_msg.contains("conflict") || err_msg.contains("CommitConflict") {
+                    NamespaceError::ConcurrentModification {
+                        message: format!(
+                            "Concurrent modification on table at '{}': {:?}",
+                            table_uri, e
+                        ),
+                    }
+                } else {
+                    NamespaceError::Internal {
+                        message: format!("Failed to insert into table at '{}': {:?}", table_uri, e),
+                    }
+                }
+            })?;
+
+        Ok(InsertIntoTableResponse {
+            transaction_id: None,
+        })
+    }
+
+    async fn query_table(&self, request: QueryTableRequest) -> Result<Bytes> {
+        use arrow::ipc::writer::FileWriter;
+
+        self.record_op("query_table");
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let dataset = self
+            .load_dataset(&table_uri, request.version, "query_table")
+            .await?;
+
+        // Build scanner
+        let mut scanner = dataset.scan();
+
+        // Check if this is a vector search query
+        // vector is Box<QueryTableRequestVector>, not Option
+        let has_vector_query = request
+            .vector
+            .single_vector
+            .as_ref()
+            .map(|sv| !sv.is_empty())
+            .unwrap_or(false)
+            || request
+                .vector
+                .multi_vector
+                .as_ref()
+                .map(|mv| !mv.is_empty())
+                .unwrap_or(false);
+
+        // Apply prefilter setting (must be set before nearest)
+        if let Some(prefilter) = request.prefilter {
+            scanner.prefilter(prefilter);
+        }
+
+        // Apply vector search if query vector is provided
+        if has_vector_query {
+            let vector_column = request.vector_column.as_deref().unwrap_or("vector");
+
+            // Get the query vector(s)
+            let query_vector: Vec<f32> = request
+                .vector
+                .single_vector
+                .clone()
+                .or_else(|| {
+                    request
+                        .vector
+                        .multi_vector
+                        .as_ref()
+                        .and_then(|mv| mv.first().cloned())
+                })
+                .unwrap_or_default();
+
+            if !query_vector.is_empty() {
+                let k = if request.k > 0 {
+                    request.k as usize
+                } else {
+                    10
+                };
+                let query_array = Float32Array::from(query_vector);
+                scanner
+                    .nearest(vector_column, &query_array, k)
+                    .map_err(|e| NamespaceError::InvalidInput {
+                        message: format!("Invalid vector search: {:?}", e),
+                    })?;
+
+                // Apply distance type if specified
+                if let Some(ref distance_type) = request.distance_type {
+                    let metric = match distance_type.to_lowercase().as_str() {
+                        "l2" | "euclidean" => MetricType::L2,
+                        "cosine" => MetricType::Cosine,
+                        "dot" | "inner_product" => MetricType::Dot,
+                        "hamming" => MetricType::Hamming,
+                        _ => {
+                            return Err(NamespaceError::InvalidInput {
+                                message: format!("Unknown distance type: {}", distance_type),
+                            }
+                            .into());
+                        }
+                    };
+                    scanner.distance_metric(metric);
+                }
+
+                // Apply nprobes if specified (maps to minimum_nprobes, matching lancedb behavior)
+                if let Some(nprobes) = request.nprobes {
+                    scanner.minimum_nprobes(nprobes as usize);
+                }
+
+                // Apply ef (HNSW search effort) if specified
+                if let Some(ef) = request.ef {
+                    scanner.ef(ef as usize);
+                }
+
+                // Apply refine_factor if specified
+                if let Some(refine_factor) = request.refine_factor {
+                    scanner.refine(refine_factor as u32);
+                }
+
+                // Apply distance bounds if specified
+                if request.lower_bound.is_some() || request.upper_bound.is_some() {
+                    scanner.distance_range(request.lower_bound, request.upper_bound);
+                }
+
+                // Apply use_index (inverse of bypass_vector_index)
+                if let Some(bypass) = request.bypass_vector_index {
+                    scanner.use_index(!bypass);
+                }
+
+                // Apply fast_search if specified
+                if request.fast_search == Some(true) {
+                    scanner.fast_search();
+                }
+            }
+        }
+
+        // Apply full text search if specified
+        if let Some(ref fts_query) = request.full_text_query {
+            // Handle string_query (simple string FTS)
+            if let Some(ref string_query) = fts_query.string_query {
+                let mut fts = FullTextSearchQuery::new(string_query.query.clone());
+
+                // Apply column filter if specified
+                if let Some(ref columns) = string_query.columns
+                    && !columns.is_empty()
+                {
+                    fts = fts
+                        .with_columns(columns)
+                        .map_err(|e| NamespaceError::InvalidInput {
+                            message: format!("Invalid FTS columns: {:?}", e),
+                        })?;
+                }
+
+                scanner
+                    .full_text_search(fts)
+                    .map_err(|e| NamespaceError::InvalidInput {
+                        message: format!("Invalid full text search: {:?}", e),
+                    })?;
+            }
+            // Note: structured_query would require more complex parsing
+            // For now, we only support string_query
+        }
+
+        // Apply column projection if specified
+        if let Some(ref columns) = request.columns {
+            if let Some(ref column_names) = columns.column_names
+                && !column_names.is_empty()
+            {
+                scanner
+                    .project(column_names)
+                    .map_err(|e| NamespaceError::InvalidInput {
+                        message: format!("Invalid column projection: {:?}", e),
+                    })?;
+            } else if let Some(ref column_aliases) = columns.column_aliases
+                && !column_aliases.is_empty()
+            {
+                // column_aliases is HashMap<String, String> where key is alias, value is SQL expression
+                let transform_pairs: Vec<(String, String)> = column_aliases
+                    .iter()
+                    .map(|(alias, sql)| (alias.clone(), sql.clone()))
+                    .collect();
+                scanner
+                    .project_with_transform(
+                        &transform_pairs
+                            .iter()
+                            .map(|(a, s)| (a.as_str(), s.as_str()))
+                            .collect::<Vec<_>>(),
+                    )
+                    .map_err(|e| NamespaceError::InvalidInput {
+                        message: format!("Invalid column alias expression: {:?}", e),
+                    })?;
+            }
+        }
+
+        // Apply filter if specified
+        if let Some(ref filter) = request.filter
+            && !filter.is_empty()
+        {
+            scanner
+                .filter(filter)
+                .map_err(|e| NamespaceError::InvalidInput {
+                    message: format!("Invalid filter expression: {:?}", e),
+                })?;
+        }
+
+        // Apply with_row_id if requested
+        if request.with_row_id == Some(true) {
+            scanner.with_row_id();
+        }
+
+        // Apply limit if specified (k is the number of results to return)
+        // k == 0 means no limit
+        // Note: For vector search, limit is already applied via nearest()
+        if !has_vector_query && request.k > 0 {
+            let offset = request.offset.map(|o| o as i64);
+            scanner.limit(Some(request.k as i64), offset).map_err(|e| {
+                NamespaceError::InvalidInput {
+                    message: format!("Invalid limit/offset: {:?}", e),
+                }
+            })?;
+        } else if has_vector_query && request.offset.is_some() {
+            // For vector search, offset is handled separately
+            let offset = request.offset.map(|o| o as i64);
+            scanner
+                .limit(None, offset)
+                .map_err(|e| NamespaceError::InvalidInput {
+                    message: format!("Invalid offset: {:?}", e),
+                })?;
+        }
+
+        // Execute the scan and collect results
+        let batch = scanner
+            .try_into_batch()
+            .await
+            .map_err(|e| NamespaceError::Internal {
+                message: format!("Failed to execute query: {:?}", e),
+            })?;
+
+        // Serialize to Arrow IPC file format
+        let schema = batch.schema();
+        let mut buffer = Vec::new();
+        {
+            let mut writer = FileWriter::try_new(&mut buffer, &schema).map_err(|e| {
+                NamespaceError::Internal {
+                    message: format!("Failed to create IPC writer: {:?}", e),
+                }
+            })?;
+            writer.write(&batch).map_err(|e| NamespaceError::Internal {
+                message: format!("Failed to write batch to IPC: {:?}", e),
+            })?;
+            writer.finish().map_err(|e| NamespaceError::Internal {
+                message: format!("Failed to finish IPC writer: {:?}", e),
+            })?;
+        }
+
+        Ok(Bytes::from(buffer))
+    }
+
+    fn namespace_id(&self) -> String {
+        format!("DirectoryNamespace {{ root: {:?} }}", self.root)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow_ipc::reader::{FileReader, StreamReader};
+    use lance::dataset::Dataset;
+    use lance::index::DatasetIndexExt;
+    use lance_core::utils::tempfile::{TempStdDir, TempStrDir};
+    use lance_core::utils::testing::CountingObjectStore;
+    use lance_io::object_store::providers::local::FileStoreProvider;
+    use lance_namespace::models::{
+        CreateTableRequest, JsonArrowDataType, JsonArrowField, JsonArrowSchema, ListTablesRequest,
+        QueryTableRequestColumns,
+    };
+    use lance_namespace::schema::convert_json_arrow_schema;
+    use std::io::Cursor;
+    use std::sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    };
+    use url::Url;
+
+    fn assert_plan_contains_all(plan: &str, expected_fragments: &[&str], context: &str) {
+        for expected_fragment in expected_fragments {
+            assert!(
+                plan.contains(expected_fragment),
+                "{}. Missing fragment: '{}'. Plan:\n{}",
+                context,
+                expected_fragment,
+                plan
+            );
+        }
+    }
+
+    /// Helper to create a test DirectoryNamespace with a temporary directory
+    async fn create_test_namespace() -> (DirectoryNamespace, TempStdDir) {
+        let temp_dir = TempStdDir::default();
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_dir.to_str().unwrap())
+            .build()
+            .await
+            .unwrap();
+        (namespace, temp_dir)
+    }
+
+    #[derive(Debug)]
+    struct CountingFileStoreProvider {
+        listing_count: Arc<AtomicUsize>,
+    }
+
+    #[async_trait]
+    impl lance_io::object_store::ObjectStoreProvider for CountingFileStoreProvider {
+        async fn new_store(
+            &self,
+            base_path: Url,
+            params: &ObjectStoreParams,
+        ) -> Result<ObjectStore> {
+            let provider = FileStoreProvider;
+            let mut store = provider.new_store(base_path, params).await?;
+            store.inner = Arc::new(CountingObjectStore::new(
+                store.inner.clone(),
+                self.listing_count.clone(),
+            ));
+            Ok(store)
+        }
+
+        fn extract_path(&self, url: &Url) -> Result<Path> {
+            let provider = FileStoreProvider;
+            provider.extract_path(url)
+        }
+
+        fn calculate_object_store_prefix(
+            &self,
+            url: &Url,
+            storage_options: Option<&HashMap<String, String>>,
+        ) -> Result<String> {
+            let provider = FileStoreProvider;
+            provider.calculate_object_store_prefix(url, storage_options)
+        }
+    }
+
+    fn file_object_store_uri(path: &str) -> String {
+        let path_prefix = if path.starts_with('/') { "" } else { "/" };
+        format!("file-object-store://{path_prefix}{path}")
+    }
+
+    fn build_listing_counting_session(listing_count: Arc<AtomicUsize>) -> Arc<Session> {
+        let registry = Arc::new(ObjectStoreRegistry::default());
+        registry.insert(
+            "file-object-store",
+            Arc::new(CountingFileStoreProvider { listing_count }),
+        );
+        Arc::new(Session::new(0, 0, registry))
+    }
+
+    /// Helper to create test IPC data from a schema
+    fn create_test_ipc_data(schema: &JsonArrowSchema) -> Vec<u8> {
+        use arrow::ipc::writer::StreamWriter;
+
+        let arrow_schema = convert_json_arrow_schema(schema).unwrap();
+        let arrow_schema = Arc::new(arrow_schema);
+        let batch = arrow::record_batch::RecordBatch::new_empty(arrow_schema.clone());
+        let mut buffer = Vec::new();
+        {
+            let mut writer = StreamWriter::try_new(&mut buffer, &arrow_schema).unwrap();
+            writer.write(&batch).unwrap();
+            writer.finish().unwrap();
+        }
+        buffer
+    }
+
+    fn create_ipc_data_from_batches(
+        schema: Arc<arrow_schema::Schema>,
+        batches: Vec<arrow::record_batch::RecordBatch>,
+    ) -> Vec<u8> {
+        use arrow::ipc::writer::StreamWriter;
+
+        let mut buffer = Vec::new();
+        {
+            let mut writer = StreamWriter::try_new(&mut buffer, &schema).unwrap();
+            for batch in &batches {
+                writer.write(batch).unwrap();
+            }
+            writer.finish().unwrap();
+        }
+        buffer
+    }
+
+    /// Helper to create a simple test schema
+    fn create_test_schema() -> JsonArrowSchema {
+        let int_type = JsonArrowDataType::new("int32".to_string());
+        let string_type = JsonArrowDataType::new("utf8".to_string());
+
+        let id_field = JsonArrowField {
+            name: "id".to_string(),
+            r#type: Box::new(int_type),
+            nullable: false,
+            metadata: None,
+        };
+
+        let name_field = JsonArrowField {
+            name: "name".to_string(),
+            r#type: Box::new(string_type),
+            nullable: true,
+            metadata: None,
+        };
+
+        JsonArrowSchema {
+            fields: vec![id_field, name_field],
+            metadata: None,
+        }
+    }
+
+    fn create_scalar_table_ipc_data() -> Vec<u8> {
+        use arrow::array::{Int32Array, StringArray};
+        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("name", DataType::Utf8, true),
+        ]));
+        let batch = arrow::record_batch::RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(StringArray::from(vec!["alice", "bob", "cory"])),
+            ],
+        )
+        .unwrap();
+        create_ipc_data_from_batches(schema, vec![batch])
+    }
+
+    fn create_vector_table_ipc_data() -> Vec<u8> {
+        use arrow::array::{FixedSizeListArray, Float32Array, Int32Array};
+        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
+                true,
+            ),
+        ]));
+        let vector_field = Arc::new(Field::new("item", DataType::Float32, true));
+        let vectors = FixedSizeListArray::try_new(
+            vector_field,
+            2,
+            Arc::new(Float32Array::from(vec![0.1, 0.2, 0.3, 0.4, 0.5, 0.6])),
+            None,
+        )
+        .unwrap();
+        let batch = arrow::record_batch::RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3])), Arc::new(vectors)],
+        )
+        .unwrap();
+        create_ipc_data_from_batches(schema, vec![batch])
+    }
+
+    async fn create_scalar_table(namespace: &DirectoryNamespace, table_name: &str) {
+        let mut create_table_request = CreateTableRequest::new();
+        create_table_request.id = Some(vec![table_name.to_string()]);
+        namespace
+            .create_table(
+                create_table_request,
+                Bytes::from(create_scalar_table_ipc_data()),
+            )
+            .await
+            .unwrap();
+    }
+
+    async fn create_vector_table(namespace: &DirectoryNamespace, table_name: &str) {
+        let mut create_table_request = CreateTableRequest::new();
+        create_table_request.id = Some(vec![table_name.to_string()]);
+        namespace
+            .create_table(
+                create_table_request,
+                Bytes::from(create_vector_table_ipc_data()),
+            )
+            .await
+            .unwrap();
+    }
+
+    async fn open_dataset(namespace: &DirectoryNamespace, table_name: &str) -> Dataset {
+        let mut describe_request = DescribeTableRequest::new();
+        describe_request.id = Some(vec![table_name.to_string()]);
+        let table_uri = namespace
+            .describe_table(describe_request)
+            .await
+            .unwrap()
+            .location
+            .expect("table location should exist");
+        Dataset::open(&table_uri).await.unwrap()
+    }
+
+    async fn create_scalar_index(
+        namespace: &DirectoryNamespace,
+        table_name: &str,
+        index_name: &str,
+    ) -> Option<String> {
+        use lance_namespace::models::CreateTableIndexRequest;
+
+        let mut create_index_request =
+            CreateTableIndexRequest::new("id".to_string(), "BTREE".to_string());
+        create_index_request.id = Some(vec![table_name.to_string()]);
+        create_index_request.name = Some(index_name.to_string());
+        namespace
+            .create_table_scalar_index(create_index_request)
+            .await
+            .unwrap()
+            .transaction_id
+    }
+
+    #[tokio::test]
+    async fn test_create_table() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Create test IPC data
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec!["test_table".to_string()]);
+
+        let response = namespace
+            .create_table(request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        assert!(response.location.is_some());
+        assert!(response.location.unwrap().ends_with("test_table.lance"));
+        assert_eq!(response.version, Some(1));
+    }
+
+    #[tokio::test]
+    async fn test_create_table_without_data() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec!["test_table".to_string()]);
+
+        let result = namespace.create_table(request, bytes::Bytes::new()).await;
+        assert!(result.is_err());
+        assert!(
+            result
+                .unwrap_err()
+                .to_string()
+                .contains("Arrow IPC stream) is required")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_create_table_with_invalid_id() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Create test IPC data
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        // Test with empty ID
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec![]);
+
+        let result = namespace
+            .create_table(request, bytes::Bytes::from(ipc_data.clone()))
+            .await;
+        assert!(result.is_err());
+
+        // Test with multi-level ID - should now work with manifest enabled
+        // First create the parent namespace
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["test_namespace".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
+
+        // Now create table in the namespace
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec!["test_namespace".to_string(), "table".to_string()]);
+
+        let result = namespace
+            .create_table(request, bytes::Bytes::from(ipc_data))
+            .await;
+        // Should succeed with manifest enabled
+        assert!(
+            result.is_ok(),
+            "Multi-level table IDs should work with manifest enabled"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_list_tables() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Initially, no tables
+        let mut request = ListTablesRequest::new();
+        request.id = Some(vec![]);
+        let response = namespace.list_tables(request).await.unwrap();
+        assert_eq!(response.tables.len(), 0);
+
+        // Create test IPC data
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        // Create a table
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["table1".to_string()]);
+        namespace
+            .create_table(create_request, bytes::Bytes::from(ipc_data.clone()))
+            .await
+            .unwrap();
+
+        // Create another table
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["table2".to_string()]);
+        namespace
+            .create_table(create_request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        // List tables should return both
+        let mut request = ListTablesRequest::new();
+        request.id = Some(vec![]);
+        let response = namespace.list_tables(request).await.unwrap();
+        let tables = response.tables;
+        assert_eq!(tables.len(), 2);
+        assert!(tables.contains(&"table1".to_string()));
+        assert!(tables.contains(&"table2".to_string()));
+    }
+
+    #[tokio::test]
+    async fn test_list_tables_pagination() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        for name in ["alpha", "bravo", "charlie"] {
+            let mut req = CreateTableRequest::new();
+            req.id = Some(vec![name.to_string()]);
+            namespace
+                .create_table(req, bytes::Bytes::from(ipc_data.clone()))
+                .await
+                .unwrap();
+        }
+
+        // First page: limit=2, no page_token
+        let first_page = namespace
+            .list_tables(ListTablesRequest {
+                id: Some(vec![]),
+                limit: Some(2),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(first_page.tables, vec!["alpha", "bravo"]);
+        assert_eq!(first_page.page_token.as_deref(), Some("bravo"));
+
+        // Second page: use page_token from first response
+        let second_page = namespace
+            .list_tables(ListTablesRequest {
+                id: Some(vec![]),
+                limit: Some(2),
+                page_token: first_page.page_token.clone(),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(second_page.tables, vec!["charlie"]);
+        assert!(second_page.page_token.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_list_tables_pagination_limit_zero() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        let mut req = CreateTableRequest::new();
+        req.id = Some(vec!["alpha".to_string()]);
+        namespace
+            .create_table(req, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        let response = namespace
+            .list_tables(ListTablesRequest {
+                id: Some(vec![]),
+                limit: Some(0),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert!(response.tables.is_empty());
+        assert!(response.page_token.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_list_tables_with_namespace_id() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // First create a child namespace
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["test_namespace".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
+
+        // Now list tables in the child namespace
+        let mut request = ListTablesRequest::new();
+        request.id = Some(vec!["test_namespace".to_string()]);
+
+        let result = namespace.list_tables(request).await;
+        // Should succeed (with manifest enabled) and return empty list (no tables yet)
+        assert!(
+            result.is_ok(),
+            "list_tables should work with child namespace when manifest is enabled"
+        );
+        let response = result.unwrap();
+        assert_eq!(
+            response.tables.len(),
+            0,
+            "Namespace should have no tables yet"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_create_scalar_index() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+
+        let transaction_id = create_scalar_index(&namespace, "users", "users_id_idx").await;
+        let dataset = open_dataset(&namespace, "users").await;
+        let expected_transaction_id = dataset
+            .read_transaction()
+            .await
+            .unwrap()
+            .map(|transaction| transaction.uuid);
+        assert_eq!(transaction_id, expected_transaction_id);
+        let indices = dataset.load_indices().await.unwrap();
+        assert!(indices.iter().any(|index| index.name == "users_id_idx"));
+    }
+
+    #[tokio::test]
+    async fn test_create_vector_index() {
+        use lance_namespace::models::CreateTableIndexRequest;
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_vector_table(&namespace, "vectors").await;
+
+        let mut create_index_request =
+            CreateTableIndexRequest::new("vector".to_string(), "IVF_FLAT".to_string());
+        create_index_request.id = Some(vec!["vectors".to_string()]);
+        create_index_request.name = Some("vector_idx".to_string());
+        create_index_request.distance_type = Some("l2".to_string());
+        let transaction_id = namespace
+            .create_table_index(create_index_request)
+            .await
+            .unwrap()
+            .transaction_id;
+
+        let dataset = open_dataset(&namespace, "vectors").await;
+        let expected_transaction_id = dataset
+            .read_transaction()
+            .await
+            .unwrap()
+            .map(|transaction| transaction.uuid);
+        assert_eq!(transaction_id, expected_transaction_id);
+        let indices = dataset.load_indices().await.unwrap();
+        assert!(indices.iter().any(|index| index.name == "vector_idx"));
+    }
+
+    #[tokio::test]
+    async fn test_list_table_indices() {
+        use lance_namespace::models::ListTableIndicesRequest;
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        create_scalar_index(&namespace, "users", "a_idx").await;
+        create_scalar_index(&namespace, "users", "b_idx").await;
+        let transaction_id = create_scalar_index(&namespace, "users", "users_id_idx").await;
+
+        let response = namespace
+            .list_table_indices(ListTableIndicesRequest {
+                id: Some(vec!["users".to_string()]),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(response.indexes.len(), 3);
+        assert_eq!(response.indexes[0].index_name, "a_idx");
+        assert_eq!(response.indexes[1].index_name, "b_idx");
+        assert_eq!(response.indexes[2].index_name, "users_id_idx");
+        assert!(response.page_token.is_none());
+        let users_id_idx = response
+            .indexes
+            .iter()
+            .find(|index| index.index_name == "users_id_idx")
+            .unwrap();
+        assert_eq!(users_id_idx.columns, vec!["id"]);
+        assert_eq!(users_id_idx.status, "SUCCEEDED");
+
+        let dataset = open_dataset(&namespace, "users").await;
+        let expected_transaction_id = dataset
+            .read_transaction()
+            .await
+            .unwrap()
+            .map(|transaction| transaction.uuid);
+        assert_eq!(transaction_id, expected_transaction_id);
+        let indices = dataset.load_indices().await.unwrap();
+        assert_eq!(
+            indices
+                .iter()
+                .filter(|index| index.name == "users_id_idx")
+                .count(),
+            1
+        );
+
+        let first_page = namespace
+            .list_table_indices(ListTableIndicesRequest {
+                id: Some(vec!["users".to_string()]),
+                limit: Some(2),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(first_page.indexes.len(), 2);
+        assert_eq!(first_page.indexes[0].index_name, "a_idx");
+        assert_eq!(first_page.indexes[1].index_name, "b_idx");
+        assert_eq!(first_page.page_token.as_deref(), Some("b_idx"));
+
+        let second_page = namespace
+            .list_table_indices(ListTableIndicesRequest {
+                id: Some(vec!["users".to_string()]),
+                page_token: first_page.page_token.clone(),
+                limit: Some(2),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(second_page.indexes.len(), 1);
+        assert_eq!(second_page.indexes[0].index_name, "users_id_idx");
+        assert!(second_page.page_token.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_describe_table_index_stats() {
+        use lance_namespace::models::DescribeTableIndexStatsRequest;
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        let transaction_id = create_scalar_index(&namespace, "users", "users_id_idx").await;
+
+        let response = namespace
+            .describe_table_index_stats(DescribeTableIndexStatsRequest {
+                id: Some(vec!["users".to_string()]),
+                index_name: Some("users_id_idx".to_string()),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(response.index_type, Some("BTree".to_string()));
+        assert_eq!(response.num_indices, Some(1));
+        assert_eq!(response.num_indexed_rows, Some(3));
+        assert_eq!(response.num_unindexed_rows, Some(0));
+
+        let dataset = open_dataset(&namespace, "users").await;
+        let expected_transaction_id = dataset
+            .read_transaction()
+            .await
+            .unwrap()
+            .map(|transaction| transaction.uuid);
+        assert_eq!(transaction_id, expected_transaction_id);
+        let stats: serde_json::Value =
+            serde_json::from_str(&dataset.index_statistics("users_id_idx").await.unwrap()).unwrap();
+        assert_eq!(stats["index_type"], "BTree");
+        assert_eq!(stats["num_indices"], 1);
+        assert_eq!(stats["num_indexed_rows"], 3);
+        assert_eq!(stats["num_unindexed_rows"], 0);
+    }
+
+    #[tokio::test]
+    async fn test_describe_transaction() {
+        use lance_namespace::models::DescribeTransactionRequest;
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        let transaction_id = create_scalar_index(&namespace, "users", "users_id_idx").await;
+        let dataset = open_dataset(&namespace, "users").await;
+        let latest_transaction = dataset.read_transaction().await.unwrap();
+        assert_eq!(
+            transaction_id,
+            latest_transaction
+                .as_ref()
+                .map(|transaction| transaction.uuid.clone())
+        );
+
+        if let Some(transaction_id) = transaction_id {
+            let response = namespace
+                .describe_transaction(DescribeTransactionRequest {
+                    id: Some(vec!["users".to_string(), transaction_id.clone()]),
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            assert_eq!(response.status, "SUCCEEDED");
+            assert_eq!(
+                response
+                    .properties
+                    .as_ref()
+                    .and_then(|props| props.get("operation")),
+                Some(&"CreateIndex".to_string())
+            );
+            assert_eq!(
+                response
+                    .properties
+                    .as_ref()
+                    .and_then(|props| props.get("uuid")),
+                Some(&transaction_id)
+            );
+        } else {
+            assert!(latest_transaction.is_none());
+        }
+    }
+
+    #[tokio::test]
+    async fn test_drop_table_index() {
+        use lance_namespace::models::{DropTableIndexRequest, ListTableIndicesRequest};
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        let create_transaction_id = create_scalar_index(&namespace, "users", "users_id_idx").await;
+
+        let drop_transaction_id = namespace
+            .drop_table_index(DropTableIndexRequest {
+                id: Some(vec!["users".to_string()]),
+                index_name: Some("users_id_idx".to_string()),
+                ..Default::default()
+            })
+            .await
+            .unwrap()
+            .transaction_id;
+
+        let dataset = open_dataset(&namespace, "users").await;
+        let previous_dataset = dataset
+            .checkout_version(dataset.version().version - 1)
+            .await
+            .unwrap();
+        let previous_transaction_id = previous_dataset
+            .read_transaction()
+            .await
+            .unwrap()
+            .map(|transaction| transaction.uuid);
+        assert_eq!(create_transaction_id, previous_transaction_id);
+        let expected_drop_transaction_id = dataset
+            .read_transaction()
+            .await
+            .unwrap()
+            .map(|transaction| transaction.uuid);
+        assert_eq!(drop_transaction_id, expected_drop_transaction_id);
+        let indices = dataset.load_indices().await.unwrap();
+        assert!(!indices.iter().any(|index| index.name == "users_id_idx"));
+
+        let list_response = namespace
+            .list_table_indices(ListTableIndicesRequest {
+                id: Some(vec!["users".to_string()]),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert!(list_response.indexes.is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_describe_table() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Create a table first
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["test_table".to_string()]);
+        namespace
+            .create_table(create_request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        // Describe the table
+        let mut request = DescribeTableRequest::new();
+        request.id = Some(vec!["test_table".to_string()]);
+        let response = namespace.describe_table(request).await.unwrap();
+
+        assert!(response.location.is_some());
+        assert!(response.location.unwrap().ends_with("test_table.lance"));
+    }
+
+    #[tokio::test]
+    async fn test_describe_nonexistent_table() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        let mut request = DescribeTableRequest::new();
+        request.id = Some(vec!["nonexistent".to_string()]);
+
+        let result = namespace.describe_table(request).await;
+        assert!(result.is_err());
+        assert!(result.unwrap_err().to_string().contains("Table not found"));
+    }
+
+    #[tokio::test]
+    async fn test_table_exists() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Create a table
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["existing_table".to_string()]);
+        namespace
+            .create_table(create_request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        // Check existing table
+        let mut request = TableExistsRequest::new();
+        request.id = Some(vec!["existing_table".to_string()]);
+        let result = namespace.table_exists(request).await;
+        assert!(result.is_ok());
+
+        // Check non-existent table
+        let mut request = TableExistsRequest::new();
+        request.id = Some(vec!["nonexistent".to_string()]);
+        let result = namespace.table_exists(request).await;
+        assert!(result.is_err());
+        assert!(result.unwrap_err().to_string().contains("Table not found"));
+    }
+
+    #[tokio::test]
+    async fn test_drop_table() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Create a table
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["table_to_drop".to_string()]);
+        namespace
+            .create_table(create_request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        // Verify it exists
+        let mut exists_request = TableExistsRequest::new();
+        exists_request.id = Some(vec!["table_to_drop".to_string()]);
+        assert!(namespace.table_exists(exists_request.clone()).await.is_ok());
+
+        // Drop the table
+        let mut drop_request = DropTableRequest::new();
+        drop_request.id = Some(vec!["table_to_drop".to_string()]);
+        let response = namespace.drop_table(drop_request).await.unwrap();
+        assert!(response.location.is_some());
+
+        // Verify it no longer exists
+        assert!(namespace.table_exists(exists_request).await.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_drop_nonexistent_table() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        let mut request = DropTableRequest::new();
+        request.id = Some(vec!["nonexistent".to_string()]);
+
+        // Should not fail when dropping non-existent table (idempotent)
+        let result = namespace.drop_table(request).await;
+        // The operation might succeed or fail depending on implementation
+        // But it should not panic
+        let _ = result;
+    }
+
+    #[tokio::test]
+    async fn test_root_namespace_operations() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Test list_namespaces - should return empty list for root
+        let mut request = ListNamespacesRequest::new();
+        request.id = Some(vec![]);
+        let result = namespace.list_namespaces(request).await;
+        assert!(result.is_ok());
+        assert_eq!(result.unwrap().namespaces.len(), 0);
+
+        // Test describe_namespace - should succeed for root
+        let mut request = DescribeNamespaceRequest::new();
+        request.id = Some(vec![]);
+        let result = namespace.describe_namespace(request).await;
+        assert!(result.is_ok());
+
+        // Test namespace_exists - root always exists
+        let mut request = NamespaceExistsRequest::new();
+        request.id = Some(vec![]);
+        let result = namespace.namespace_exists(request).await;
+        assert!(result.is_ok());
+
+        // Test create_namespace - root cannot be created
+        let mut request = CreateNamespaceRequest::new();
+        request.id = Some(vec![]);
+        let result = namespace.create_namespace(request).await;
+        assert!(result.is_err());
+        assert!(result.unwrap_err().to_string().contains("already exists"));
+
+        // Test drop_namespace - root cannot be dropped
+        let mut request = DropNamespaceRequest::new();
+        request.id = Some(vec![]);
+        let result = namespace.drop_namespace(request).await;
+        assert!(result.is_err());
+        assert!(
+            result
+                .unwrap_err()
+                .to_string()
+                .contains("cannot be dropped")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_non_root_namespace_operations() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // With manifest enabled (default), child namespaces are now supported
+        // Test create_namespace for non-root - should succeed with manifest
+        let mut request = CreateNamespaceRequest::new();
+        request.id = Some(vec!["child".to_string()]);
+        let result = namespace.create_namespace(request).await;
+        assert!(
+            result.is_ok(),
+            "Child namespace creation should succeed with manifest enabled"
+        );
+
+        // Test namespace_exists for non-root - should exist after creation
+        let mut request = NamespaceExistsRequest::new();
+        request.id = Some(vec!["child".to_string()]);
+        let result = namespace.namespace_exists(request).await;
+        assert!(
+            result.is_ok(),
+            "Child namespace should exist after creation"
+        );
+
+        // Test drop_namespace for non-root - should succeed
+        let mut request = DropNamespaceRequest::new();
+        request.id = Some(vec!["child".to_string()]);
+        let result = namespace.drop_namespace(request).await;
+        assert!(
+            result.is_ok(),
+            "Child namespace drop should succeed with manifest enabled"
+        );
+
+        // Verify namespace no longer exists
+        let mut request = NamespaceExistsRequest::new();
+        request.id = Some(vec!["child".to_string()]);
+        let result = namespace.namespace_exists(request).await;
+        assert!(
+            result.is_err(),
+            "Child namespace should not exist after drop"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_config_custom_root() {
+        let temp_dir = TempStdDir::default();
+        let custom_path = temp_dir.join("custom");
+        std::fs::create_dir(&custom_path).unwrap();
+
+        let namespace = DirectoryNamespaceBuilder::new(custom_path.to_string_lossy().to_string())
+            .build()
+            .await
+            .unwrap();
+
+        // Create test IPC data
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        // Create a table and verify location
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec!["test_table".to_string()]);
+
+        let response = namespace
+            .create_table(request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        assert!(response.location.unwrap().contains("custom"));
+    }
+
+    #[tokio::test]
+    async fn test_config_storage_options() {
+        let temp_dir = TempStdDir::default();
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_dir.to_str().unwrap())
+            .storage_option("option1", "value1")
+            .storage_option("option2", "value2")
+            .build()
+            .await
+            .unwrap();
+
+        // Create test IPC data
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        // Create a table and check storage options are included
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec!["test_table".to_string()]);
+
+        let response = namespace
+            .create_table(request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        let storage_options = response.storage_options.unwrap();
+        assert_eq!(storage_options.get("option1"), Some(&"value1".to_string()));
+        assert_eq!(storage_options.get("option2"), Some(&"value2".to_string()));
+    }
+
+    /// When no credential vendor is configured, `describe_table` and
+    /// `declare_table` must strip credential keys from storage options
+    /// while preserving non-credential config (region, endpoint, etc.).
+    #[tokio::test]
+    async fn test_no_storage_options_without_vendor() {
+        use lance_namespace::models::DeclareTableRequest;
+
+        let temp_dir = TempStdDir::default();
+
+        // No manifest, no credential vendor, but storage options with credentials
+        let namespace = DirectoryNamespaceBuilder::new(temp_dir.to_str().unwrap())
+            .manifest_enabled(false)
+            .storage_option("aws_access_key_id", "AKID")
+            .storage_option("aws_secret_access_key", "SECRET")
+            .storage_option("region", "us-east-1")
+            .build()
+            .await
+            .unwrap();
+
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        // create_table
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["t1".to_string()]);
+        namespace
+            .create_table(create_req, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        // describe_table should not return storage options without a vendor
+        let mut desc_req = DescribeTableRequest::new();
+        desc_req.id = Some(vec!["t1".to_string()]);
+        let resp = namespace.describe_table(desc_req).await.unwrap();
+        assert!(resp.storage_options.is_none());
+
+        // declare_table should not return storage options without a vendor
+        let mut decl_req = DeclareTableRequest::new();
+        decl_req.id = Some(vec!["t2".to_string()]);
+        let resp = namespace.declare_table(decl_req).await.unwrap();
+        assert!(resp.storage_options.is_none());
+    }
+
+    /// Same test with manifest mode enabled.
+    #[tokio::test]
+    async fn test_no_storage_options_without_vendor_manifest() {
+        let temp_dir = TempStdDir::default();
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_dir.to_str().unwrap())
+            .storage_option("aws_access_key_id", "AKID")
+            .storage_option("aws_secret_access_key", "SECRET")
+            .storage_option("region", "us-east-1")
+            .build()
+            .await
+            .unwrap();
+
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["t1".to_string()]);
+        namespace
+            .create_table(create_req, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        // describe_table through manifest should not return storage options without a vendor
+        let mut desc_req = DescribeTableRequest::new();
+        desc_req.id = Some(vec!["t1".to_string()]);
+        let resp = namespace.describe_table(desc_req).await.unwrap();
+        assert!(resp.storage_options.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_from_properties_manifest_enabled() {
+        let temp_dir = TempStdDir::default();
+
+        let mut properties = HashMap::new();
+        properties.insert("root".to_string(), temp_dir.to_str().unwrap().to_string());
+        properties.insert("manifest_enabled".to_string(), "true".to_string());
+        properties.insert("dir_listing_enabled".to_string(), "false".to_string());
+
+        let builder = DirectoryNamespaceBuilder::from_properties(properties, None).unwrap();
+        assert!(builder.manifest_enabled);
+        assert!(!builder.dir_listing_enabled);
+
+        let namespace = builder.build().await.unwrap();
+
+        // Create test IPC data
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        // Create a table
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec!["test_table".to_string()]);
+
+        let response = namespace
+            .create_table(request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        assert!(response.location.is_some());
+    }
+
+    #[tokio::test]
+    async fn test_from_properties_dir_listing_enabled() {
+        let temp_dir = TempStdDir::default();
+
+        let mut properties = HashMap::new();
+        properties.insert("root".to_string(), temp_dir.to_str().unwrap().to_string());
+        properties.insert("manifest_enabled".to_string(), "false".to_string());
+        properties.insert("dir_listing_enabled".to_string(), "true".to_string());
+
+        let builder = DirectoryNamespaceBuilder::from_properties(properties, None).unwrap();
+        assert!(!builder.manifest_enabled);
+        assert!(builder.dir_listing_enabled);
+
+        let namespace = builder.build().await.unwrap();
+
+        // Create test IPC data
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        // Create a table
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec!["test_table".to_string()]);
+
+        let response = namespace
+            .create_table(request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        assert!(response.location.is_some());
+    }
+
+    #[tokio::test]
+    async fn test_from_properties_defaults() {
+        let temp_dir = TempStdDir::default();
+
+        let mut properties = HashMap::new();
+        properties.insert("root".to_string(), temp_dir.to_str().unwrap().to_string());
+
+        let builder = DirectoryNamespaceBuilder::from_properties(properties, None).unwrap();
+        // Both should default to true
+        assert!(builder.manifest_enabled);
+        assert!(builder.dir_listing_enabled);
+    }
+
+    #[tokio::test]
+    async fn test_from_properties_with_storage_options() {
+        let temp_dir = TempStdDir::default();
+
+        let mut properties = HashMap::new();
+        properties.insert("root".to_string(), temp_dir.to_str().unwrap().to_string());
+        properties.insert("manifest_enabled".to_string(), "true".to_string());
+        properties.insert("storage.region".to_string(), "us-west-2".to_string());
+        properties.insert("storage.bucket".to_string(), "my-bucket".to_string());
+
+        let builder = DirectoryNamespaceBuilder::from_properties(properties, None).unwrap();
+        assert!(builder.manifest_enabled);
+        assert!(builder.storage_options.is_some());
+
+        let storage_options = builder.storage_options.unwrap();
+        assert_eq!(
+            storage_options.get("region"),
+            Some(&"us-west-2".to_string())
+        );
+        assert_eq!(
+            storage_options.get("bucket"),
+            Some(&"my-bucket".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn test_various_arrow_types() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Create schema with various types
+        let fields = vec![
+            JsonArrowField {
+                name: "bool_col".to_string(),
+                r#type: Box::new(JsonArrowDataType::new("bool".to_string())),
+                nullable: true,
+                metadata: None,
+            },
+            JsonArrowField {
+                name: "int8_col".to_string(),
+                r#type: Box::new(JsonArrowDataType::new("int8".to_string())),
+                nullable: true,
+                metadata: None,
+            },
+            JsonArrowField {
+                name: "float64_col".to_string(),
+                r#type: Box::new(JsonArrowDataType::new("float64".to_string())),
+                nullable: true,
+                metadata: None,
+            },
+            JsonArrowField {
+                name: "binary_col".to_string(),
+                r#type: Box::new(JsonArrowDataType::new("binary".to_string())),
+                nullable: true,
+                metadata: None,
+            },
+        ];
+
+        let schema = JsonArrowSchema {
+            fields,
+            metadata: None,
+        };
+
+        // Create IPC data
+        let ipc_data = create_test_ipc_data(&schema);
+
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec!["complex_table".to_string()]);
+
+        let response = namespace
+            .create_table(request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        assert!(response.location.is_some());
+    }
+
+    #[tokio::test]
+    async fn test_connect_dir() {
+        let temp_dir = TempStdDir::default();
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_dir.to_str().unwrap())
+            .build()
+            .await
+            .unwrap();
 
-        let mut table_versions: Vec<TableVersion> = manifest_metas
-            .into_iter()
-            .filter_map(|meta| {
-                let filename = meta.location.filename()?;
-                let version_str = filename.strip_suffix(".manifest")?;
-                if version_str.starts_with('d') {
-                    return None;
-                }
-                let file_version: u64 = version_str.parse().ok()?;
+        // Test basic operation through the concrete type
+        let mut request = ListTablesRequest::new();
+        request.id = Some(vec![]);
+        let response = namespace.list_tables(request).await.unwrap();
+        assert_eq!(response.tables.len(), 0);
+    }
 
-                let actual_version = if file_version > u64::MAX / 2 {
-                    u64::MAX - file_version
-                } else {
-                    file_version
-                };
+    #[tokio::test]
+    async fn test_create_table_with_ipc_data() {
+        use arrow::array::{Int32Array, StringArray};
+        use arrow::ipc::writer::StreamWriter;
 
-                // Use full path from object_store (relative to object store root)
-                Some(TableVersion {
-                    version: actual_version as i64,
-                    manifest_path: meta.location.to_string(),
-                    manifest_size: Some(meta.size as i64),
-                    e_tag: meta.e_tag,
-                    timestamp_millis: Some(meta.last_modified.timestamp_millis()),
-                    metadata: None,
-                })
-            })
-            .collect();
+        let (namespace, _temp_dir) = create_test_namespace().await;
 
-        let list_is_ordered = self.object_store.list_is_lexically_ordered;
-        let want_descending = request.descending == Some(true);
+        // Create a schema with some fields
+        let schema = create_test_schema();
 
-        let needs_sort = if list_is_ordered {
-            if is_v2_naming {
-                !want_descending
-            } else {
-                want_descending
-            }
-        } else {
-            true
-        };
+        // Create some test data that matches the schema
+        let arrow_schema = convert_json_arrow_schema(&schema).unwrap();
+        let arrow_schema = Arc::new(arrow_schema);
 
-        if needs_sort {
-            if want_descending {
-                table_versions.sort_by(|a, b| b.version.cmp(&a.version));
-            } else {
-                table_versions.sort_by(|a, b| a.version.cmp(&b.version));
-            }
+        // Create a RecordBatch with actual data
+        let id_array = Int32Array::from(vec![1, 2, 3]);
+        let name_array = StringArray::from(vec!["Alice", "Bob", "Charlie"]);
+        let batch = arrow::record_batch::RecordBatch::try_new(
+            arrow_schema.clone(),
+            vec![Arc::new(id_array), Arc::new(name_array)],
+        )
+        .unwrap();
+
+        // Write the batch to an IPC stream
+        let mut buffer = Vec::new();
+        {
+            let mut writer = StreamWriter::try_new(&mut buffer, &arrow_schema).unwrap();
+            writer.write(&batch).unwrap();
+            writer.finish().unwrap();
         }
 
-        if let Some(limit) = request.limit {
-            table_versions.truncate(limit as usize);
+        // Create table with the IPC data
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec!["test_table_with_data".to_string()]);
+
+        let response = namespace
+            .create_table(request, Bytes::from(buffer))
+            .await
+            .unwrap();
+
+        assert_eq!(response.version, Some(1));
+        assert!(
+            response
+                .location
+                .unwrap()
+                .contains("test_table_with_data.lance")
+        );
+
+        // Verify table exists
+        let mut exists_request = TableExistsRequest::new();
+        exists_request.id = Some(vec!["test_table_with_data".to_string()]);
+        namespace.table_exists(exists_request).await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_child_namespace_create_and_list() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Create multiple child namespaces
+        for i in 1..=3 {
+            let mut create_req = CreateNamespaceRequest::new();
+            create_req.id = Some(vec![format!("ns{}", i)]);
+            let result = namespace.create_namespace(create_req).await;
+            assert!(result.is_ok(), "Failed to create child namespace ns{}", i);
         }
 
-        Ok(ListTableVersionsResponse {
-            versions: table_versions,
-            page_token: None,
-        })
+        // List child namespaces
+        let list_req = ListNamespacesRequest {
+            id: Some(vec![]),
+            ..Default::default()
+        };
+        let result = namespace.list_namespaces(list_req).await;
+        assert!(result.is_ok());
+        let namespaces = result.unwrap().namespaces;
+        assert_eq!(namespaces.len(), 3);
+        assert!(namespaces.contains(&"ns1".to_string()));
+        assert!(namespaces.contains(&"ns2".to_string()));
+        assert!(namespaces.contains(&"ns3".to_string()));
     }
 
-    async fn create_table_version(
-        &self,
-        request: CreateTableVersionRequest,
-    ) -> Result<CreateTableVersionResponse> {
-        let table_uri = self.resolve_table_location(&request.id).await?;
+    #[tokio::test]
+    async fn test_nested_namespace_hierarchy() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
 
-        let staging_manifest_path = &request.manifest_path;
-        let version = request.version as u64;
+        // Create parent namespace
+        let mut create_req = CreateNamespaceRequest::new();
+        create_req.id = Some(vec!["parent".to_string()]);
+        namespace.create_namespace(create_req).await.unwrap();
 
-        let table_path = Self::uri_to_object_store_path(&table_uri);
+        // Create nested children
+        let mut create_req = CreateNamespaceRequest::new();
+        create_req.id = Some(vec!["parent".to_string(), "child1".to_string()]);
+        namespace.create_namespace(create_req).await.unwrap();
 
-        // Determine naming scheme from request, default to V2
-        let naming_scheme = match request.naming_scheme.as_deref() {
-            Some("V1") => ManifestNamingScheme::V1,
-            _ => ManifestNamingScheme::V2,
+        let mut create_req = CreateNamespaceRequest::new();
+        create_req.id = Some(vec!["parent".to_string(), "child2".to_string()]);
+        namespace.create_namespace(create_req).await.unwrap();
+
+        // List children of parent
+        let list_req = ListNamespacesRequest {
+            id: Some(vec!["parent".to_string()]),
+            ..Default::default()
         };
+        let result = namespace.list_namespaces(list_req).await;
+        assert!(result.is_ok());
+        let children = result.unwrap().namespaces;
+        assert_eq!(children.len(), 2);
+        assert!(children.contains(&"child1".to_string()));
+        assert!(children.contains(&"child2".to_string()));
 
-        // Compute final path using the naming scheme
-        let final_path = naming_scheme.manifest_path(&table_path, version);
+        // List root should only show parent
+        let list_req = ListNamespacesRequest {
+            id: Some(vec![]),
+            ..Default::default()
+        };
+        let result = namespace.list_namespaces(list_req).await;
+        assert!(result.is_ok());
+        let root_namespaces = result.unwrap().namespaces;
+        assert_eq!(root_namespaces.len(), 1);
+        assert_eq!(root_namespaces[0], "parent");
+    }
 
-        let staging_path = Self::uri_to_object_store_path(staging_manifest_path);
-        let manifest_data = self
-            .object_store
-            .inner
-            .get(&staging_path)
-            .await
-            .map_err(|e| {
-                Error::namespace_source(
-                    format!(
-                        "Failed to read staging manifest at '{}': {}",
-                        staging_manifest_path, e
-                    )
-                    .into(),
-                )
-            })?
-            .bytes()
-            .await
-            .map_err(|e| {
-                Error::namespace_source(
-                    format!(
-                        "Failed to read staging manifest bytes at '{}': {}",
-                        staging_manifest_path, e
-                    )
-                    .into(),
-                )
-            })?;
+    #[tokio::test]
+    async fn test_table_in_child_namespace() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
 
-        let manifest_size = manifest_data.len() as i64;
+        // Create child namespace
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["test_ns".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
 
-        let put_result = self
-            .object_store
-            .inner
-            .put_opts(
-                &final_path,
-                manifest_data.into(),
-                PutOptions {
-                    mode: PutMode::Create,
-                    ..Default::default()
-                },
-            )
-            .await
-            .map_err(|e| match e {
-                object_store::Error::AlreadyExists { .. }
-                | object_store::Error::Precondition { .. } => Error::namespace_source(
-                    format!(
-                        "Version {} already exists for table at '{}'",
-                        version, table_uri
-                    )
-                    .into(),
-                ),
-                _ => Error::namespace_source(
-                    format!(
-                        "Failed to create version {} for table at '{}': {}",
-                        version, table_uri, e
-                    )
-                    .into(),
-                ),
-            })?;
+        // Create table in child namespace
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+        let mut create_table_req = CreateTableRequest::new();
+        create_table_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
+        let result = namespace
+            .create_table(create_table_req, bytes::Bytes::from(ipc_data))
+            .await;
+        assert!(result.is_ok(), "Failed to create table in child namespace");
 
-        // Delete the staging manifest after successful copy
-        if let Err(e) = self.object_store.inner.delete(&staging_path).await {
-            log::warn!(
-                "Failed to delete staging manifest at '{}': {:?}",
-                staging_path,
-                e
-            );
-        }
+        // List tables in child namespace
+        let list_req = ListTablesRequest {
+            id: Some(vec!["test_ns".to_string()]),
+            ..Default::default()
+        };
+        let result = namespace.list_tables(list_req).await;
+        assert!(result.is_ok());
+        let tables = result.unwrap().tables;
+        assert_eq!(tables.len(), 1);
+        assert_eq!(tables[0], "table1");
 
-        Ok(CreateTableVersionResponse {
-            transaction_id: None,
-            version: Some(Box::new(TableVersion {
-                version: version as i64,
-                manifest_path: final_path.to_string(),
-                manifest_size: Some(manifest_size),
-                e_tag: put_result.e_tag,
-                timestamp_millis: None,
-                metadata: None,
-            })),
-        })
+        // Verify table exists
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
+        let result = namespace.table_exists(exists_req).await;
+        assert!(result.is_ok());
+
+        // Describe table in child namespace
+        let mut describe_req = DescribeTableRequest::new();
+        describe_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
+        let result = namespace.describe_table(describe_req).await;
+        assert!(result.is_ok());
+        let response = result.unwrap();
+        assert!(response.location.is_some());
     }
 
-    async fn describe_table_version(
-        &self,
-        request: DescribeTableVersionRequest,
-    ) -> Result<DescribeTableVersionResponse> {
-        let table_uri = self.resolve_table_location(&request.id).await?;
+    #[tokio::test]
+    async fn test_multiple_tables_in_child_namespace() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
 
-        // Use DatasetBuilder with storage options to support S3 with custom endpoints
-        let mut builder = DatasetBuilder::from_uri(&table_uri);
-        if let Some(opts) = &self.storage_options {
-            builder = builder.with_storage_options(opts.clone());
-        }
-        if let Some(sess) = &self.session {
-            builder = builder.with_session(sess.clone());
-        }
-        let mut dataset = builder.load().await.map_err(|e| {
-            Error::namespace_source(
-                format!("Failed to open table at '{}': {}", table_uri, e).into(),
-            )
-        })?;
+        // Create child namespace
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["test_ns".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
 
-        if let Some(version) = request.version {
-            dataset = dataset
-                .checkout_version(version as u64)
+        // Create multiple tables
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+        for i in 1..=3 {
+            let mut create_table_req = CreateTableRequest::new();
+            create_table_req.id = Some(vec!["test_ns".to_string(), format!("table{}", i)]);
+            namespace
+                .create_table(create_table_req, bytes::Bytes::from(ipc_data.clone()))
                 .await
-                .map_err(|e| {
-                    Error::namespace_source(
-                        format!(
-                            "Failed to checkout version {} for table at '{}': {}",
-                            version, table_uri, e
-                        )
-                        .into(),
-                    )
-                })?;
+                .unwrap();
         }
 
-        let version_info = dataset.version();
-        let manifest_location = dataset.manifest_location();
-        let metadata: std::collections::HashMap<String, String> =
-            version_info.metadata.into_iter().collect();
-
-        let table_version = TableVersion {
-            version: version_info.version as i64,
-            manifest_path: manifest_location.path.to_string(),
-            manifest_size: manifest_location.size.map(|s| s as i64),
-            e_tag: manifest_location.e_tag.clone(),
-            timestamp_millis: Some(version_info.timestamp.timestamp_millis()),
-            metadata: if metadata.is_empty() {
-                None
-            } else {
-                Some(metadata)
-            },
+        // List tables
+        let list_req = ListTablesRequest {
+            id: Some(vec!["test_ns".to_string()]),
+            ..Default::default()
         };
-
-        Ok(DescribeTableVersionResponse {
-            version: Box::new(table_version),
-        })
+        let result = namespace.list_tables(list_req).await;
+        assert!(result.is_ok());
+        let tables = result.unwrap().tables;
+        assert_eq!(tables.len(), 3);
+        assert!(tables.contains(&"table1".to_string()));
+        assert!(tables.contains(&"table2".to_string()));
+        assert!(tables.contains(&"table3".to_string()));
     }
 
-    async fn batch_delete_table_versions(
-        &self,
-        request: BatchDeleteTableVersionsRequest,
-    ) -> Result<BatchDeleteTableVersionsResponse> {
-        let table_uri = self.resolve_table_location(&request.id).await?;
-
-        let table_path = Self::uri_to_object_store_path(&table_uri);
-        let table_path_str = table_path.as_ref();
-        let versions_dir_path = Path::from(format!("{}_versions", table_path_str));
+    #[tokio::test]
+    async fn test_drop_table_in_child_namespace() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
 
-        let mut deleted_count = 0i64;
+        // Create child namespace
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["test_ns".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
 
-        for range in &request.ranges {
-            let start = range.start_version as u64;
-            let end = if range.end_version > 0 {
-                range.end_version as u64
-            } else {
-                start
-            };
+        // Create table
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+        let mut create_table_req = CreateTableRequest::new();
+        create_table_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
+        namespace
+            .create_table(create_table_req, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
 
-            for version in start..=end {
-                let version_path = versions_dir_path.child(format!("{}.manifest", version));
-                match self.object_store.inner.delete(&version_path).await {
-                    Ok(_) => {
-                        deleted_count += 1;
-                    }
-                    Err(object_store::Error::NotFound { .. }) => {}
-                    Err(e) => {
-                        return Err(Error::namespace_source(
-                            format!(
-                                "Failed to delete version {} for table at '{}': {}",
-                                version, table_uri, e
-                            )
-                            .into(),
-                        ));
-                    }
-                }
-            }
-        }
+        // Drop table
+        let mut drop_req = DropTableRequest::new();
+        drop_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
+        let result = namespace.drop_table(drop_req).await;
+        assert!(result.is_ok(), "Failed to drop table in child namespace");
 
-        Ok(BatchDeleteTableVersionsResponse {
-            deleted_count: Some(deleted_count),
-            transaction_id: None,
-        })
+        // Verify table no longer exists
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
+        let result = namespace.table_exists(exists_req).await;
+        assert!(result.is_err());
     }
 
-    fn namespace_id(&self) -> String {
-        format!("DirectoryNamespace {{ root: {:?} }}", self.root)
-    }
-}
+    #[tokio::test]
+    async fn test_deeply_nested_namespace() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
 
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use arrow_ipc::reader::StreamReader;
-    use lance::dataset::Dataset;
-    use lance_core::utils::tempfile::{TempStdDir, TempStrDir};
-    use lance_namespace::models::{
-        CreateTableRequest, JsonArrowDataType, JsonArrowField, JsonArrowSchema, ListTablesRequest,
-    };
-    use lance_namespace::schema::convert_json_arrow_schema;
-    use std::io::Cursor;
-    use std::sync::Arc;
+        // Create deeply nested namespace hierarchy
+        let mut create_req = CreateNamespaceRequest::new();
+        create_req.id = Some(vec!["level1".to_string()]);
+        namespace.create_namespace(create_req).await.unwrap();
 
-    /// Helper to create a test DirectoryNamespace with a temporary directory
-    async fn create_test_namespace() -> (DirectoryNamespace, TempStdDir) {
-        let temp_dir = TempStdDir::default();
+        let mut create_req = CreateNamespaceRequest::new();
+        create_req.id = Some(vec!["level1".to_string(), "level2".to_string()]);
+        namespace.create_namespace(create_req).await.unwrap();
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_dir.to_str().unwrap())
-            .build()
-            .await
-            .unwrap();
-        (namespace, temp_dir)
-    }
+        let mut create_req = CreateNamespaceRequest::new();
+        create_req.id = Some(vec![
+            "level1".to_string(),
+            "level2".to_string(),
+            "level3".to_string(),
+        ]);
+        namespace.create_namespace(create_req).await.unwrap();
 
-    /// Helper to create test IPC data from a schema
-    fn create_test_ipc_data(schema: &JsonArrowSchema) -> Vec<u8> {
-        use arrow::ipc::writer::StreamWriter;
+        // Create table in deeply nested namespace
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+        let mut create_table_req = CreateTableRequest::new();
+        create_table_req.id = Some(vec![
+            "level1".to_string(),
+            "level2".to_string(),
+            "level3".to_string(),
+            "table1".to_string(),
+        ]);
+        let result = namespace
+            .create_table(create_table_req, bytes::Bytes::from(ipc_data))
+            .await;
+        assert!(
+            result.is_ok(),
+            "Failed to create table in deeply nested namespace"
+        );
 
-        let arrow_schema = convert_json_arrow_schema(schema).unwrap();
-        let arrow_schema = Arc::new(arrow_schema);
-        let batch = arrow::record_batch::RecordBatch::new_empty(arrow_schema.clone());
-        let mut buffer = Vec::new();
-        {
-            let mut writer = StreamWriter::try_new(&mut buffer, &arrow_schema).unwrap();
-            writer.write(&batch).unwrap();
-            writer.finish().unwrap();
-        }
-        buffer
+        // Verify table exists
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec![
+            "level1".to_string(),
+            "level2".to_string(),
+            "level3".to_string(),
+            "table1".to_string(),
+        ]);
+        let result = namespace.table_exists(exists_req).await;
+        assert!(result.is_ok());
     }
 
-    /// Helper to create a simple test schema
-    fn create_test_schema() -> JsonArrowSchema {
-        let int_type = JsonArrowDataType::new("int32".to_string());
-        let string_type = JsonArrowDataType::new("utf8".to_string());
+    #[tokio::test]
+    async fn test_namespace_with_properties() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
 
-        let id_field = JsonArrowField {
-            name: "id".to_string(),
-            r#type: Box::new(int_type),
-            nullable: false,
-            metadata: None,
-        };
+        // Create namespace with properties
+        let mut properties = HashMap::new();
+        properties.insert("owner".to_string(), "test_user".to_string());
+        properties.insert("description".to_string(), "Test namespace".to_string());
 
-        let name_field = JsonArrowField {
-            name: "name".to_string(),
-            r#type: Box::new(string_type),
-            nullable: true,
-            metadata: None,
+        let mut create_req = CreateNamespaceRequest::new();
+        create_req.id = Some(vec!["test_ns".to_string()]);
+        create_req.properties = Some(properties.clone());
+        namespace.create_namespace(create_req).await.unwrap();
+
+        // Describe namespace and verify properties
+        let describe_req = DescribeNamespaceRequest {
+            id: Some(vec!["test_ns".to_string()]),
+            ..Default::default()
         };
-
-        JsonArrowSchema {
-            fields: vec![id_field, name_field],
-            metadata: None,
-        }
+        let result = namespace.describe_namespace(describe_req).await;
+        assert!(result.is_ok());
+        let response = result.unwrap();
+        assert!(response.properties.is_some());
+        let props = response.properties.unwrap();
+        assert_eq!(props.get("owner"), Some(&"test_user".to_string()));
+        assert_eq!(
+            props.get("description"),
+            Some(&"Test namespace".to_string())
+        );
     }
 
     #[tokio::test]
-    async fn test_create_table() {
+    async fn test_cannot_drop_namespace_with_tables() {
         let (namespace, _temp_dir) = create_test_namespace().await;
 
-        // Create test IPC data
+        // Create namespace
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["test_ns".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
+
+        // Create table in namespace
         let schema = create_test_schema();
         let ipc_data = create_test_ipc_data(&schema);
-
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec!["test_table".to_string()]);
-
-        let response = namespace
-            .create_table(request, bytes::Bytes::from(ipc_data))
+        let mut create_table_req = CreateTableRequest::new();
+        create_table_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
+        namespace
+            .create_table(create_table_req, bytes::Bytes::from(ipc_data))
             .await
             .unwrap();
 
-        assert!(response.location.is_some());
-        assert!(response.location.unwrap().ends_with("test_table.lance"));
-        assert_eq!(response.version, Some(1));
-    }
-
-    #[tokio::test]
-    async fn test_create_table_without_data() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
-
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec!["test_table".to_string()]);
-
-        let result = namespace.create_table(request, bytes::Bytes::new()).await;
-        assert!(result.is_err());
+        // Try to drop namespace - should fail
+        let mut drop_req = DropNamespaceRequest::new();
+        drop_req.id = Some(vec!["test_ns".to_string()]);
+        let result = namespace.drop_namespace(drop_req).await;
         assert!(
-            result
-                .unwrap_err()
-                .to_string()
-                .contains("Arrow IPC stream) is required")
+            result.is_err(),
+            "Should not be able to drop namespace with tables"
         );
     }
 
     #[tokio::test]
-    async fn test_create_table_with_invalid_id() {
+    async fn test_isolation_between_namespaces() {
         let (namespace, _temp_dir) = create_test_namespace().await;
 
-        // Create test IPC data
+        // Create two namespaces
+        let mut create_req = CreateNamespaceRequest::new();
+        create_req.id = Some(vec!["ns1".to_string()]);
+        namespace.create_namespace(create_req).await.unwrap();
+
+        let mut create_req = CreateNamespaceRequest::new();
+        create_req.id = Some(vec!["ns2".to_string()]);
+        namespace.create_namespace(create_req).await.unwrap();
+
+        // Create table with same name in both namespaces
         let schema = create_test_schema();
         let ipc_data = create_test_ipc_data(&schema);
 
-        // Test with empty ID
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec![]);
+        let mut create_table_req = CreateTableRequest::new();
+        create_table_req.id = Some(vec!["ns1".to_string(), "table1".to_string()]);
+        namespace
+            .create_table(create_table_req, bytes::Bytes::from(ipc_data.clone()))
+            .await
+            .unwrap();
 
-        let result = namespace
-            .create_table(request, bytes::Bytes::from(ipc_data.clone()))
-            .await;
-        assert!(result.is_err());
+        let mut create_table_req = CreateTableRequest::new();
+        create_table_req.id = Some(vec!["ns2".to_string(), "table1".to_string()]);
+        namespace
+            .create_table(create_table_req, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
 
-        // Test with multi-level ID - should now work with manifest enabled
-        // First create the parent namespace
-        let mut create_ns_req = CreateNamespaceRequest::new();
-        create_ns_req.id = Some(vec!["test_namespace".to_string()]);
-        namespace.create_namespace(create_ns_req).await.unwrap();
+        // List tables in each namespace
+        let list_req = ListTablesRequest {
+            id: Some(vec!["ns1".to_string()]),
+            page_token: None,
+            limit: None,
+            ..Default::default()
+        };
+        let result = namespace.list_tables(list_req).await.unwrap();
+        assert_eq!(result.tables.len(), 1);
+        assert_eq!(result.tables[0], "table1");
 
-        // Now create table in the namespace
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec!["test_namespace".to_string(), "table".to_string()]);
+        let list_req = ListTablesRequest {
+            id: Some(vec!["ns2".to_string()]),
+            page_token: None,
+            limit: None,
+            ..Default::default()
+        };
+        let result = namespace.list_tables(list_req).await.unwrap();
+        assert_eq!(result.tables.len(), 1);
+        assert_eq!(result.tables[0], "table1");
 
-        let result = namespace
-            .create_table(request, bytes::Bytes::from(ipc_data))
-            .await;
-        // Should succeed with manifest enabled
-        assert!(
-            result.is_ok(),
-            "Multi-level table IDs should work with manifest enabled"
-        );
+        // Drop table in ns1 shouldn't affect ns2
+        let mut drop_req = DropTableRequest::new();
+        drop_req.id = Some(vec!["ns1".to_string(), "table1".to_string()]);
+        namespace.drop_table(drop_req).await.unwrap();
+
+        // Verify ns1 table is gone but ns2 table still exists
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["ns1".to_string(), "table1".to_string()]);
+        assert!(namespace.table_exists(exists_req).await.is_err());
+
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["ns2".to_string(), "table1".to_string()]);
+        assert!(namespace.table_exists(exists_req).await.is_ok());
     }
 
     #[tokio::test]
-    async fn test_list_tables() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+    async fn test_migrate_directory_tables() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        // Initially, no tables
-        let mut request = ListTablesRequest::new();
-        request.id = Some(vec![]);
-        let response = namespace.list_tables(request).await.unwrap();
-        assert_eq!(response.tables.len(), 0);
+        // Step 1: Create tables in directory-only mode
+        let dir_only_ns = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
 
-        // Create test IPC data
+        // Create some tables
         let schema = create_test_schema();
         let ipc_data = create_test_ipc_data(&schema);
 
-        // Create a table
-        let mut create_request = CreateTableRequest::new();
-        create_request.id = Some(vec!["table1".to_string()]);
-        namespace
-            .create_table(create_request, bytes::Bytes::from(ipc_data.clone()))
+        for i in 1..=3 {
+            let mut create_req = CreateTableRequest::new();
+            create_req.id = Some(vec![format!("table{}", i)]);
+            dir_only_ns
+                .create_table(create_req, bytes::Bytes::from(ipc_data.clone()))
+                .await
+                .unwrap();
+        }
+
+        drop(dir_only_ns);
+
+        // Step 2: Create namespace with dual mode (manifest + directory listing)
+        let dual_mode_ns = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(true)
+            .dir_listing_enabled(true)
+            .build()
             .await
             .unwrap();
 
-        // Create another table
-        let mut create_request = CreateTableRequest::new();
-        create_request.id = Some(vec!["table2".to_string()]);
-        namespace
-            .create_table(create_request, bytes::Bytes::from(ipc_data))
+        // Before migration, tables should be visible (via directory listing fallback)
+        let mut list_req = ListTablesRequest::new();
+        list_req.id = Some(vec![]);
+        let tables = dual_mode_ns.list_tables(list_req).await.unwrap().tables;
+        assert_eq!(tables.len(), 3);
+
+        // Run migration
+        let migrated_count = dual_mode_ns.migrate().await.unwrap();
+        assert_eq!(migrated_count, 3, "Should migrate all 3 tables");
+
+        // Verify tables are now in manifest
+        let mut list_req = ListTablesRequest::new();
+        list_req.id = Some(vec![]);
+        let tables = dual_mode_ns.list_tables(list_req).await.unwrap().tables;
+        assert_eq!(tables.len(), 3);
+
+        // Run migration again - should be idempotent
+        let migrated_count = dual_mode_ns.migrate().await.unwrap();
+        assert_eq!(
+            migrated_count, 0,
+            "Should not migrate already-migrated tables"
+        );
+
+        drop(dual_mode_ns);
+
+        // Step 3: Create namespace with manifest-only mode
+        let manifest_only_ns = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(true)
+            .dir_listing_enabled(false)
+            .build()
             .await
             .unwrap();
 
-        // List tables should return both
-        let mut request = ListTablesRequest::new();
-        request.id = Some(vec![]);
-        let response = namespace.list_tables(request).await.unwrap();
-        let tables = response.tables;
-        assert_eq!(tables.len(), 2);
+        // Tables should still be accessible (now from manifest only)
+        let mut list_req = ListTablesRequest::new();
+        list_req.id = Some(vec![]);
+        let tables = manifest_only_ns.list_tables(list_req).await.unwrap().tables;
+        assert_eq!(tables.len(), 3);
         assert!(tables.contains(&"table1".to_string()));
         assert!(tables.contains(&"table2".to_string()));
+        assert!(tables.contains(&"table3".to_string()));
     }
 
     #[tokio::test]
-    async fn test_list_tables_with_namespace_id() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
-
-        // First create a child namespace
-        let mut create_ns_req = CreateNamespaceRequest::new();
-        create_ns_req.id = Some(vec!["test_namespace".to_string()]);
-        namespace.create_namespace(create_ns_req).await.unwrap();
-
-        // Now list tables in the child namespace
-        let mut request = ListTablesRequest::new();
-        request.id = Some(vec!["test_namespace".to_string()]);
+    async fn test_migrate_without_manifest() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        let result = namespace.list_tables(request).await;
-        // Should succeed (with manifest enabled) and return empty list (no tables yet)
-        assert!(
-            result.is_ok(),
-            "list_tables should work with child namespace when manifest is enabled"
-        );
-        let response = result.unwrap();
-        assert_eq!(
-            response.tables.len(),
-            0,
-            "Namespace should have no tables yet"
-        );
+        // Create namespace without manifest
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
+
+        // migrate() should return 0 when manifest is not enabled
+        let migrated_count = namespace.migrate().await.unwrap();
+        assert_eq!(migrated_count, 0);
     }
 
     #[tokio::test]
-    async fn test_describe_table() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+    async fn test_register_table() {
+        use lance_namespace::models::{RegisterTableRequest, TableExistsRequest};
 
-        // Create a table first
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .dir_listing_to_manifest_migration_enabled(true)
+            .build()
+            .await
+            .unwrap();
+
+        // Create a physical table first using lance directly
         let schema = create_test_schema();
         let ipc_data = create_test_ipc_data(&schema);
 
-        let mut create_request = CreateTableRequest::new();
-        create_request.id = Some(vec!["test_table".to_string()]);
-        namespace
-            .create_table(create_request, bytes::Bytes::from(ipc_data))
+        let table_uri = format!("{}/external_table.lance", temp_path);
+        let cursor = Cursor::new(ipc_data);
+        let stream_reader = StreamReader::try_new(cursor, None).unwrap();
+        let batches: Vec<_> = stream_reader
+            .collect::<std::result::Result<Vec<_>, _>>()
+            .unwrap();
+        let schema = batches[0].schema();
+        let batch_results: Vec<_> = batches.into_iter().map(Ok).collect();
+        let reader = RecordBatchIterator::new(batch_results, schema);
+        Dataset::write(Box::new(reader), &table_uri, None)
             .await
             .unwrap();
 
-        // Describe the table
-        let mut request = DescribeTableRequest::new();
-        request.id = Some(vec!["test_table".to_string()]);
-        let response = namespace.describe_table(request).await.unwrap();
-
-        assert!(response.location.is_some());
-        assert!(response.location.unwrap().ends_with("test_table.lance"));
-    }
+        // Register the table
+        let mut register_req = RegisterTableRequest::new("external_table.lance".to_string());
+        register_req.id = Some(vec!["registered_table".to_string()]);
 
-    #[tokio::test]
-    async fn test_describe_nonexistent_table() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+        let response = namespace.register_table(register_req).await.unwrap();
+        assert_eq!(response.location, Some("external_table.lance".to_string()));
 
-        let mut request = DescribeTableRequest::new();
-        request.id = Some(vec!["nonexistent".to_string()]);
+        // Verify table exists in namespace
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["registered_table".to_string()]);
+        assert!(namespace.table_exists(exists_req).await.is_ok());
 
-        let result = namespace.describe_table(request).await;
-        assert!(result.is_err());
-        assert!(
-            result
-                .unwrap_err()
-                .to_string()
-                .contains("Table does not exist")
-        );
+        // Verify we can list the table
+        let mut list_req = ListTablesRequest::new();
+        list_req.id = Some(vec![]);
+        let tables = namespace.list_tables(list_req).await.unwrap();
+        assert!(tables.tables.contains(&"registered_table".to_string()));
     }
 
     #[tokio::test]
-    async fn test_table_exists() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+    async fn test_register_table_duplicate_fails() {
+        use lance_namespace::models::RegisterTableRequest;
 
-        // Create a table
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        let mut create_request = CreateTableRequest::new();
-        create_request.id = Some(vec!["existing_table".to_string()]);
-        namespace
-            .create_table(create_request, bytes::Bytes::from(ipc_data))
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .build()
             .await
             .unwrap();
 
-        // Check existing table
-        let mut request = TableExistsRequest::new();
-        request.id = Some(vec!["existing_table".to_string()]);
-        let result = namespace.table_exists(request).await;
-        assert!(result.is_ok());
+        // Register a table
+        let mut register_req = RegisterTableRequest::new("test_table.lance".to_string());
+        register_req.id = Some(vec!["test_table".to_string()]);
 
-        // Check non-existent table
-        let mut request = TableExistsRequest::new();
-        request.id = Some(vec!["nonexistent".to_string()]);
-        let result = namespace.table_exists(request).await;
+        namespace
+            .register_table(register_req.clone())
+            .await
+            .unwrap();
+
+        // Try to register again - should fail
+        let result = namespace.register_table(register_req).await;
         assert!(result.is_err());
-        assert!(
-            result
-                .unwrap_err()
-                .to_string()
-                .contains("Table does not exist")
-        );
+        assert!(result.unwrap_err().to_string().contains("already exists"));
     }
 
     #[tokio::test]
-    async fn test_drop_table() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+    async fn test_deregister_table() {
+        use lance_namespace::models::{DeregisterTableRequest, TableExistsRequest};
+
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        // Create namespace with manifest-only mode (no directory listing fallback)
+        // This ensures deregistered tables are truly invisible
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(true)
+            .dir_listing_enabled(false)
+            .build()
+            .await
+            .unwrap();
 
         // Create a table
         let schema = create_test_schema();
         let ipc_data = create_test_ipc_data(&schema);
 
-        let mut create_request = CreateTableRequest::new();
-        create_request.id = Some(vec!["table_to_drop".to_string()]);
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["test_table".to_string()]);
         namespace
-            .create_table(create_request, bytes::Bytes::from(ipc_data))
+            .create_table(create_req, bytes::Bytes::from(ipc_data))
             .await
             .unwrap();
 
-        // Verify it exists
-        let mut exists_request = TableExistsRequest::new();
-        exists_request.id = Some(vec!["table_to_drop".to_string()]);
-        assert!(namespace.table_exists(exists_request.clone()).await.is_ok());
-
-        // Drop the table
-        let mut drop_request = DropTableRequest::new();
-        drop_request.id = Some(vec!["table_to_drop".to_string()]);
-        let response = namespace.drop_table(drop_request).await.unwrap();
-        assert!(response.location.is_some());
+        // Verify table exists
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["test_table".to_string()]);
+        assert!(namespace.table_exists(exists_req.clone()).await.is_ok());
 
-        // Verify it no longer exists
-        assert!(namespace.table_exists(exists_request).await.is_err());
-    }
+        // Deregister the table
+        let mut deregister_req = DeregisterTableRequest::new();
+        deregister_req.id = Some(vec!["test_table".to_string()]);
+        let response = namespace.deregister_table(deregister_req).await.unwrap();
 
-    #[tokio::test]
-    async fn test_drop_nonexistent_table() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+        // Should return location and id
+        assert!(
+            response.location.is_some(),
+            "Deregister should return location"
+        );
+        let location = response.location.as_ref().unwrap();
+        // Location should be a proper file:// URI with the temp path
+        // Use uri_to_url to normalize the temp path to a URL for comparison
+        let expected_url = lance_io::object_store::uri_to_url(temp_path)
+            .expect("Failed to convert temp path to URL");
+        let expected_prefix = expected_url.to_string();
+        assert!(
+            location.starts_with(&expected_prefix),
+            "Location should start with '{}', got: {}",
+            expected_prefix,
+            location
+        );
+        assert!(
+            location.contains("test_table"),
+            "Location should contain table name: {}",
+            location
+        );
+        assert_eq!(response.id, Some(vec!["test_table".to_string()]));
 
-        let mut request = DropTableRequest::new();
-        request.id = Some(vec!["nonexistent".to_string()]);
+        // Verify table no longer exists in namespace (removed from manifest)
+        assert!(namespace.table_exists(exists_req).await.is_err());
 
-        // Should not fail when dropping non-existent table (idempotent)
-        let result = namespace.drop_table(request).await;
-        // The operation might succeed or fail depending on implementation
-        // But it should not panic
-        let _ = result;
+        // Verify physical data still exists at the returned location
+        let dataset = Dataset::open(location).await;
+        assert!(
+            dataset.is_ok(),
+            "Physical table data should still exist at {}",
+            location
+        );
     }
 
     #[tokio::test]
-    async fn test_root_namespace_operations() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
-
-        // Test list_namespaces - should return empty list for root
-        let mut request = ListNamespacesRequest::new();
-        request.id = Some(vec![]);
-        let result = namespace.list_namespaces(request).await;
-        assert!(result.is_ok());
-        assert_eq!(result.unwrap().namespaces.len(), 0);
+    async fn test_deregister_table_in_child_namespace() {
+        use lance_namespace::models::{
+            CreateNamespaceRequest, DeregisterTableRequest, TableExistsRequest,
+        };
 
-        // Test describe_namespace - should succeed for root
-        let mut request = DescribeNamespaceRequest::new();
-        request.id = Some(vec![]);
-        let result = namespace.describe_namespace(request).await;
-        assert!(result.is_ok());
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        // Test namespace_exists - root always exists
-        let mut request = NamespaceExistsRequest::new();
-        request.id = Some(vec![]);
-        let result = namespace.namespace_exists(request).await;
-        assert!(result.is_ok());
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .build()
+            .await
+            .unwrap();
 
-        // Test create_namespace - root cannot be created
-        let mut request = CreateNamespaceRequest::new();
-        request.id = Some(vec![]);
-        let result = namespace.create_namespace(request).await;
-        assert!(result.is_err());
-        assert!(result.unwrap_err().to_string().contains("already exists"));
+        // Create child namespace
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["test_ns".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
 
-        // Test drop_namespace - root cannot be dropped
-        let mut request = DropNamespaceRequest::new();
-        request.id = Some(vec![]);
-        let result = namespace.drop_namespace(request).await;
-        assert!(result.is_err());
-        assert!(
-            result
-                .unwrap_err()
-                .to_string()
-                .contains("cannot be dropped")
-        );
-    }
+        // Create a table in the child namespace
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
 
-    #[tokio::test]
-    async fn test_non_root_namespace_operations() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["test_ns".to_string(), "test_table".to_string()]);
+        namespace
+            .create_table(create_req, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
 
-        // With manifest enabled (default), child namespaces are now supported
-        // Test create_namespace for non-root - should succeed with manifest
-        let mut request = CreateNamespaceRequest::new();
-        request.id = Some(vec!["child".to_string()]);
-        let result = namespace.create_namespace(request).await;
-        assert!(
-            result.is_ok(),
-            "Child namespace creation should succeed with manifest enabled"
-        );
+        // Deregister the table
+        let mut deregister_req = DeregisterTableRequest::new();
+        deregister_req.id = Some(vec!["test_ns".to_string(), "test_table".to_string()]);
+        let response = namespace.deregister_table(deregister_req).await.unwrap();
 
-        // Test namespace_exists for non-root - should exist after creation
-        let mut request = NamespaceExistsRequest::new();
-        request.id = Some(vec!["child".to_string()]);
-        let result = namespace.namespace_exists(request).await;
+        // Should return location and id in child namespace
         assert!(
-            result.is_ok(),
-            "Child namespace should exist after creation"
+            response.location.is_some(),
+            "Deregister should return location"
         );
-
-        // Test drop_namespace for non-root - should succeed
-        let mut request = DropNamespaceRequest::new();
-        request.id = Some(vec!["child".to_string()]);
-        let result = namespace.drop_namespace(request).await;
+        let location = response.location.as_ref().unwrap();
+        // Location should be a proper file:// URI with the temp path
+        // Use uri_to_url to normalize the temp path to a URL for comparison
+        let expected_url = lance_io::object_store::uri_to_url(temp_path)
+            .expect("Failed to convert temp path to URL");
+        let expected_prefix = expected_url.to_string();
         assert!(
-            result.is_ok(),
-            "Child namespace drop should succeed with manifest enabled"
+            location.starts_with(&expected_prefix),
+            "Location should start with '{}', got: {}",
+            expected_prefix,
+            location
         );
-
-        // Verify namespace no longer exists
-        let mut request = NamespaceExistsRequest::new();
-        request.id = Some(vec!["child".to_string()]);
-        let result = namespace.namespace_exists(request).await;
         assert!(
-            result.is_err(),
-            "Child namespace should not exist after drop"
+            location.contains("test_ns") && location.contains("test_table"),
+            "Location should contain namespace and table name: {}",
+            location
+        );
+        assert_eq!(
+            response.id,
+            Some(vec!["test_ns".to_string(), "test_table".to_string()])
         );
+
+        // Verify table no longer exists
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["test_ns".to_string(), "test_table".to_string()]);
+        assert!(namespace.table_exists(exists_req).await.is_err());
     }
 
     #[tokio::test]
-    async fn test_config_custom_root() {
+    async fn test_register_without_manifest_fails() {
+        use lance_namespace::models::RegisterTableRequest;
+
         let temp_dir = TempStdDir::default();
-        let custom_path = temp_dir.join("custom");
-        std::fs::create_dir(&custom_path).unwrap();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        let namespace = DirectoryNamespaceBuilder::new(custom_path.to_string_lossy().to_string())
+        // Create namespace without manifest
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(false)
             .build()
             .await
             .unwrap();
 
-        // Create test IPC data
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
+        // Try to register - should fail (register requires manifest)
+        let mut register_req = RegisterTableRequest::new("test_table.lance".to_string());
+        register_req.id = Some(vec!["test_table".to_string()]);
+        let result = namespace.register_table(register_req).await;
+        assert!(result.is_err());
+        assert!(
+            result
+                .unwrap_err()
+                .to_string()
+                .contains("manifest mode is enabled")
+        );
 
-        // Create a table and verify location
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec!["test_table".to_string()]);
+        // Note: deregister_table now works in V1 mode via .lance-deregistered marker files
+        // See test_deregister_table_v1_mode for that test case
+    }
 
-        let response = namespace
-            .create_table(request, bytes::Bytes::from(ipc_data))
+    #[tokio::test]
+    async fn test_register_table_rejects_absolute_uri() {
+        use lance_namespace::models::RegisterTableRequest;
+
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .build()
             .await
             .unwrap();
 
-        assert!(response.location.unwrap().contains("custom"));
+        // Try to register with absolute URI - should fail
+        let mut register_req = RegisterTableRequest::new("s3://bucket/table.lance".to_string());
+        register_req.id = Some(vec!["test_table".to_string()]);
+        let result = namespace.register_table(register_req).await;
+        assert!(result.is_err());
+        let err_msg = result.unwrap_err().to_string();
+        assert!(err_msg.contains("Absolute URIs are not allowed"));
     }
 
     #[tokio::test]
-    async fn test_config_storage_options() {
+    async fn test_register_table_rejects_absolute_path() {
+        use lance_namespace::models::RegisterTableRequest;
+
         let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_dir.to_str().unwrap())
-            .storage_option("option1", "value1")
-            .storage_option("option2", "value2")
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
             .build()
             .await
             .unwrap();
 
-        // Create test IPC data
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
+        // Try to register with absolute path - should fail
+        let mut register_req = RegisterTableRequest::new("/tmp/table.lance".to_string());
+        register_req.id = Some(vec!["test_table".to_string()]);
+        let result = namespace.register_table(register_req).await;
+        assert!(result.is_err());
+        let err_msg = result.unwrap_err().to_string();
+        assert!(err_msg.contains("Absolute paths are not allowed"));
+    }
 
-        // Create a table and check storage options are included
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec!["test_table".to_string()]);
+    #[tokio::test]
+    async fn test_register_table_rejects_path_traversal() {
+        use lance_namespace::models::RegisterTableRequest;
 
-        let response = namespace
-            .create_table(request, bytes::Bytes::from(ipc_data))
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .build()
             .await
             .unwrap();
 
-        let storage_options = response.storage_options.unwrap();
-        assert_eq!(storage_options.get("option1"), Some(&"value1".to_string()));
-        assert_eq!(storage_options.get("option2"), Some(&"value2".to_string()));
+        // Try to register with path traversal - should fail
+        let mut register_req = RegisterTableRequest::new("../outside/table.lance".to_string());
+        register_req.id = Some(vec!["test_table".to_string()]);
+        let result = namespace.register_table(register_req).await;
+        assert!(result.is_err());
+        let err_msg = result.unwrap_err().to_string();
+        assert!(err_msg.contains("Path traversal is not allowed"));
     }
 
     #[tokio::test]
-    async fn test_from_properties_manifest_enabled() {
-        let temp_dir = TempStdDir::default();
+    async fn test_namespace_write() {
+        use arrow::array::Int32Array;
+        use arrow::datatypes::{DataType, Field as ArrowField, Schema as ArrowSchema};
+        use arrow::record_batch::{RecordBatch, RecordBatchIterator};
+        use lance::dataset::{Dataset, WriteMode, WriteParams};
+        use lance_namespace::LanceNamespace;
 
-        let mut properties = HashMap::new();
-        properties.insert("root".to_string(), temp_dir.to_str().unwrap().to_string());
-        properties.insert("manifest_enabled".to_string(), "true".to_string());
-        properties.insert("dir_listing_enabled".to_string(), "false".to_string());
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        let namespace = Arc::new(namespace) as Arc<dyn LanceNamespace>;
 
-        let builder = DirectoryNamespaceBuilder::from_properties(properties, None).unwrap();
-        assert!(builder.manifest_enabled);
-        assert!(!builder.dir_listing_enabled);
+        // Use child namespace instead of root
+        let table_id = vec!["test_ns".to_string(), "test_table".to_string()];
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("a", DataType::Int32, false),
+            ArrowField::new("b", DataType::Int32, false),
+        ]));
 
-        let namespace = builder.build().await.unwrap();
+        // Test 1: CREATE mode
+        let data1 = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(Int32Array::from(vec![10, 20, 30])),
+            ],
+        )
+        .unwrap();
 
-        // Create test IPC data
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
+        let reader1 = RecordBatchIterator::new(vec![data1].into_iter().map(Ok), schema.clone());
+        let dataset =
+            Dataset::write_into_namespace(reader1, namespace.clone(), table_id.clone(), None)
+                .await
+                .unwrap();
 
-        // Create a table
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec!["test_table".to_string()]);
+        assert_eq!(dataset.count_rows(None).await.unwrap(), 3);
+        assert_eq!(dataset.version().version, 1);
 
-        let response = namespace
-            .create_table(request, bytes::Bytes::from(ipc_data))
-            .await
-            .unwrap();
+        // Test 2: APPEND mode
+        let data2 = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![4, 5])),
+                Arc::new(Int32Array::from(vec![40, 50])),
+            ],
+        )
+        .unwrap();
 
-        assert!(response.location.is_some());
-    }
+        let params_append = WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        };
 
-    #[tokio::test]
-    async fn test_from_properties_dir_listing_enabled() {
-        let temp_dir = TempStdDir::default();
+        let reader2 = RecordBatchIterator::new(vec![data2].into_iter().map(Ok), schema.clone());
+        let dataset = Dataset::write_into_namespace(
+            reader2,
+            namespace.clone(),
+            table_id.clone(),
+            Some(params_append),
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(dataset.count_rows(None).await.unwrap(), 5);
+        assert_eq!(dataset.version().version, 2);
+
+        // Test 3: OVERWRITE mode
+        let data3 = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![100, 200])),
+                Arc::new(Int32Array::from(vec![1000, 2000])),
+            ],
+        )
+        .unwrap();
+
+        let params_overwrite = WriteParams {
+            mode: WriteMode::Overwrite,
+            ..Default::default()
+        };
+
+        let reader3 = RecordBatchIterator::new(vec![data3].into_iter().map(Ok), schema.clone());
+        let dataset = Dataset::write_into_namespace(
+            reader3,
+            namespace.clone(),
+            table_id.clone(),
+            Some(params_overwrite),
+        )
+        .await
+        .unwrap();
 
-        let mut properties = HashMap::new();
-        properties.insert("root".to_string(), temp_dir.to_str().unwrap().to_string());
-        properties.insert("manifest_enabled".to_string(), "false".to_string());
-        properties.insert("dir_listing_enabled".to_string(), "true".to_string());
+        assert_eq!(dataset.count_rows(None).await.unwrap(), 2);
+        assert_eq!(dataset.version().version, 3);
 
-        let builder = DirectoryNamespaceBuilder::from_properties(properties, None).unwrap();
-        assert!(!builder.manifest_enabled);
-        assert!(builder.dir_listing_enabled);
+        // Verify old data was replaced
+        let result = dataset.scan().try_into_batch().await.unwrap();
+        let a_col = result
+            .column_by_name("a")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        assert_eq!(a_col.values(), &[100, 200]);
+    }
 
-        let namespace = builder.build().await.unwrap();
+    // ============================================================
+    // Tests for declare_table
+    // ============================================================
 
-        // Create test IPC data
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
+    #[tokio::test]
+    async fn test_declare_table_v1_mode() {
+        use lance_namespace::models::{
+            DeclareTableRequest, DescribeTableRequest, TableExistsRequest,
+        };
 
-        // Create a table
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec!["test_table".to_string()]);
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        let response = namespace
-            .create_table(request, bytes::Bytes::from(ipc_data))
+        // Create namespace in V1 mode (no manifest)
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(false)
+            .build()
             .await
             .unwrap();
 
-        assert!(response.location.is_some());
-    }
+        // Declare a table
+        let mut declare_req = DeclareTableRequest::new();
+        declare_req.id = Some(vec!["test_table".to_string()]);
+        let response = namespace.declare_table(declare_req).await.unwrap();
 
-    #[tokio::test]
-    async fn test_from_properties_defaults() {
-        let temp_dir = TempStdDir::default();
+        // Should return location
+        assert!(response.location.is_some());
+        let location = response.location.as_ref().unwrap();
+        assert!(location.ends_with("test_table.lance"));
 
-        let mut properties = HashMap::new();
-        properties.insert("root".to_string(), temp_dir.to_str().unwrap().to_string());
+        // Table should exist (via reserved file)
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["test_table".to_string()]);
+        assert!(namespace.table_exists(exists_req).await.is_ok());
 
-        let builder = DirectoryNamespaceBuilder::from_properties(properties, None).unwrap();
-        // Both should default to true
-        assert!(builder.manifest_enabled);
-        assert!(builder.dir_listing_enabled);
+        // Describe should work but return no version/schema (not written yet)
+        let mut describe_req = DescribeTableRequest::new();
+        describe_req.id = Some(vec!["test_table".to_string()]);
+        let describe_response = namespace.describe_table(describe_req).await.unwrap();
+        assert!(describe_response.location.is_some());
+        assert!(describe_response.version.is_none()); // Not written yet
+        assert!(describe_response.schema.is_none()); // Not written yet
     }
 
     #[tokio::test]
-    async fn test_from_properties_with_storage_options() {
+    async fn test_declare_table_with_manifest() {
+        use lance_namespace::models::{DeclareTableRequest, TableExistsRequest};
+
         let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        let mut properties = HashMap::new();
-        properties.insert("root".to_string(), temp_dir.to_str().unwrap().to_string());
-        properties.insert("manifest_enabled".to_string(), "true".to_string());
-        properties.insert("storage.region".to_string(), "us-west-2".to_string());
-        properties.insert("storage.bucket".to_string(), "my-bucket".to_string());
+        // Create namespace with manifest
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(true)
+            .dir_listing_enabled(false)
+            .build()
+            .await
+            .unwrap();
 
-        let builder = DirectoryNamespaceBuilder::from_properties(properties, None).unwrap();
-        assert!(builder.manifest_enabled);
-        assert!(builder.storage_options.is_some());
+        // Declare a table
+        let mut declare_req = DeclareTableRequest::new();
+        declare_req.id = Some(vec!["test_table".to_string()]);
+        let response = namespace.declare_table(declare_req).await.unwrap();
 
-        let storage_options = builder.storage_options.unwrap();
-        assert_eq!(
-            storage_options.get("region"),
-            Some(&"us-west-2".to_string())
-        );
-        assert_eq!(
-            storage_options.get("bucket"),
-            Some(&"my-bucket".to_string())
-        );
+        // Should return location
+        assert!(response.location.is_some());
+
+        // Table should exist in manifest
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["test_table".to_string()]);
+        assert!(namespace.table_exists(exists_req).await.is_ok());
     }
 
     #[tokio::test]
-    async fn test_various_arrow_types() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+    async fn test_declare_table_when_table_exists() {
+        use lance_namespace::models::DeclareTableRequest;
 
-        // Create schema with various types
-        let fields = vec![
-            JsonArrowField {
-                name: "bool_col".to_string(),
-                r#type: Box::new(JsonArrowDataType::new("bool".to_string())),
-                nullable: true,
-                metadata: None,
-            },
-            JsonArrowField {
-                name: "int8_col".to_string(),
-                r#type: Box::new(JsonArrowDataType::new("int8".to_string())),
-                nullable: true,
-                metadata: None,
-            },
-            JsonArrowField {
-                name: "float64_col".to_string(),
-                r#type: Box::new(JsonArrowDataType::new("float64".to_string())),
-                nullable: true,
-                metadata: None,
-            },
-            JsonArrowField {
-                name: "binary_col".to_string(),
-                r#type: Box::new(JsonArrowDataType::new("binary".to_string())),
-                nullable: true,
-                metadata: None,
-            },
-        ];
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        let schema = JsonArrowSchema {
-            fields,
-            metadata: None,
-        };
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(false)
+            .build()
+            .await
+            .unwrap();
 
-        // Create IPC data
+        // First create a table with actual data
+        let schema = create_test_schema();
         let ipc_data = create_test_ipc_data(&schema);
-
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec!["complex_table".to_string()]);
-
-        let response = namespace
-            .create_table(request, bytes::Bytes::from(ipc_data))
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["test_table".to_string()]);
+        namespace
+            .create_table(create_req, bytes::Bytes::from(ipc_data))
             .await
             .unwrap();
 
-        assert!(response.location.is_some());
+        // Try to declare the same table - should fail because it already has data
+        let mut declare_req = DeclareTableRequest::new();
+        declare_req.id = Some(vec!["test_table".to_string()]);
+        let result = namespace.declare_table(declare_req).await;
+        assert!(result.is_err());
     }
 
+    // ============================================================
+    // Tests for deregister_table in V1 mode
+    // ============================================================
+
     #[tokio::test]
-    async fn test_connect_dir() {
+    async fn test_deregister_table_v1_mode() {
+        use lance_namespace::models::{DeregisterTableRequest, TableExistsRequest};
+
         let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_dir.to_str().unwrap())
+        // Create namespace in V1 mode (no manifest, with dir listing)
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
             .build()
             .await
             .unwrap();
 
-        // Test basic operation through the concrete type
-        let mut request = ListTablesRequest::new();
-        request.id = Some(vec![]);
-        let response = namespace.list_tables(request).await.unwrap();
-        assert_eq!(response.tables.len(), 0);
-    }
+        // Create a table with data
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["test_table".to_string()]);
+        namespace
+            .create_table(create_req, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
 
-    #[tokio::test]
-    async fn test_create_table_with_ipc_data() {
-        use arrow::array::{Int32Array, StringArray};
-        use arrow::ipc::writer::StreamWriter;
+        // Verify table exists
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["test_table".to_string()]);
+        assert!(namespace.table_exists(exists_req.clone()).await.is_ok());
 
-        let (namespace, _temp_dir) = create_test_namespace().await;
+        // Deregister the table
+        let mut deregister_req = DeregisterTableRequest::new();
+        deregister_req.id = Some(vec!["test_table".to_string()]);
+        let response = namespace.deregister_table(deregister_req).await.unwrap();
 
-        // Create a schema with some fields
-        let schema = create_test_schema();
+        // Should return location
+        assert!(response.location.is_some());
+        let location = response.location.as_ref().unwrap();
+        assert!(location.contains("test_table"));
 
-        // Create some test data that matches the schema
-        let arrow_schema = convert_json_arrow_schema(&schema).unwrap();
-        let arrow_schema = Arc::new(arrow_schema);
+        // Table should no longer exist (deregistered)
+        let result = namespace.table_exists(exists_req).await;
+        assert!(result.is_err());
+        assert!(result.unwrap_err().to_string().contains("deregistered"));
 
-        // Create a RecordBatch with actual data
-        let id_array = Int32Array::from(vec![1, 2, 3]);
-        let name_array = StringArray::from(vec!["Alice", "Bob", "Charlie"]);
-        let batch = arrow::record_batch::RecordBatch::try_new(
-            arrow_schema.clone(),
-            vec![Arc::new(id_array), Arc::new(name_array)],
-        )
-        .unwrap();
+        // Physical data should still exist
+        let dataset = Dataset::open(location).await;
+        assert!(dataset.is_ok(), "Physical table data should still exist");
+    }
 
-        // Write the batch to an IPC stream
-        let mut buffer = Vec::new();
-        {
-            let mut writer = StreamWriter::try_new(&mut buffer, &arrow_schema).unwrap();
-            writer.write(&batch).unwrap();
-            writer.finish().unwrap();
-        }
+    #[tokio::test]
+    async fn test_deregister_table_v1_already_deregistered() {
+        use lance_namespace::models::DeregisterTableRequest;
+
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
 
-        // Create table with the IPC data
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec!["test_table_with_data".to_string()]);
+        // Create a table
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["test_table".to_string()]);
+        namespace
+            .create_table(create_req, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
 
-        let response = namespace
-            .create_table(request, Bytes::from(buffer))
+        // Deregister once
+        let mut deregister_req = DeregisterTableRequest::new();
+        deregister_req.id = Some(vec!["test_table".to_string()]);
+        namespace
+            .deregister_table(deregister_req.clone())
             .await
             .unwrap();
 
-        assert_eq!(response.version, Some(1));
+        // Try to deregister again - should fail
+        let result = namespace.deregister_table(deregister_req).await;
+        assert!(result.is_err());
         assert!(
-            response
-                .location
-                .unwrap()
-                .contains("test_table_with_data.lance")
+            result
+                .unwrap_err()
+                .to_string()
+                .contains("already deregistered")
         );
-
-        // Verify table exists
-        let mut exists_request = TableExistsRequest::new();
-        exists_request.id = Some(vec!["test_table_with_data".to_string()]);
-        namespace.table_exists(exists_request).await.unwrap();
     }
 
+    // ============================================================
+    // Tests for list_tables skipping deregistered tables
+    // ============================================================
+
     #[tokio::test]
-    async fn test_child_namespace_create_and_list() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+    async fn test_list_tables_skips_deregistered_v1() {
+        use lance_namespace::models::DeregisterTableRequest;
 
-        // Create multiple child namespaces
-        for i in 1..=3 {
-            let mut create_req = CreateNamespaceRequest::new();
-            create_req.id = Some(vec![format!("ns{}", i)]);
-            let result = namespace.create_namespace(create_req).await;
-            assert!(result.is_ok(), "Failed to create child namespace ns{}", i);
-        }
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        // List child namespaces
-        let list_req = ListNamespacesRequest {
-            id: Some(vec![]),
-            ..Default::default()
-        };
-        let result = namespace.list_namespaces(list_req).await;
-        assert!(result.is_ok());
-        let namespaces = result.unwrap().namespaces;
-        assert_eq!(namespaces.len(), 3);
-        assert!(namespaces.contains(&"ns1".to_string()));
-        assert!(namespaces.contains(&"ns2".to_string()));
-        assert!(namespaces.contains(&"ns3".to_string()));
-    }
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
 
-    #[tokio::test]
-    async fn test_nested_namespace_hierarchy() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+        // Create two tables
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
 
-        // Create parent namespace
-        let mut create_req = CreateNamespaceRequest::new();
-        create_req.id = Some(vec!["parent".to_string()]);
-        namespace.create_namespace(create_req).await.unwrap();
+        let mut create_req1 = CreateTableRequest::new();
+        create_req1.id = Some(vec!["table1".to_string()]);
+        namespace
+            .create_table(create_req1, bytes::Bytes::from(ipc_data.clone()))
+            .await
+            .unwrap();
 
-        // Create nested children
-        let mut create_req = CreateNamespaceRequest::new();
-        create_req.id = Some(vec!["parent".to_string(), "child1".to_string()]);
-        namespace.create_namespace(create_req).await.unwrap();
+        let mut create_req2 = CreateTableRequest::new();
+        create_req2.id = Some(vec!["table2".to_string()]);
+        namespace
+            .create_table(create_req2, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
 
-        let mut create_req = CreateNamespaceRequest::new();
-        create_req.id = Some(vec!["parent".to_string(), "child2".to_string()]);
-        namespace.create_namespace(create_req).await.unwrap();
+        // List tables - should see both (root namespace = empty vec)
+        let mut list_req = ListTablesRequest::new();
+        list_req.id = Some(vec![]);
+        let list_response = namespace.list_tables(list_req.clone()).await.unwrap();
+        assert_eq!(list_response.tables.len(), 2);
 
-        // List children of parent
-        let list_req = ListNamespacesRequest {
-            id: Some(vec!["parent".to_string()]),
-            ..Default::default()
-        };
-        let result = namespace.list_namespaces(list_req).await;
-        assert!(result.is_ok());
-        let children = result.unwrap().namespaces;
-        assert_eq!(children.len(), 2);
-        assert!(children.contains(&"child1".to_string()));
-        assert!(children.contains(&"child2".to_string()));
+        // Deregister table1
+        let mut deregister_req = DeregisterTableRequest::new();
+        deregister_req.id = Some(vec!["table1".to_string()]);
+        namespace.deregister_table(deregister_req).await.unwrap();
 
-        // List root should only show parent
-        let list_req = ListNamespacesRequest {
-            id: Some(vec![]),
-            ..Default::default()
-        };
-        let result = namespace.list_namespaces(list_req).await;
-        assert!(result.is_ok());
-        let root_namespaces = result.unwrap().namespaces;
-        assert_eq!(root_namespaces.len(), 1);
-        assert_eq!(root_namespaces[0], "parent");
+        // List tables - should only see table2
+        let list_response = namespace.list_tables(list_req).await.unwrap();
+        assert_eq!(list_response.tables.len(), 1);
+        assert!(list_response.tables.contains(&"table2".to_string()));
+        assert!(!list_response.tables.contains(&"table1".to_string()));
     }
 
+    // ============================================================
+    // Tests for describe_table and table_exists with deregistered tables
+    // ============================================================
+
     #[tokio::test]
-    async fn test_table_in_child_namespace() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+    async fn test_describe_table_fails_for_deregistered_v1() {
+        use lance_namespace::models::{DeregisterTableRequest, DescribeTableRequest};
 
-        // Create child namespace
-        let mut create_ns_req = CreateNamespaceRequest::new();
-        create_ns_req.id = Some(vec!["test_ns".to_string()]);
-        namespace.create_namespace(create_ns_req).await.unwrap();
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        // Create table in child namespace
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
+
+        // Create a table
         let schema = create_test_schema();
         let ipc_data = create_test_ipc_data(&schema);
-        let mut create_table_req = CreateTableRequest::new();
-        create_table_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
-        let result = namespace
-            .create_table(create_table_req, bytes::Bytes::from(ipc_data))
-            .await;
-        assert!(result.is_ok(), "Failed to create table in child namespace");
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["test_table".to_string()]);
+        namespace
+            .create_table(create_req, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
 
-        // List tables in child namespace
-        let list_req = ListTablesRequest {
-            id: Some(vec!["test_ns".to_string()]),
-            ..Default::default()
-        };
-        let result = namespace.list_tables(list_req).await;
-        assert!(result.is_ok());
-        let tables = result.unwrap().tables;
-        assert_eq!(tables.len(), 1);
-        assert_eq!(tables[0], "table1");
+        // Describe should work before deregistration
+        let mut describe_req = DescribeTableRequest::new();
+        describe_req.id = Some(vec!["test_table".to_string()]);
+        assert!(namespace.describe_table(describe_req.clone()).await.is_ok());
 
-        // Verify table exists
-        let mut exists_req = TableExistsRequest::new();
-        exists_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
-        let result = namespace.table_exists(exists_req).await;
-        assert!(result.is_ok());
+        // Deregister
+        let mut deregister_req = DeregisterTableRequest::new();
+        deregister_req.id = Some(vec!["test_table".to_string()]);
+        namespace.deregister_table(deregister_req).await.unwrap();
 
-        // Describe table in child namespace
-        let mut describe_req = DescribeTableRequest::new();
-        describe_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
+        // Describe should fail after deregistration
         let result = namespace.describe_table(describe_req).await;
-        assert!(result.is_ok());
-        let response = result.unwrap();
-        assert!(response.location.is_some());
+        assert!(result.is_err());
+        let err = result.unwrap_err();
+        assert!(matches!(err, Error::Namespace { .. }));
+        let err_msg = err.to_string();
+        assert!(err_msg.contains("deregistered"));
+        assert!(err_msg.contains("table id 'test_table'"));
     }
 
     #[tokio::test]
-    async fn test_multiple_tables_in_child_namespace() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
-
-        // Create child namespace
-        let mut create_ns_req = CreateNamespaceRequest::new();
-        create_ns_req.id = Some(vec!["test_ns".to_string()]);
-        namespace.create_namespace(create_ns_req).await.unwrap();
-
-        // Create multiple tables
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-        for i in 1..=3 {
-            let mut create_table_req = CreateTableRequest::new();
-            create_table_req.id = Some(vec!["test_ns".to_string(), format!("table{}", i)]);
-            namespace
-                .create_table(create_table_req, bytes::Bytes::from(ipc_data.clone()))
-                .await
-                .unwrap();
-        }
-
-        // List tables
-        let list_req = ListTablesRequest {
-            id: Some(vec!["test_ns".to_string()]),
-            ..Default::default()
-        };
-        let result = namespace.list_tables(list_req).await;
-        assert!(result.is_ok());
-        let tables = result.unwrap().tables;
-        assert_eq!(tables.len(), 3);
-        assert!(tables.contains(&"table1".to_string()));
-        assert!(tables.contains(&"table2".to_string()));
-        assert!(tables.contains(&"table3".to_string()));
-    }
+    async fn test_table_exists_fails_for_deregistered_v1() {
+        use lance_namespace::models::{DeregisterTableRequest, TableExistsRequest};
 
-    #[tokio::test]
-    async fn test_drop_table_in_child_namespace() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        // Create child namespace
-        let mut create_ns_req = CreateNamespaceRequest::new();
-        create_ns_req.id = Some(vec!["test_ns".to_string()]);
-        namespace.create_namespace(create_ns_req).await.unwrap();
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
 
-        // Create table
+        // Create a table
         let schema = create_test_schema();
         let ipc_data = create_test_ipc_data(&schema);
-        let mut create_table_req = CreateTableRequest::new();
-        create_table_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["test_table".to_string()]);
         namespace
-            .create_table(create_table_req, bytes::Bytes::from(ipc_data))
+            .create_table(create_req, bytes::Bytes::from(ipc_data))
             .await
             .unwrap();
 
-        // Drop table
-        let mut drop_req = DropTableRequest::new();
-        drop_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
-        let result = namespace.drop_table(drop_req).await;
-        assert!(result.is_ok(), "Failed to drop table in child namespace");
-
-        // Verify table no longer exists
+        // Table exists should work before deregistration
         let mut exists_req = TableExistsRequest::new();
-        exists_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
+        exists_req.id = Some(vec!["test_table".to_string()]);
+        assert!(namespace.table_exists(exists_req.clone()).await.is_ok());
+
+        // Deregister
+        let mut deregister_req = DeregisterTableRequest::new();
+        deregister_req.id = Some(vec!["test_table".to_string()]);
+        namespace.deregister_table(deregister_req).await.unwrap();
+
+        // Table exists should fail after deregistration
         let result = namespace.table_exists(exists_req).await;
         assert!(result.is_err());
+        let err = result.unwrap_err();
+        assert!(matches!(err, Error::Namespace { .. }));
+        let err_msg = err.to_string();
+        assert!(err_msg.contains("deregistered"));
+        assert!(err_msg.contains("table id 'test_table'"));
     }
 
     #[tokio::test]
-    async fn test_deeply_nested_namespace() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
-
-        // Create deeply nested namespace hierarchy
-        let mut create_req = CreateNamespaceRequest::new();
-        create_req.id = Some(vec!["level1".to_string()]);
-        namespace.create_namespace(create_req).await.unwrap();
+    async fn test_atomic_table_status_check() {
+        // This test verifies that the TableStatus check is atomic
+        // by ensuring a single directory listing is used
 
-        let mut create_req = CreateNamespaceRequest::new();
-        create_req.id = Some(vec!["level1".to_string(), "level2".to_string()]);
-        namespace.create_namespace(create_req).await.unwrap();
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        let mut create_req = CreateNamespaceRequest::new();
-        create_req.id = Some(vec![
-            "level1".to_string(),
-            "level2".to_string(),
-            "level3".to_string(),
-        ]);
-        namespace.create_namespace(create_req).await.unwrap();
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
 
-        // Create table in deeply nested namespace
+        // Create a table
         let schema = create_test_schema();
         let ipc_data = create_test_ipc_data(&schema);
-        let mut create_table_req = CreateTableRequest::new();
-        create_table_req.id = Some(vec![
-            "level1".to_string(),
-            "level2".to_string(),
-            "level3".to_string(),
-            "table1".to_string(),
-        ]);
-        let result = namespace
-            .create_table(create_table_req, bytes::Bytes::from(ipc_data))
-            .await;
-        assert!(
-            result.is_ok(),
-            "Failed to create table in deeply nested namespace"
-        );
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["test_table".to_string()]);
+        namespace
+            .create_table(create_req, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
 
-        // Verify table exists
-        let mut exists_req = TableExistsRequest::new();
-        exists_req.id = Some(vec![
-            "level1".to_string(),
-            "level2".to_string(),
-            "level3".to_string(),
-            "table1".to_string(),
-        ]);
-        let result = namespace.table_exists(exists_req).await;
-        assert!(result.is_ok());
+        // Table status should show exists=true, is_deregistered=false
+        let status = namespace.check_table_status("test_table").await;
+        assert!(status.exists);
+        assert!(!status.is_deregistered);
+        assert!(!status.has_reserved_file);
     }
 
     #[tokio::test]
-    async fn test_namespace_with_properties() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+    async fn test_table_version_tracking_enabled_managed_versioning() {
+        use lance_namespace::models::DescribeTableRequest;
 
-        // Create namespace with properties
-        let mut properties = HashMap::new();
-        properties.insert("owner".to_string(), "test_user".to_string());
-        properties.insert("description".to_string(), "Test namespace".to_string());
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        let mut create_req = CreateNamespaceRequest::new();
-        create_req.id = Some(vec!["test_ns".to_string()]);
-        create_req.properties = Some(properties.clone());
-        namespace.create_namespace(create_req).await.unwrap();
+        // Create namespace with table_version_tracking_enabled=true
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .table_version_tracking_enabled(true)
+            .build()
+            .await
+            .unwrap();
 
-        // Describe namespace and verify properties
-        let describe_req = DescribeNamespaceRequest {
-            id: Some(vec!["test_ns".to_string()]),
-            ..Default::default()
-        };
-        let result = namespace.describe_namespace(describe_req).await;
-        assert!(result.is_ok());
-        let response = result.unwrap();
-        assert!(response.properties.is_some());
-        let props = response.properties.unwrap();
-        assert_eq!(props.get("owner"), Some(&"test_user".to_string()));
+        // Create a table
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["test_table".to_string()]);
+        namespace
+            .create_table(create_req, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        // Describe table should return managed_versioning=true
+        let mut describe_req = DescribeTableRequest::new();
+        describe_req.id = Some(vec!["test_table".to_string()]);
+        let describe_resp = namespace.describe_table(describe_req).await.unwrap();
+
+        // managed_versioning should be true
         assert_eq!(
-            props.get("description"),
-            Some(&"Test namespace".to_string())
+            describe_resp.managed_versioning,
+            Some(true),
+            "managed_versioning should be true when table_version_tracking_enabled=true"
         );
     }
 
     #[tokio::test]
-    async fn test_cannot_drop_namespace_with_tables() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+    async fn test_table_version_tracking_disabled_no_managed_versioning() {
+        use lance_namespace::models::DescribeTableRequest;
 
-        // Create namespace
-        let mut create_ns_req = CreateNamespaceRequest::new();
-        create_ns_req.id = Some(vec!["test_ns".to_string()]);
-        namespace.create_namespace(create_ns_req).await.unwrap();
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-        // Create table in namespace
+        // Create namespace with table_version_tracking_enabled=false (default)
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .table_version_tracking_enabled(false)
+            .build()
+            .await
+            .unwrap();
+
+        // Create a table
         let schema = create_test_schema();
         let ipc_data = create_test_ipc_data(&schema);
-        let mut create_table_req = CreateTableRequest::new();
-        create_table_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]);
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["test_table".to_string()]);
         namespace
-            .create_table(create_table_req, bytes::Bytes::from(ipc_data))
+            .create_table(create_req, bytes::Bytes::from(ipc_data))
             .await
             .unwrap();
 
-        // Try to drop namespace - should fail
-        let mut drop_req = DropNamespaceRequest::new();
-        drop_req.id = Some(vec!["test_ns".to_string()]);
-        let result = namespace.drop_namespace(drop_req).await;
+        // Describe table should not have managed_versioning set
+        let mut describe_req = DescribeTableRequest::new();
+        describe_req.id = Some(vec!["test_table".to_string()]);
+        let describe_resp = namespace.describe_table(describe_req).await.unwrap();
+
+        // managed_versioning should be None when table_version_tracking_enabled=false
         assert!(
-            result.is_err(),
-            "Should not be able to drop namespace with tables"
+            describe_resp.managed_versioning.is_none(),
+            "managed_versioning should be None when table_version_tracking_enabled=false, got: {:?}",
+            describe_resp.managed_versioning
         );
     }
 
-    #[tokio::test]
-    async fn test_isolation_between_namespaces() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+    #[tokio::test]
+    #[cfg(not(windows))]
+    async fn test_list_table_versions() {
+        use arrow::array::{Int32Array, RecordBatchIterator};
+        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+        use arrow::record_batch::RecordBatch;
+        use lance::dataset::{Dataset, WriteMode, WriteParams};
+        use lance_namespace::models::{CreateNamespaceRequest, ListTableVersionsRequest};
+
+        let temp_dir = TempStrDir::default();
+        let temp_path: &str = &temp_dir;
+
+        let namespace: Arc<dyn LanceNamespace> = Arc::new(
+            DirectoryNamespaceBuilder::new(temp_path)
+                .table_version_tracking_enabled(true)
+                .build()
+                .await
+                .unwrap(),
+        );
+
+        // Create parent namespace first
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["workspace".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
+
+        // Create a table using write_into_namespace (version 1)
+        let table_id = vec!["workspace".to_string(), "test_table".to_string()];
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            arrow_schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )
+        .unwrap();
+        let batches = RecordBatchIterator::new(vec![Ok(batch.clone())], arrow_schema.clone());
+        let write_params = WriteParams {
+            mode: WriteMode::Create,
+            ..Default::default()
+        };
+        let mut dataset = Dataset::write_into_namespace(
+            batches,
+            namespace.clone(),
+            table_id.clone(),
+            Some(write_params),
+        )
+        .await
+        .unwrap();
+
+        // Append to create version 2
+        let batch2 = RecordBatch::try_new(
+            arrow_schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![100, 200]))],
+        )
+        .unwrap();
+        let batches = RecordBatchIterator::new(vec![Ok(batch2)], arrow_schema.clone());
+        dataset.append(batches, None).await.unwrap();
+
+        // Append to create version 3
+        let batch3 = RecordBatch::try_new(
+            arrow_schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![300, 400]))],
+        )
+        .unwrap();
+        let batches = RecordBatchIterator::new(vec![Ok(batch3)], arrow_schema);
+        dataset.append(batches, None).await.unwrap();
+
+        // List versions - should have versions 1, 2, and 3
+        let mut list_req = ListTableVersionsRequest::new();
+        list_req.id = Some(table_id.clone());
+        let list_resp = namespace.list_table_versions(list_req).await.unwrap();
+
+        assert_eq!(
+            list_resp.versions.len(),
+            3,
+            "Should have 3 versions, got: {:?}",
+            list_resp.versions
+        );
 
-        // Create two namespaces
-        let mut create_req = CreateNamespaceRequest::new();
-        create_req.id = Some(vec!["ns1".to_string()]);
-        namespace.create_namespace(create_req).await.unwrap();
+        // Verify each version
+        for expected_version in 1..=3 {
+            let version = list_resp
+                .versions
+                .iter()
+                .find(|v| v.version == expected_version)
+                .unwrap_or_else(|| panic!("Expected version {}", expected_version));
 
-        let mut create_req = CreateNamespaceRequest::new();
-        create_req.id = Some(vec!["ns2".to_string()]);
-        namespace.create_namespace(create_req).await.unwrap();
+            assert!(
+                !version.manifest_path.is_empty(),
+                "manifest_path should be set for version {}",
+                expected_version
+            );
+            assert!(
+                version.manifest_path.contains(".manifest"),
+                "manifest_path should contain .manifest for version {}",
+                expected_version
+            );
+            assert!(
+                version.manifest_size.is_some(),
+                "manifest_size should be set for version {}",
+                expected_version
+            );
+            assert!(
+                version.manifest_size.unwrap() > 0,
+                "manifest_size should be > 0 for version {}",
+                expected_version
+            );
+            assert!(
+                version.timestamp_millis.is_some(),
+                "timestamp_millis should be set for version {}",
+                expected_version
+            );
+        }
+    }
 
-        // Create table with same name in both namespaces
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
+    #[tokio::test]
+    #[cfg(not(windows))]
+    async fn test_describe_table_version() {
+        use arrow::array::{Int32Array, RecordBatchIterator};
+        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+        use arrow::record_batch::RecordBatch;
+        use lance::dataset::{Dataset, WriteMode, WriteParams};
+        use lance_namespace::models::{CreateNamespaceRequest, DescribeTableVersionRequest};
 
-        let mut create_table_req = CreateTableRequest::new();
-        create_table_req.id = Some(vec!["ns1".to_string(), "table1".to_string()]);
-        namespace
-            .create_table(create_table_req, bytes::Bytes::from(ipc_data.clone()))
-            .await
-            .unwrap();
+        let temp_dir = TempStrDir::default();
+        let temp_path: &str = &temp_dir;
 
-        let mut create_table_req = CreateTableRequest::new();
-        create_table_req.id = Some(vec!["ns2".to_string(), "table1".to_string()]);
-        namespace
-            .create_table(create_table_req, bytes::Bytes::from(ipc_data))
-            .await
-            .unwrap();
+        let namespace: Arc<dyn LanceNamespace> = Arc::new(
+            DirectoryNamespaceBuilder::new(temp_path)
+                .table_version_tracking_enabled(true)
+                .build()
+                .await
+                .unwrap(),
+        );
 
-        // List tables in each namespace
-        let list_req = ListTablesRequest {
-            id: Some(vec!["ns1".to_string()]),
-            page_token: None,
-            limit: None,
-            ..Default::default()
-        };
-        let result = namespace.list_tables(list_req).await.unwrap();
-        assert_eq!(result.tables.len(), 1);
-        assert_eq!(result.tables[0], "table1");
+        // Create parent namespace first
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["workspace".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
 
-        let list_req = ListTablesRequest {
-            id: Some(vec!["ns2".to_string()]),
-            page_token: None,
-            limit: None,
+        // Create a table using write_into_namespace (version 1)
+        let table_id = vec!["workspace".to_string(), "test_table".to_string()];
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            arrow_schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )
+        .unwrap();
+        let batches = RecordBatchIterator::new(vec![Ok(batch)], arrow_schema.clone());
+        let write_params = WriteParams {
+            mode: WriteMode::Create,
             ..Default::default()
         };
-        let result = namespace.list_tables(list_req).await.unwrap();
-        assert_eq!(result.tables.len(), 1);
-        assert_eq!(result.tables[0], "table1");
-
-        // Drop table in ns1 shouldn't affect ns2
-        let mut drop_req = DropTableRequest::new();
-        drop_req.id = Some(vec!["ns1".to_string(), "table1".to_string()]);
-        namespace.drop_table(drop_req).await.unwrap();
-
-        // Verify ns1 table is gone but ns2 table still exists
-        let mut exists_req = TableExistsRequest::new();
-        exists_req.id = Some(vec!["ns1".to_string(), "table1".to_string()]);
-        assert!(namespace.table_exists(exists_req).await.is_err());
-
-        let mut exists_req = TableExistsRequest::new();
-        exists_req.id = Some(vec!["ns2".to_string(), "table1".to_string()]);
-        assert!(namespace.table_exists(exists_req).await.is_ok());
-    }
+        let mut dataset = Dataset::write_into_namespace(
+            batches,
+            namespace.clone(),
+            table_id.clone(),
+            Some(write_params),
+        )
+        .await
+        .unwrap();
 
-    #[tokio::test]
-    async fn test_migrate_directory_tables() {
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+        // Append data to create version 2
+        let batch2 = RecordBatch::try_new(
+            arrow_schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![100, 200]))],
+        )
+        .unwrap();
+        let batches = RecordBatchIterator::new(vec![Ok(batch2)], arrow_schema);
+        dataset.append(batches, None).await.unwrap();
 
-        // Step 1: Create tables in directory-only mode
-        let dir_only_ns = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(false)
-            .dir_listing_enabled(true)
-            .build()
+        // Describe version 1
+        let mut describe_req = DescribeTableVersionRequest::new();
+        describe_req.id = Some(table_id.clone());
+        describe_req.version = Some(1);
+        let describe_resp = namespace
+            .describe_table_version(describe_req)
             .await
             .unwrap();
 
-        // Create some tables
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-
-        for i in 1..=3 {
-            let mut create_req = CreateTableRequest::new();
-            create_req.id = Some(vec![format!("table{}", i)]);
-            dir_only_ns
-                .create_table(create_req, bytes::Bytes::from(ipc_data.clone()))
-                .await
-                .unwrap();
-        }
-
-        drop(dir_only_ns);
+        let version = &describe_resp.version;
+        assert_eq!(version.version, 1);
+        assert!(version.timestamp_millis.is_some());
+        assert!(
+            !version.manifest_path.is_empty(),
+            "manifest_path should be set"
+        );
+        assert!(
+            version.manifest_path.contains(".manifest"),
+            "manifest_path should contain .manifest"
+        );
+        assert!(
+            version.manifest_size.is_some(),
+            "manifest_size should be set"
+        );
+        assert!(
+            version.manifest_size.unwrap() > 0,
+            "manifest_size should be > 0"
+        );
 
-        // Step 2: Create namespace with dual mode (manifest + directory listing)
-        let dual_mode_ns = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(true)
-            .dir_listing_enabled(true)
-            .build()
+        // Describe version 2
+        let mut describe_req = DescribeTableVersionRequest::new();
+        describe_req.id = Some(table_id.clone());
+        describe_req.version = Some(2);
+        let describe_resp = namespace
+            .describe_table_version(describe_req)
             .await
             .unwrap();
 
-        // Before migration, tables should be visible (via directory listing fallback)
-        let mut list_req = ListTablesRequest::new();
-        list_req.id = Some(vec![]);
-        let tables = dual_mode_ns.list_tables(list_req).await.unwrap().tables;
-        assert_eq!(tables.len(), 3);
+        let version = &describe_resp.version;
+        assert_eq!(version.version, 2);
+        assert!(version.timestamp_millis.is_some());
+        assert!(
+            !version.manifest_path.is_empty(),
+            "manifest_path should be set"
+        );
+        assert!(
+            version.manifest_size.is_some(),
+            "manifest_size should be set"
+        );
+        assert!(
+            version.manifest_size.unwrap() > 0,
+            "manifest_size should be > 0"
+        );
+    }
 
-        // Run migration
-        let migrated_count = dual_mode_ns.migrate().await.unwrap();
-        assert_eq!(migrated_count, 3, "Should migrate all 3 tables");
+    #[tokio::test]
+    #[cfg(not(windows))]
+    async fn test_describe_table_version_latest() {
+        use arrow::array::{Int32Array, RecordBatchIterator};
+        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+        use arrow::record_batch::RecordBatch;
+        use lance::dataset::{Dataset, WriteMode, WriteParams};
+        use lance_namespace::models::{CreateNamespaceRequest, DescribeTableVersionRequest};
 
-        // Verify tables are now in manifest
-        let mut list_req = ListTablesRequest::new();
-        list_req.id = Some(vec![]);
-        let tables = dual_mode_ns.list_tables(list_req).await.unwrap().tables;
-        assert_eq!(tables.len(), 3);
+        let temp_dir = TempStrDir::default();
+        let temp_path: &str = &temp_dir;
 
-        // Run migration again - should be idempotent
-        let migrated_count = dual_mode_ns.migrate().await.unwrap();
-        assert_eq!(
-            migrated_count, 0,
-            "Should not migrate already-migrated tables"
+        let namespace: Arc<dyn LanceNamespace> = Arc::new(
+            DirectoryNamespaceBuilder::new(temp_path)
+                .table_version_tracking_enabled(true)
+                .build()
+                .await
+                .unwrap(),
         );
 
-        drop(dual_mode_ns);
+        // Create parent namespace first
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["workspace".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
 
-        // Step 3: Create namespace with manifest-only mode
-        let manifest_only_ns = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(true)
-            .dir_listing_enabled(false)
-            .build()
-            .await
-            .unwrap();
+        // Create a table using write_into_namespace (version 1)
+        let table_id = vec!["workspace".to_string(), "test_table".to_string()];
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            arrow_schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )
+        .unwrap();
+        let batches = RecordBatchIterator::new(vec![Ok(batch)], arrow_schema.clone());
+        let write_params = WriteParams {
+            mode: WriteMode::Create,
+            ..Default::default()
+        };
+        let mut dataset = Dataset::write_into_namespace(
+            batches,
+            namespace.clone(),
+            table_id.clone(),
+            Some(write_params),
+        )
+        .await
+        .unwrap();
 
-        // Tables should still be accessible (now from manifest only)
-        let mut list_req = ListTablesRequest::new();
-        list_req.id = Some(vec![]);
-        let tables = manifest_only_ns.list_tables(list_req).await.unwrap().tables;
-        assert_eq!(tables.len(), 3);
-        assert!(tables.contains(&"table1".to_string()));
-        assert!(tables.contains(&"table2".to_string()));
-        assert!(tables.contains(&"table3".to_string()));
-    }
+        // Append to create version 2
+        let batch2 = RecordBatch::try_new(
+            arrow_schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![100, 200]))],
+        )
+        .unwrap();
+        let batches = RecordBatchIterator::new(vec![Ok(batch2)], arrow_schema.clone());
+        dataset.append(batches, None).await.unwrap();
 
-    #[tokio::test]
-    async fn test_migrate_without_manifest() {
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+        // Append to create version 3
+        let batch3 = RecordBatch::try_new(
+            arrow_schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![300, 400]))],
+        )
+        .unwrap();
+        let batches = RecordBatchIterator::new(vec![Ok(batch3)], arrow_schema);
+        dataset.append(batches, None).await.unwrap();
 
-        // Create namespace without manifest
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(false)
-            .dir_listing_enabled(true)
-            .build()
+        // Describe latest version (no version specified)
+        let mut describe_req = DescribeTableVersionRequest::new();
+        describe_req.id = Some(table_id.clone());
+        describe_req.version = None;
+        let describe_resp = namespace
+            .describe_table_version(describe_req)
             .await
             .unwrap();
 
-        // migrate() should return 0 when manifest is not enabled
-        let migrated_count = namespace.migrate().await.unwrap();
-        assert_eq!(migrated_count, 0);
+        // Should return version 3 as it's the latest
+        assert_eq!(describe_resp.version.version, 3);
     }
 
     #[tokio::test]
-    async fn test_register_table() {
-        use lance_namespace::models::{RegisterTableRequest, TableExistsRequest};
+    #[cfg(not(windows))]
+    async fn test_create_table_version() {
+        use futures::TryStreamExt;
+        use lance::dataset::builder::DatasetBuilder;
+        use lance_namespace::models::CreateTableVersionRequest;
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+        let temp_dir = TempStrDir::default();
+        let temp_path: &str = &temp_dir;
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .build()
-            .await
-            .unwrap();
+        let namespace: Arc<dyn LanceNamespace> = Arc::new(
+            DirectoryNamespaceBuilder::new(temp_path)
+                .table_version_tracking_enabled(true)
+                .build()
+                .await
+                .unwrap(),
+        );
 
-        // Create a physical table first using lance directly
+        // Create a table
         let schema = create_test_schema();
         let ipc_data = create_test_ipc_data(&schema);
-
-        let table_uri = format!("{}/external_table.lance", temp_path);
-        let cursor = Cursor::new(ipc_data);
-        let stream_reader = StreamReader::try_new(cursor, None).unwrap();
-        let batches: Vec<_> = stream_reader
-            .collect::<std::result::Result<Vec<_>, _>>()
-            .unwrap();
-        let schema = batches[0].schema();
-        let batch_results: Vec<_> = batches.into_iter().map(Ok).collect();
-        let reader = RecordBatchIterator::new(batch_results, schema);
-        Dataset::write(Box::new(reader), &table_uri, None)
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["test_table".to_string()]);
+        namespace
+            .create_table(create_req, bytes::Bytes::from(ipc_data))
             .await
             .unwrap();
 
-        // Register the table
-        let mut register_req = RegisterTableRequest::new("external_table.lance".to_string());
-        register_req.id = Some(vec!["registered_table".to_string()]);
-
-        let response = namespace.register_table(register_req).await.unwrap();
-        assert_eq!(response.location, Some("external_table.lance".to_string()));
-
-        // Verify table exists in namespace
-        let mut exists_req = TableExistsRequest::new();
-        exists_req.id = Some(vec!["registered_table".to_string()]);
-        assert!(namespace.table_exists(exists_req).await.is_ok());
-
-        // Verify we can list the table
-        let mut list_req = ListTablesRequest::new();
-        list_req.id = Some(vec![]);
-        let tables = namespace.list_tables(list_req).await.unwrap();
-        assert!(tables.tables.contains(&"registered_table".to_string()));
-    }
-
-    #[tokio::test]
-    async fn test_register_table_duplicate_fails() {
-        use lance_namespace::models::RegisterTableRequest;
-
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
-
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .build()
+        // Open the dataset using from_namespace to get proper object_store and paths
+        let table_id = vec!["test_table".to_string()];
+        let dataset = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone())
+            .await
+            .unwrap()
+            .load()
             .await
             .unwrap();
 
-        // Register a table
-        let mut register_req = RegisterTableRequest::new("test_table.lance".to_string());
-        register_req.id = Some(vec!["test_table".to_string()]);
-
-        namespace
-            .register_table(register_req.clone())
+        // Use dataset's object_store to find and copy the manifest
+        let versions_path = dataset.versions_dir();
+        let manifest_metas: Vec<_> = dataset
+            .object_store()
+            .inner
+            .list(Some(&versions_path))
+            .try_collect()
             .await
             .unwrap();
 
-        // Try to register again - should fail
-        let result = namespace.register_table(register_req).await;
-        assert!(result.is_err());
-        assert!(result.unwrap_err().to_string().contains("already exists"));
-    }
-
-    #[tokio::test]
-    async fn test_deregister_table() {
-        use lance_namespace::models::{DeregisterTableRequest, TableExistsRequest};
-
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+        let manifest_meta = manifest_metas
+            .iter()
+            .find(|m| {
+                m.location
+                    .filename()
+                    .map(|f| f.ends_with(".manifest"))
+                    .unwrap_or(false)
+            })
+            .expect("No manifest file found");
 
-        // Create namespace with manifest-only mode (no directory listing fallback)
-        // This ensures deregistered tables are truly invisible
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(true)
-            .dir_listing_enabled(false)
-            .build()
+        // Read the existing manifest data
+        let manifest_data = dataset
+            .object_store()
+            .inner
+            .get(&manifest_meta.location)
+            .await
+            .unwrap()
+            .bytes()
             .await
             .unwrap();
 
-        // Create a table
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-
-        let mut create_req = CreateTableRequest::new();
-        create_req.id = Some(vec!["test_table".to_string()]);
-        namespace
-            .create_table(create_req, bytes::Bytes::from(ipc_data))
+        // Write to a staging location using the dataset's object_store
+        let staging_path = dataset.versions_dir().child("staging_manifest");
+        dataset
+            .object_store()
+            .inner
+            .put(&staging_path, manifest_data.into())
             .await
             .unwrap();
 
-        // Verify table exists
-        let mut exists_req = TableExistsRequest::new();
-        exists_req.id = Some(vec!["test_table".to_string()]);
-        assert!(namespace.table_exists(exists_req.clone()).await.is_ok());
-
-        // Deregister the table
-        let mut deregister_req = DeregisterTableRequest::new();
-        deregister_req.id = Some(vec!["test_table".to_string()]);
-        let response = namespace.deregister_table(deregister_req).await.unwrap();
+        // Create version 2 from staging manifest
+        // Use the same naming scheme as the existing dataset (V2)
+        let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string());
+        create_version_req.id = Some(table_id.clone());
+        create_version_req.naming_scheme = Some("V2".to_string());
 
-        // Should return location and id
-        assert!(
-            response.location.is_some(),
-            "Deregister should return location"
-        );
-        let location = response.location.as_ref().unwrap();
-        // Location should be a proper file:// URI with the temp path
-        // Use uri_to_url to normalize the temp path to a URL for comparison
-        let expected_url = lance_io::object_store::uri_to_url(temp_path)
-            .expect("Failed to convert temp path to URL");
-        let expected_prefix = expected_url.to_string();
-        assert!(
-            location.starts_with(&expected_prefix),
-            "Location should start with '{}', got: {}",
-            expected_prefix,
-            location
-        );
+        let result = namespace.create_table_version(create_version_req).await;
         assert!(
-            location.contains("test_table"),
-            "Location should contain table name: {}",
-            location
+            result.is_ok(),
+            "create_table_version should succeed: {:?}",
+            result
         );
-        assert_eq!(response.id, Some(vec!["test_table".to_string()]));
 
-        // Verify table no longer exists in namespace (removed from manifest)
-        assert!(namespace.table_exists(exists_req).await.is_err());
+        // Verify version 2 was created at the path returned in the response
+        let response = result.unwrap();
+        let version_info = response
+            .version
+            .expect("response should contain version info");
+        let version_2_path = Path::from(version_info.manifest_path);
+        let head_result = dataset.object_store().inner.head(&version_2_path).await;
+        assert!(
+            head_result.is_ok(),
+            "Version 2 manifest should exist at {}",
+            version_2_path
+        );
 
-        // Verify physical data still exists at the returned location
-        let dataset = Dataset::open(location).await;
+        // Verify the staging file has been deleted
+        let staging_head_result = dataset.object_store().inner.head(&staging_path).await;
         assert!(
-            dataset.is_ok(),
-            "Physical table data should still exist at {}",
-            location
+            staging_head_result.is_err(),
+            "Staging manifest should have been deleted after create_table_version"
         );
     }
 
     #[tokio::test]
-    async fn test_deregister_table_in_child_namespace() {
-        use lance_namespace::models::{
-            CreateNamespaceRequest, DeregisterTableRequest, TableExistsRequest,
-        };
-
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+    #[cfg(not(windows))]
+    async fn test_create_table_version_conflict() {
+        // create_table_version should fail if the version already exists.
+        // Each version always writes to a new file location.
+        use futures::TryStreamExt;
+        use lance::dataset::builder::DatasetBuilder;
+        use lance_namespace::models::CreateTableVersionRequest;
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .build()
-            .await
-            .unwrap();
+        let temp_dir = TempStrDir::default();
+        let temp_path: &str = &temp_dir;
 
-        // Create child namespace
-        let mut create_ns_req = CreateNamespaceRequest::new();
-        create_ns_req.id = Some(vec!["test_ns".to_string()]);
-        namespace.create_namespace(create_ns_req).await.unwrap();
+        let namespace: Arc<dyn LanceNamespace> = Arc::new(
+            DirectoryNamespaceBuilder::new(temp_path)
+                .table_version_tracking_enabled(true)
+                .build()
+                .await
+                .unwrap(),
+        );
 
-        // Create a table in the child namespace
+        // Create a table
         let schema = create_test_schema();
         let ipc_data = create_test_ipc_data(&schema);
-
         let mut create_req = CreateTableRequest::new();
-        create_req.id = Some(vec!["test_ns".to_string(), "test_table".to_string()]);
+        create_req.id = Some(vec!["test_table".to_string()]);
         namespace
             .create_table(create_req, bytes::Bytes::from(ipc_data))
             .await
             .unwrap();
 
-        // Deregister the table
-        let mut deregister_req = DeregisterTableRequest::new();
-        deregister_req.id = Some(vec!["test_ns".to_string(), "test_table".to_string()]);
-        let response = namespace.deregister_table(deregister_req).await.unwrap();
+        // Open the dataset using from_namespace to get proper object_store and paths
+        let table_id = vec!["test_table".to_string()];
+        let dataset = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone())
+            .await
+            .unwrap()
+            .load()
+            .await
+            .unwrap();
 
-        // Should return location and id in child namespace
+        // Use dataset's object_store to find and copy the manifest
+        let versions_path = dataset.versions_dir();
+        let manifest_metas: Vec<_> = dataset
+            .object_store()
+            .inner
+            .list(Some(&versions_path))
+            .try_collect()
+            .await
+            .unwrap();
+
+        let manifest_meta = manifest_metas
+            .iter()
+            .find(|m| {
+                m.location
+                    .filename()
+                    .map(|f| f.ends_with(".manifest"))
+                    .unwrap_or(false)
+            })
+            .expect("No manifest file found");
+
+        // Read the existing manifest data
+        let manifest_data = dataset
+            .object_store()
+            .inner
+            .get(&manifest_meta.location)
+            .await
+            .unwrap()
+            .bytes()
+            .await
+            .unwrap();
+
+        // Write to a staging location using the dataset's object_store
+        let staging_path = dataset.versions_dir().child("staging_manifest");
+        dataset
+            .object_store()
+            .inner
+            .put(&staging_path, manifest_data.into())
+            .await
+            .unwrap();
+
+        // First create version 2 (should succeed)
+        let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string());
+        create_version_req.id = Some(table_id.clone());
+        create_version_req.naming_scheme = Some("V2".to_string());
+        let first_result = namespace.create_table_version(create_version_req).await;
         assert!(
-            response.location.is_some(),
-            "Deregister should return location"
+            first_result.is_ok(),
+            "First create_table_version for version 2 should succeed: {:?}",
+            first_result
         );
-        let location = response.location.as_ref().unwrap();
-        // Location should be a proper file:// URI with the temp path
-        // Use uri_to_url to normalize the temp path to a URL for comparison
-        let expected_url = lance_io::object_store::uri_to_url(temp_path)
-            .expect("Failed to convert temp path to URL");
-        let expected_prefix = expected_url.to_string();
-        assert!(
-            location.starts_with(&expected_prefix),
-            "Location should start with '{}', got: {}",
-            expected_prefix,
-            location
+
+        // Get the path from the response for verification
+        let version_2_path = Path::from(
+            first_result
+                .unwrap()
+                .version
+                .expect("response should contain version info")
+                .manifest_path,
         );
+
+        // Create version 2 again (should fail - conflict)
+        let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string());
+        create_version_req.id = Some(table_id.clone());
+        create_version_req.naming_scheme = Some("V2".to_string());
+
+        let result = namespace.create_table_version(create_version_req).await;
         assert!(
-            location.contains("test_ns") && location.contains("test_table"),
-            "Location should contain namespace and table name: {}",
-            location
-        );
-        assert_eq!(
-            response.id,
-            Some(vec!["test_ns".to_string(), "test_table".to_string()])
+            result.is_err(),
+            "create_table_version should fail for existing version"
         );
 
-        // Verify table no longer exists
-        let mut exists_req = TableExistsRequest::new();
-        exists_req.id = Some(vec!["test_ns".to_string(), "test_table".to_string()]);
-        assert!(namespace.table_exists(exists_req).await.is_err());
+        // Verify version 2 still exists using the dataset's object_store
+        let head_result = dataset.object_store().inner.head(&version_2_path).await;
+        assert!(
+            head_result.is_ok(),
+            "Version 2 manifest should still exist at {}",
+            version_2_path
+        );
     }
 
     #[tokio::test]
-    async fn test_register_without_manifest_fails() {
-        use lance_namespace::models::RegisterTableRequest;
+    async fn test_create_table_version_table_not_found() {
+        use lance_namespace::models::CreateTableVersionRequest;
 
         let temp_dir = TempStdDir::default();
         let temp_path = temp_dir.to_str().unwrap();
 
-        // Create namespace without manifest
         let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(false)
+            .table_version_tracking_enabled(true)
             .build()
             .await
             .unwrap();
 
-        // Try to register - should fail (register requires manifest)
-        let mut register_req = RegisterTableRequest::new("test_table.lance".to_string());
-        register_req.id = Some(vec!["test_table".to_string()]);
-        let result = namespace.register_table(register_req).await;
-        assert!(result.is_err());
+        // Try to create version for non-existent table
+        let mut create_version_req =
+            CreateTableVersionRequest::new(1, "/some/staging/path".to_string());
+        create_version_req.id = Some(vec!["non_existent_table".to_string()]);
+
+        let result = namespace.create_table_version(create_version_req).await;
         assert!(
-            result
-                .unwrap_err()
-                .to_string()
-                .contains("manifest mode is enabled")
+            result.is_err(),
+            "create_table_version should fail for non-existent table"
+        );
+        let err_msg = result.unwrap_err().to_string();
+        assert!(
+            err_msg.contains("Table not found"),
+            "Error should mention table not found, got: {}",
+            err_msg
         );
-
-        // Note: deregister_table now works in V1 mode via .lance-deregistered marker files
-        // See test_deregister_table_v1_mode for that test case
     }
 
-    #[tokio::test]
-    async fn test_register_table_rejects_absolute_uri() {
-        use lance_namespace::models::RegisterTableRequest;
-
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+    /// End-to-end integration test module for table version tracking.
+    mod e2e_table_version_tracking {
+        use super::*;
+        use std::sync::atomic::{AtomicUsize, Ordering};
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .build()
-            .await
-            .unwrap();
+        /// Tracking wrapper around a namespace that counts method invocations.
+        struct TrackingNamespace {
+            inner: DirectoryNamespace,
+            create_table_version_count: AtomicUsize,
+            describe_table_version_count: AtomicUsize,
+            list_table_versions_count: AtomicUsize,
+        }
 
-        // Try to register with absolute URI - should fail
-        let mut register_req = RegisterTableRequest::new("s3://bucket/table.lance".to_string());
-        register_req.id = Some(vec!["test_table".to_string()]);
-        let result = namespace.register_table(register_req).await;
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("Absolute URIs are not allowed"));
-    }
+        impl TrackingNamespace {
+            fn new(inner: DirectoryNamespace) -> Self {
+                Self {
+                    inner,
+                    create_table_version_count: AtomicUsize::new(0),
+                    describe_table_version_count: AtomicUsize::new(0),
+                    list_table_versions_count: AtomicUsize::new(0),
+                }
+            }
 
-    #[tokio::test]
-    async fn test_register_table_rejects_absolute_path() {
-        use lance_namespace::models::RegisterTableRequest;
+            fn create_table_version_calls(&self) -> usize {
+                self.create_table_version_count.load(Ordering::SeqCst)
+            }
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+            fn describe_table_version_calls(&self) -> usize {
+                self.describe_table_version_count.load(Ordering::SeqCst)
+            }
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .build()
-            .await
-            .unwrap();
+            fn list_table_versions_calls(&self) -> usize {
+                self.list_table_versions_count.load(Ordering::SeqCst)
+            }
+        }
 
-        // Try to register with absolute path - should fail
-        let mut register_req = RegisterTableRequest::new("/tmp/table.lance".to_string());
-        register_req.id = Some(vec!["test_table".to_string()]);
-        let result = namespace.register_table(register_req).await;
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("Absolute paths are not allowed"));
-    }
+        impl std::fmt::Debug for TrackingNamespace {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                f.debug_struct("TrackingNamespace")
+                    .field(
+                        "create_table_version_calls",
+                        &self.create_table_version_calls(),
+                    )
+                    .finish()
+            }
+        }
 
-    #[tokio::test]
-    async fn test_register_table_rejects_path_traversal() {
-        use lance_namespace::models::RegisterTableRequest;
+        #[async_trait]
+        impl LanceNamespace for TrackingNamespace {
+            async fn create_namespace(
+                &self,
+                request: CreateNamespaceRequest,
+            ) -> Result<CreateNamespaceResponse> {
+                self.inner.create_namespace(request).await
+            }
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+            async fn describe_namespace(
+                &self,
+                request: DescribeNamespaceRequest,
+            ) -> Result<DescribeNamespaceResponse> {
+                self.inner.describe_namespace(request).await
+            }
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .build()
-            .await
-            .unwrap();
+            async fn namespace_exists(&self, request: NamespaceExistsRequest) -> Result<()> {
+                self.inner.namespace_exists(request).await
+            }
 
-        // Try to register with path traversal - should fail
-        let mut register_req = RegisterTableRequest::new("../outside/table.lance".to_string());
-        register_req.id = Some(vec!["test_table".to_string()]);
-        let result = namespace.register_table(register_req).await;
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("Path traversal is not allowed"));
-    }
+            async fn list_namespaces(
+                &self,
+                request: ListNamespacesRequest,
+            ) -> Result<ListNamespacesResponse> {
+                self.inner.list_namespaces(request).await
+            }
 
-    #[tokio::test]
-    async fn test_namespace_write() {
-        use arrow::array::Int32Array;
-        use arrow::datatypes::{DataType, Field as ArrowField, Schema as ArrowSchema};
-        use arrow::record_batch::{RecordBatch, RecordBatchIterator};
-        use lance::dataset::{Dataset, WriteMode, WriteParams};
-        use lance_namespace::LanceNamespace;
+            async fn drop_namespace(
+                &self,
+                request: DropNamespaceRequest,
+            ) -> Result<DropNamespaceResponse> {
+                self.inner.drop_namespace(request).await
+            }
 
-        let (namespace, _temp_dir) = create_test_namespace().await;
-        let namespace = Arc::new(namespace) as Arc<dyn LanceNamespace>;
+            async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
+                self.inner.list_tables(request).await
+            }
 
-        // Use child namespace instead of root
-        let table_id = vec!["test_ns".to_string(), "test_table".to_string()];
-        let schema = Arc::new(ArrowSchema::new(vec![
-            ArrowField::new("a", DataType::Int32, false),
-            ArrowField::new("b", DataType::Int32, false),
-        ]));
+            async fn describe_table(
+                &self,
+                request: DescribeTableRequest,
+            ) -> Result<DescribeTableResponse> {
+                self.inner.describe_table(request).await
+            }
 
-        // Test 1: CREATE mode
-        let data1 = RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(Int32Array::from(vec![1, 2, 3])),
-                Arc::new(Int32Array::from(vec![10, 20, 30])),
-            ],
-        )
-        .unwrap();
+            async fn table_exists(&self, request: TableExistsRequest) -> Result<()> {
+                self.inner.table_exists(request).await
+            }
 
-        let reader1 = RecordBatchIterator::new(vec![data1].into_iter().map(Ok), schema.clone());
-        let dataset =
-            Dataset::write_into_namespace(reader1, namespace.clone(), table_id.clone(), None)
-                .await
-                .unwrap();
+            async fn drop_table(&self, request: DropTableRequest) -> Result<DropTableResponse> {
+                self.inner.drop_table(request).await
+            }
 
-        assert_eq!(dataset.count_rows(None).await.unwrap(), 3);
-        assert_eq!(dataset.version().version, 1);
+            async fn create_table(
+                &self,
+                request: CreateTableRequest,
+                request_data: Bytes,
+            ) -> Result<CreateTableResponse> {
+                self.inner.create_table(request, request_data).await
+            }
 
-        // Test 2: APPEND mode
-        let data2 = RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(Int32Array::from(vec![4, 5])),
-                Arc::new(Int32Array::from(vec![40, 50])),
-            ],
-        )
-        .unwrap();
+            async fn declare_table(
+                &self,
+                request: DeclareTableRequest,
+            ) -> Result<DeclareTableResponse> {
+                self.inner.declare_table(request).await
+            }
 
-        let params_append = WriteParams {
-            mode: WriteMode::Append,
-            ..Default::default()
-        };
+            async fn list_table_versions(
+                &self,
+                request: ListTableVersionsRequest,
+            ) -> Result<ListTableVersionsResponse> {
+                self.list_table_versions_count
+                    .fetch_add(1, Ordering::SeqCst);
+                self.inner.list_table_versions(request).await
+            }
 
-        let reader2 = RecordBatchIterator::new(vec![data2].into_iter().map(Ok), schema.clone());
-        let dataset = Dataset::write_into_namespace(
-            reader2,
-            namespace.clone(),
-            table_id.clone(),
-            Some(params_append),
-        )
-        .await
-        .unwrap();
+            async fn create_table_version(
+                &self,
+                request: CreateTableVersionRequest,
+            ) -> Result<CreateTableVersionResponse> {
+                self.create_table_version_count
+                    .fetch_add(1, Ordering::SeqCst);
+                self.inner.create_table_version(request).await
+            }
 
-        assert_eq!(dataset.count_rows(None).await.unwrap(), 5);
-        assert_eq!(dataset.version().version, 2);
+            async fn describe_table_version(
+                &self,
+                request: DescribeTableVersionRequest,
+            ) -> Result<DescribeTableVersionResponse> {
+                self.describe_table_version_count
+                    .fetch_add(1, Ordering::SeqCst);
+                self.inner.describe_table_version(request).await
+            }
 
-        // Test 3: OVERWRITE mode
-        let data3 = RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(Int32Array::from(vec![100, 200])),
-                Arc::new(Int32Array::from(vec![1000, 2000])),
-            ],
-        )
-        .unwrap();
+            async fn batch_delete_table_versions(
+                &self,
+                request: BatchDeleteTableVersionsRequest,
+            ) -> Result<BatchDeleteTableVersionsResponse> {
+                self.inner.batch_delete_table_versions(request).await
+            }
 
-        let params_overwrite = WriteParams {
-            mode: WriteMode::Overwrite,
-            ..Default::default()
-        };
+            fn namespace_id(&self) -> String {
+                self.inner.namespace_id()
+            }
+        }
 
-        let reader3 = RecordBatchIterator::new(vec![data3].into_iter().map(Ok), schema.clone());
-        let dataset = Dataset::write_into_namespace(
-            reader3,
-            namespace.clone(),
-            table_id.clone(),
-            Some(params_overwrite),
-        )
-        .await
-        .unwrap();
+        #[tokio::test]
+        async fn test_describe_table_returns_managed_versioning() {
+            use lance_namespace::models::{CreateNamespaceRequest, DescribeTableRequest};
 
-        assert_eq!(dataset.count_rows(None).await.unwrap(), 2);
-        assert_eq!(dataset.version().version, 3);
+            let temp_dir = TempStdDir::default();
+            let temp_path = temp_dir.to_str().unwrap();
 
-        // Verify old data was replaced
-        let result = dataset.scan().try_into_batch().await.unwrap();
-        let a_col = result
-            .column_by_name("a")
-            .unwrap()
-            .as_any()
-            .downcast_ref::<Int32Array>()
-            .unwrap();
-        assert_eq!(a_col.values(), &[100, 200]);
-    }
+            // Create namespace with table_version_tracking_enabled and manifest_enabled
+            let ns = DirectoryNamespaceBuilder::new(temp_path)
+                .table_version_tracking_enabled(true)
+                .manifest_enabled(true)
+                .build()
+                .await
+                .unwrap();
 
-    // ============================================================
-    // Tests for declare_table
-    // ============================================================
+            // Create parent namespace
+            let mut create_ns_req = CreateNamespaceRequest::new();
+            create_ns_req.id = Some(vec!["workspace".to_string()]);
+            ns.create_namespace(create_ns_req).await.unwrap();
 
-    #[tokio::test]
-    async fn test_declare_table_v1_mode() {
-        use lance_namespace::models::{
-            DeclareTableRequest, DescribeTableRequest, TableExistsRequest,
-        };
+            // Create a table with multi-level ID (namespace + table)
+            let schema = create_test_schema();
+            let ipc_data = create_test_ipc_data(&schema);
+            let mut create_req = CreateTableRequest::new();
+            create_req.id = Some(vec!["workspace".to_string(), "test_table".to_string()]);
+            ns.create_table(create_req, bytes::Bytes::from(ipc_data))
+                .await
+                .unwrap();
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+            // Describe table should return managed_versioning=true
+            let mut describe_req = DescribeTableRequest::new();
+            describe_req.id = Some(vec!["workspace".to_string(), "test_table".to_string()]);
+            let describe_resp = ns.describe_table(describe_req).await.unwrap();
 
-        // Create namespace in V1 mode (no manifest)
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(false)
-            .build()
-            .await
-            .unwrap();
+            // managed_versioning should be true
+            assert_eq!(
+                describe_resp.managed_versioning,
+                Some(true),
+                "managed_versioning should be true when table_version_tracking_enabled=true"
+            );
+        }
 
-        // Declare a table
-        let mut declare_req = DeclareTableRequest::new();
-        declare_req.id = Some(vec!["test_table".to_string()]);
-        let response = namespace.declare_table(declare_req).await.unwrap();
+        #[tokio::test]
+        #[cfg(not(windows))]
+        async fn test_external_manifest_store_invokes_namespace_apis() {
+            use arrow::array::{Int32Array, StringArray};
+            use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+            use arrow::record_batch::RecordBatch;
+            use lance::Dataset;
+            use lance::dataset::builder::DatasetBuilder;
+            use lance::dataset::{WriteMode, WriteParams};
+            use lance_namespace::models::CreateNamespaceRequest;
 
-        // Should return location
-        assert!(response.location.is_some());
-        let location = response.location.as_ref().unwrap();
-        assert!(location.ends_with("test_table.lance"));
+            let temp_dir = TempStdDir::default();
+            let temp_path = temp_dir.to_str().unwrap();
 
-        // Table should exist (via reserved file)
-        let mut exists_req = TableExistsRequest::new();
-        exists_req.id = Some(vec!["test_table".to_string()]);
-        assert!(namespace.table_exists(exists_req).await.is_ok());
+            // Create namespace with table_version_tracking_enabled and manifest_enabled
+            let inner_ns = DirectoryNamespaceBuilder::new(temp_path)
+                .table_version_tracking_enabled(true)
+                .manifest_enabled(true)
+                .build()
+                .await
+                .unwrap();
 
-        // Describe should work but return no version/schema (not written yet)
-        let mut describe_req = DescribeTableRequest::new();
-        describe_req.id = Some(vec!["test_table".to_string()]);
-        let describe_response = namespace.describe_table(describe_req).await.unwrap();
-        assert!(describe_response.location.is_some());
-        assert!(describe_response.version.is_none()); // Not written yet
-        assert!(describe_response.schema.is_none()); // Not written yet
-    }
+            let tracking_ns = Arc::new(TrackingNamespace::new(inner_ns));
+            let ns: Arc<dyn LanceNamespace> = tracking_ns.clone();
 
-    #[tokio::test]
-    async fn test_declare_table_with_manifest() {
-        use lance_namespace::models::{DeclareTableRequest, TableExistsRequest};
+            // Create parent namespace
+            let mut create_ns_req = CreateNamespaceRequest::new();
+            create_ns_req.id = Some(vec!["workspace".to_string()]);
+            ns.create_namespace(create_ns_req).await.unwrap();
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+            // Create a table with multi-level ID (namespace + table)
+            let table_id = vec!["workspace".to_string(), "test_table".to_string()];
 
-        // Create namespace with manifest
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(true)
-            .dir_listing_enabled(false)
-            .build()
-            .await
+            // Create some initial data
+            let arrow_schema = Arc::new(ArrowSchema::new(vec![
+                Field::new("id", DataType::Int32, false),
+                Field::new("name", DataType::Utf8, true),
+            ]));
+            let batch = RecordBatch::try_new(
+                arrow_schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from(vec![1, 2, 3])),
+                    Arc::new(StringArray::from(vec!["a", "b", "c"])),
+                ],
+            )
             .unwrap();
 
-        // Declare a table
-        let mut declare_req = DeclareTableRequest::new();
-        declare_req.id = Some(vec!["test_table".to_string()]);
-        let response = namespace.declare_table(declare_req).await.unwrap();
+            // Create a table using write_into_namespace
+            let batches = RecordBatchIterator::new(vec![Ok(batch.clone())], arrow_schema.clone());
+            let write_params = WriteParams {
+                mode: WriteMode::Create,
+                ..Default::default()
+            };
+            let mut dataset = Dataset::write_into_namespace(
+                batches,
+                ns.clone(),
+                table_id.clone(),
+                Some(write_params),
+            )
+            .await
+            .unwrap();
+            assert_eq!(dataset.version().version, 1);
 
-        // Should return location
-        assert!(response.location.is_some());
+            // Verify create_table_version was called once during initial write_into_namespace
+            assert_eq!(
+                tracking_ns.create_table_version_calls(),
+                1,
+                "create_table_version should have been called once during initial write_into_namespace"
+            );
 
-        // Table should exist in manifest
-        let mut exists_req = TableExistsRequest::new();
-        exists_req.id = Some(vec!["test_table".to_string()]);
-        assert!(namespace.table_exists(exists_req).await.is_ok());
-    }
+            // Append data - this should call create_table_version again
+            let append_batch = RecordBatch::try_new(
+                arrow_schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from(vec![4, 5, 6])),
+                    Arc::new(StringArray::from(vec!["d", "e", "f"])),
+                ],
+            )
+            .unwrap();
+            let append_batches = RecordBatchIterator::new(vec![Ok(append_batch)], arrow_schema);
+            dataset.append(append_batches, None).await.unwrap();
 
-    #[tokio::test]
-    async fn test_declare_table_when_table_exists() {
-        use lance_namespace::models::DeclareTableRequest;
+            assert_eq!(
+                tracking_ns.create_table_version_calls(),
+                2,
+                "create_table_version should have been called twice (once for create, once for append)"
+            );
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+            // checkout_latest should call list_table_versions exactly once
+            let initial_list_calls = tracking_ns.list_table_versions_calls();
+            let latest_dataset = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
+                .await
+                .unwrap()
+                .load()
+                .await
+                .unwrap();
+            assert_eq!(latest_dataset.version().version, 2);
+            assert_eq!(
+                tracking_ns.list_table_versions_calls(),
+                initial_list_calls + 1,
+                "list_table_versions should have been called exactly once during checkout_latest"
+            );
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(false)
-            .build()
-            .await
-            .unwrap();
+            // checkout to specific version should call describe_table_version exactly once
+            let initial_describe_calls = tracking_ns.describe_table_version_calls();
+            let v1_dataset = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
+                .await
+                .unwrap()
+                .with_version(1)
+                .load()
+                .await
+                .unwrap();
+            assert_eq!(v1_dataset.version().version, 1);
+            assert_eq!(
+                tracking_ns.describe_table_version_calls(),
+                initial_describe_calls + 1,
+                "describe_table_version should have been called exactly once during checkout to version 1"
+            );
+        }
 
-        // First create a table with actual data
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-        let mut create_req = CreateTableRequest::new();
-        create_req.id = Some(vec!["test_table".to_string()]);
-        namespace
-            .create_table(create_req, bytes::Bytes::from(ipc_data))
-            .await
-            .unwrap();
+        #[tokio::test]
+        #[cfg(not(windows))]
+        async fn test_dataset_commit_with_external_manifest_store() {
+            use arrow::array::{Int32Array, StringArray};
+            use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+            use arrow::record_batch::RecordBatch;
+            use futures::TryStreamExt;
+            use lance::dataset::{Dataset, WriteMode, WriteParams};
+            use lance_namespace::models::CreateNamespaceRequest;
+            use lance_table::io::commit::ManifestNamingScheme;
 
-        // Try to declare the same table - should fail because it already has data
-        let mut declare_req = DeclareTableRequest::new();
-        declare_req.id = Some(vec!["test_table".to_string()]);
-        let result = namespace.declare_table(declare_req).await;
-        assert!(result.is_err());
-    }
+            let temp_dir = TempStdDir::default();
+            let temp_path = temp_dir.to_str().unwrap();
 
-    // ============================================================
-    // Tests for deregister_table in V1 mode
-    // ============================================================
+            // Create namespace with table_version_tracking_enabled and manifest_enabled
+            let inner_ns = DirectoryNamespaceBuilder::new(temp_path)
+                .table_version_tracking_enabled(true)
+                .manifest_enabled(true)
+                .build()
+                .await
+                .unwrap();
 
-    #[tokio::test]
-    async fn test_deregister_table_v1_mode() {
-        use lance_namespace::models::{DeregisterTableRequest, TableExistsRequest};
+            let tracking_ns: Arc<dyn LanceNamespace> = Arc::new(TrackingNamespace::new(inner_ns));
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+            // Create parent namespace
+            let mut create_ns_req = CreateNamespaceRequest::new();
+            create_ns_req.id = Some(vec!["workspace".to_string()]);
+            tracking_ns.create_namespace(create_ns_req).await.unwrap();
 
-        // Create namespace in V1 mode (no manifest, with dir listing)
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(false)
-            .dir_listing_enabled(true)
-            .build()
-            .await
+            // Create a table using write_into_namespace
+            let table_id = vec!["workspace".to_string(), "test_table".to_string()];
+            let arrow_schema = Arc::new(ArrowSchema::new(vec![
+                Field::new("id", DataType::Int32, false),
+                Field::new("name", DataType::Utf8, true),
+            ]));
+            let batch = RecordBatch::try_new(
+                arrow_schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from(vec![1, 2, 3])),
+                    Arc::new(StringArray::from(vec!["a", "b", "c"])),
+                ],
+            )
             .unwrap();
-
-        // Create a table with data
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-        let mut create_req = CreateTableRequest::new();
-        create_req.id = Some(vec!["test_table".to_string()]);
-        namespace
-            .create_table(create_req, bytes::Bytes::from(ipc_data))
+            let batches = RecordBatchIterator::new(vec![Ok(batch)], arrow_schema.clone());
+            let write_params = WriteParams {
+                mode: WriteMode::Create,
+                ..Default::default()
+            };
+            let dataset = Dataset::write_into_namespace(
+                batches,
+                tracking_ns.clone(),
+                table_id.clone(),
+                Some(write_params),
+            )
             .await
             .unwrap();
+            assert_eq!(dataset.version().version, 1);
 
-        // Verify table exists
-        let mut exists_req = TableExistsRequest::new();
-        exists_req.id = Some(vec!["test_table".to_string()]);
-        assert!(namespace.table_exists(exists_req.clone()).await.is_ok());
-
-        // Deregister the table
-        let mut deregister_req = DeregisterTableRequest::new();
-        deregister_req.id = Some(vec!["test_table".to_string()]);
-        let response = namespace.deregister_table(deregister_req).await.unwrap();
-
-        // Should return location
-        assert!(response.location.is_some());
-        let location = response.location.as_ref().unwrap();
-        assert!(location.contains("test_table"));
+            // Append data using write_into_namespace (APPEND mode)
+            let batch2 = RecordBatch::try_new(
+                arrow_schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from(vec![4, 5, 6])),
+                    Arc::new(StringArray::from(vec!["d", "e", "f"])),
+                ],
+            )
+            .unwrap();
+            let batches = RecordBatchIterator::new(vec![Ok(batch2)], arrow_schema);
+            let write_params = WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            };
+            Dataset::write_into_namespace(
+                batches,
+                tracking_ns.clone(),
+                table_id.clone(),
+                Some(write_params),
+            )
+            .await
+            .unwrap();
 
-        // Table should no longer exist (deregistered)
-        let result = namespace.table_exists(exists_req).await;
-        assert!(result.is_err());
-        assert!(result.unwrap_err().to_string().contains("deregistered"));
+            // Verify version 2 was created using the dataset's object_store
+            // List manifests in the versions directory to find the V2 named manifest
+            let manifest_metas: Vec<_> = dataset
+                .object_store()
+                .inner
+                .list(Some(&dataset.versions_dir()))
+                .try_collect()
+                .await
+                .unwrap();
+            let version_2_found = manifest_metas.iter().any(|m| {
+                m.location
+                    .filename()
+                    .map(|f| {
+                        f.ends_with(".manifest")
+                            && ManifestNamingScheme::V2.parse_version(f) == Some(2)
+                    })
+                    .unwrap_or(false)
+            });
+            assert!(
+                version_2_found,
+                "Version 2 manifest should exist in versions directory"
+            );
+        }
 
-        // Physical data should still exist
-        let dataset = Dataset::open(location).await;
-        assert!(dataset.is_ok(), "Physical table data should still exist");
-    }
+        /// Helper: create a namespace and a table with some rows, returning (namespace, table_id)
+        async fn create_ns_with_table() -> (DirectoryNamespace, TempStdDir, Vec<String>) {
+            use arrow::array::{Int32Array, StringArray};
+            use arrow::ipc::writer::StreamWriter;
 
-    #[tokio::test]
-    async fn test_deregister_table_v1_already_deregistered() {
-        use lance_namespace::models::DeregisterTableRequest;
+            let (namespace, temp_dir) = create_test_namespace().await;
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+            let schema = create_test_schema();
+            let arrow_schema = convert_json_arrow_schema(&schema).unwrap();
+            let arrow_schema = Arc::new(arrow_schema);
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(false)
-            .dir_listing_enabled(true)
-            .build()
-            .await
+            let id_array = Int32Array::from(vec![1, 2, 3]);
+            let name_array = StringArray::from(vec!["Alice", "Bob", "Charlie"]);
+            let batch = arrow::record_batch::RecordBatch::try_new(
+                arrow_schema.clone(),
+                vec![Arc::new(id_array), Arc::new(name_array)],
+            )
             .unwrap();
 
-        // Create a table
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-        let mut create_req = CreateTableRequest::new();
-        create_req.id = Some(vec!["test_table".to_string()]);
-        namespace
-            .create_table(create_req, bytes::Bytes::from(ipc_data))
-            .await
-            .unwrap();
+            let mut buffer = Vec::new();
+            {
+                let mut writer = StreamWriter::try_new(&mut buffer, &arrow_schema).unwrap();
+                writer.write(&batch).unwrap();
+                writer.finish().unwrap();
+            }
 
-        // Deregister once
-        let mut deregister_req = DeregisterTableRequest::new();
-        deregister_req.id = Some(vec!["test_table".to_string()]);
-        namespace
-            .deregister_table(deregister_req.clone())
-            .await
-            .unwrap();
+            let mut request = CreateTableRequest::new();
+            let table_id = vec!["test_ops_table".to_string()];
+            request.id = Some(table_id.clone());
 
-        // Try to deregister again - should fail
-        let result = namespace.deregister_table(deregister_req).await;
-        assert!(result.is_err());
-        assert!(
-            result
-                .unwrap_err()
-                .to_string()
-                .contains("already deregistered")
-        );
-    }
+            namespace
+                .create_table(request, Bytes::from(buffer))
+                .await
+                .unwrap();
 
-    // ============================================================
-    // Tests for list_tables skipping deregistered tables
-    // ============================================================
+            (namespace, temp_dir, table_id)
+        }
 
-    #[tokio::test]
-    async fn test_list_tables_skips_deregistered_v1() {
-        use lance_namespace::models::DeregisterTableRequest;
+        #[tokio::test]
+        async fn test_count_table_rows_basic() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+            let request = CountTableRowsRequest {
+                id: Some(table_id),
+                version: None,
+                predicate: None,
+                ..Default::default()
+            };
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(false)
-            .dir_listing_enabled(true)
-            .build()
-            .await
-            .unwrap();
+            let count = namespace.count_table_rows(request).await.unwrap();
+            assert_eq!(count, 3);
+        }
 
-        // Create two tables
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
+        #[tokio::test]
+        async fn test_count_table_rows_with_predicate() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
 
-        let mut create_req1 = CreateTableRequest::new();
-        create_req1.id = Some(vec!["table1".to_string()]);
-        namespace
-            .create_table(create_req1, bytes::Bytes::from(ipc_data.clone()))
-            .await
-            .unwrap();
+            let request = CountTableRowsRequest {
+                id: Some(table_id),
+                version: None,
+                predicate: Some("id > 1".to_string()),
+                ..Default::default()
+            };
 
-        let mut create_req2 = CreateTableRequest::new();
-        create_req2.id = Some(vec!["table2".to_string()]);
-        namespace
-            .create_table(create_req2, bytes::Bytes::from(ipc_data))
-            .await
-            .unwrap();
+            let count = namespace.count_table_rows(request).await.unwrap();
+            assert_eq!(count, 2);
+        }
 
-        // List tables - should see both (root namespace = empty vec)
-        let mut list_req = ListTablesRequest::new();
-        list_req.id = Some(vec![]);
-        let list_response = namespace.list_tables(list_req.clone()).await.unwrap();
-        assert_eq!(list_response.tables.len(), 2);
+        #[tokio::test]
+        async fn test_query_table_invalid_distance_type() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_vector_table().await;
 
-        // Deregister table1
-        let mut deregister_req = DeregisterTableRequest::new();
-        deregister_req.id = Some(vec!["table1".to_string()]);
-        namespace.deregister_table(deregister_req).await.unwrap();
+            let vector = Box::new(lance_namespace::models::QueryTableRequestVector {
+                single_vector: Some(vec![1.0, 0.0, 0.0, 0.0]),
+                multi_vector: None,
+            });
 
-        // List tables - should only see table2
-        let list_response = namespace.list_tables(list_req).await.unwrap();
-        assert_eq!(list_response.tables.len(), 1);
-        assert!(list_response.tables.contains(&"table2".to_string()));
-        assert!(!list_response.tables.contains(&"table1".to_string()));
-    }
+            let request = QueryTableRequest {
+                id: Some(table_id),
+                k: 2,
+                vector,
+                vector_column: Some("vector".to_string()),
+                distance_type: Some("invalid_metric".to_string()),
+                filter: None,
+                offset: None,
+                version: None,
+                ..Default::default()
+            };
 
-    // ============================================================
-    // Tests for describe_table and table_exists with deregistered tables
-    // ============================================================
+            let result = namespace.query_table(request).await;
+            assert!(result.is_err());
+            let err_msg = result.unwrap_err().to_string();
+            assert!(
+                err_msg.contains("Unknown distance type"),
+                "Expected error about unknown distance type, got: {}",
+                err_msg
+            );
+        }
 
-    #[tokio::test]
-    async fn test_describe_table_fails_for_deregistered_v1() {
-        use lance_namespace::models::{DeregisterTableRequest, DescribeTableRequest};
+        #[tokio::test]
+        async fn test_insert_into_table_append() {
+            use arrow::array::{Int32Array, StringArray};
+            use arrow::ipc::writer::StreamWriter;
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(false)
-            .dir_listing_enabled(true)
-            .build()
-            .await
-            .unwrap();
+            // Prepare new data to insert
+            let schema = create_test_schema();
+            let arrow_schema = convert_json_arrow_schema(&schema).unwrap();
+            let arrow_schema = Arc::new(arrow_schema);
 
-        // Create a table
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-        let mut create_req = CreateTableRequest::new();
-        create_req.id = Some(vec!["test_table".to_string()]);
-        namespace
-            .create_table(create_req, bytes::Bytes::from(ipc_data))
-            .await
+            let id_array = Int32Array::from(vec![4, 5]);
+            let name_array = StringArray::from(vec!["Dave", "Eve"]);
+            let batch = arrow::record_batch::RecordBatch::try_new(
+                arrow_schema.clone(),
+                vec![Arc::new(id_array), Arc::new(name_array)],
+            )
             .unwrap();
 
-        // Describe should work before deregistration
-        let mut describe_req = DescribeTableRequest::new();
-        describe_req.id = Some(vec!["test_table".to_string()]);
-        assert!(namespace.describe_table(describe_req.clone()).await.is_ok());
+            let mut buffer = Vec::new();
+            {
+                let mut writer = StreamWriter::try_new(&mut buffer, &arrow_schema).unwrap();
+                writer.write(&batch).unwrap();
+                writer.finish().unwrap();
+            }
 
-        // Deregister
-        let mut deregister_req = DeregisterTableRequest::new();
-        deregister_req.id = Some(vec!["test_table".to_string()]);
-        namespace.deregister_table(deregister_req).await.unwrap();
+            let request = InsertIntoTableRequest {
+                id: Some(table_id.clone()),
+                mode: Some("append".to_string()),
+                ..Default::default()
+            };
 
-        // Describe should fail after deregistration
-        let result = namespace.describe_table(describe_req).await;
-        assert!(result.is_err());
-        assert!(result.unwrap_err().to_string().contains("deregistered"));
-    }
+            let response = namespace
+                .insert_into_table(request, Bytes::from(buffer))
+                .await
+                .unwrap();
+            assert!(response.transaction_id.is_none());
 
-    #[tokio::test]
-    async fn test_table_exists_fails_for_deregistered_v1() {
-        use lance_namespace::models::{DeregisterTableRequest, TableExistsRequest};
+            // Verify total rows
+            let count_req = CountTableRowsRequest {
+                id: Some(table_id),
+                version: None,
+                predicate: None,
+                ..Default::default()
+            };
+            let count = namespace.count_table_rows(count_req).await.unwrap();
+            assert_eq!(count, 5);
+        }
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+        #[tokio::test]
+        async fn test_insert_into_table_overwrite() {
+            use arrow::array::{Int32Array, StringArray};
+            use arrow::ipc::writer::StreamWriter;
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(false)
-            .dir_listing_enabled(true)
-            .build()
-            .await
-            .unwrap();
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
 
-        // Create a table
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-        let mut create_req = CreateTableRequest::new();
-        create_req.id = Some(vec!["test_table".to_string()]);
-        namespace
-            .create_table(create_req, bytes::Bytes::from(ipc_data))
-            .await
+            let schema = create_test_schema();
+            let arrow_schema = convert_json_arrow_schema(&schema).unwrap();
+            let arrow_schema = Arc::new(arrow_schema);
+
+            let id_array = Int32Array::from(vec![10, 20]);
+            let name_array = StringArray::from(vec!["X", "Y"]);
+            let batch = arrow::record_batch::RecordBatch::try_new(
+                arrow_schema.clone(),
+                vec![Arc::new(id_array), Arc::new(name_array)],
+            )
             .unwrap();
 
-        // Table exists should work before deregistration
-        let mut exists_req = TableExistsRequest::new();
-        exists_req.id = Some(vec!["test_table".to_string()]);
-        assert!(namespace.table_exists(exists_req.clone()).await.is_ok());
+            let mut buffer = Vec::new();
+            {
+                let mut writer = StreamWriter::try_new(&mut buffer, &arrow_schema).unwrap();
+                writer.write(&batch).unwrap();
+                writer.finish().unwrap();
+            }
+
+            let request = InsertIntoTableRequest {
+                id: Some(table_id.clone()),
+                mode: Some("overwrite".to_string()),
+                ..Default::default()
+            };
 
-        // Deregister
-        let mut deregister_req = DeregisterTableRequest::new();
-        deregister_req.id = Some(vec!["test_table".to_string()]);
-        namespace.deregister_table(deregister_req).await.unwrap();
+            namespace
+                .insert_into_table(request, Bytes::from(buffer))
+                .await
+                .unwrap();
 
-        // Table exists should fail after deregistration
-        let result = namespace.table_exists(exists_req).await;
-        assert!(result.is_err());
-        assert!(result.unwrap_err().to_string().contains("deregistered"));
-    }
+            // Verify overwrite: only 2 rows remain
+            let count_req = CountTableRowsRequest {
+                id: Some(table_id),
+                version: None,
+                predicate: None,
+                ..Default::default()
+            };
+            let count = namespace.count_table_rows(count_req).await.unwrap();
+            assert_eq!(count, 2);
+        }
 
-    #[tokio::test]
-    async fn test_atomic_table_status_check() {
-        // This test verifies that the TableStatus check is atomic
-        // by ensuring a single directory listing is used
+        #[tokio::test]
+        async fn test_insert_into_table_empty_data() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+            let request = InsertIntoTableRequest {
+                id: Some(table_id),
+                mode: None,
+                ..Default::default()
+            };
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .manifest_enabled(false)
-            .dir_listing_enabled(true)
-            .build()
-            .await
-            .unwrap();
+            let result = namespace.insert_into_table(request, Bytes::new()).await;
+            assert!(result.is_err());
+            assert!(
+                result
+                    .unwrap_err()
+                    .to_string()
+                    .contains("Arrow IPC stream) is required")
+            );
+        }
 
-        // Create a table
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-        let mut create_req = CreateTableRequest::new();
-        create_req.id = Some(vec!["test_table".to_string()]);
-        namespace
-            .create_table(create_req, bytes::Bytes::from(ipc_data))
-            .await
-            .unwrap();
+        #[tokio::test]
+        async fn test_insert_into_table_with_storage_options() {
+            use arrow::array::{Int32Array, StringArray};
+            use arrow::ipc::writer::StreamWriter;
 
-        // Table status should show exists=true, is_deregistered=false
-        let status = namespace.check_table_status("test_table").await;
-        assert!(status.exists);
-        assert!(!status.is_deregistered);
-        assert!(!status.has_reserved_file);
-    }
+            let temp_dir = TempStdDir::default();
 
-    #[tokio::test]
-    async fn test_table_version_tracking_enabled_managed_versioning() {
-        use lance_namespace::models::DescribeTableRequest;
+            // Build namespace with a (no-op) storage option so self.storage_options is Some
+            let namespace = DirectoryNamespaceBuilder::new(temp_dir.to_str().unwrap())
+                .storage_option("allow_http", "true")
+                .build()
+                .await
+                .unwrap();
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+            // Create a table first
+            let schema = create_test_schema();
+            let ipc_data = create_test_ipc_data(&schema);
+            let mut create_req = CreateTableRequest::new();
+            let table_id = vec!["so_table".to_string()];
+            create_req.id = Some(table_id.clone());
+            namespace
+                .create_table(create_req, Bytes::from(ipc_data))
+                .await
+                .unwrap();
 
-        // Create namespace with table_version_tracking_enabled=true
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .table_version_tracking_enabled(true)
-            .build()
-            .await
-            .unwrap();
+            // Insert with storage_options present — covers store_params closure
+            let arrow_schema = convert_json_arrow_schema(&schema).unwrap();
+            let arrow_schema = Arc::new(arrow_schema);
 
-        // Create a table
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-        let mut create_req = CreateTableRequest::new();
-        create_req.id = Some(vec!["test_table".to_string()]);
-        namespace
-            .create_table(create_req, bytes::Bytes::from(ipc_data))
-            .await
+            let id_array = Int32Array::from(vec![10, 20]);
+            let name_array = StringArray::from(vec!["X", "Y"]);
+            let batch = arrow::record_batch::RecordBatch::try_new(
+                arrow_schema.clone(),
+                vec![Arc::new(id_array), Arc::new(name_array)],
+            )
             .unwrap();
 
-        // Describe table should return managed_versioning=true
-        let mut describe_req = DescribeTableRequest::new();
-        describe_req.id = Some(vec!["test_table".to_string()]);
-        let describe_resp = namespace.describe_table(describe_req).await.unwrap();
+            let mut buffer = Vec::new();
+            {
+                let mut writer = StreamWriter::try_new(&mut buffer, &arrow_schema).unwrap();
+                writer.write(&batch).unwrap();
+                writer.finish().unwrap();
+            }
 
-        // managed_versioning should be true
-        assert_eq!(
-            describe_resp.managed_versioning,
-            Some(true),
-            "managed_versioning should be true when table_version_tracking_enabled=true"
-        );
-    }
+            let request = InsertIntoTableRequest {
+                id: Some(table_id.clone()),
+                mode: Some("append".to_string()),
+                ..Default::default()
+            };
 
-    #[tokio::test]
-    async fn test_table_version_tracking_disabled_no_managed_versioning() {
-        use lance_namespace::models::DescribeTableRequest;
+            let response = namespace
+                .insert_into_table(request, Bytes::from(buffer))
+                .await
+                .unwrap();
+            assert!(response.transaction_id.is_none());
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+            // Verify rows were inserted
+            let count_req = CountTableRowsRequest {
+                id: Some(table_id),
+                version: None,
+                predicate: None,
+                ..Default::default()
+            };
+            let count = namespace.count_table_rows(count_req).await.unwrap();
+            assert_eq!(count, 2);
+        }
 
-        // Create namespace with table_version_tracking_enabled=false (default)
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .table_version_tracking_enabled(false)
-            .build()
-            .await
-            .unwrap();
+        #[tokio::test]
+        async fn test_query_table_basic() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
+
+            let request = QueryTableRequest {
+                id: Some(table_id),
+                k: 10,
+                filter: None,
+                offset: None,
+                version: None,
+                ..Default::default()
+            };
 
-        // Create a table
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-        let mut create_req = CreateTableRequest::new();
-        create_req.id = Some(vec!["test_table".to_string()]);
-        namespace
-            .create_table(create_req, bytes::Bytes::from(ipc_data))
-            .await
-            .unwrap();
+            let bytes = namespace.query_table(request).await.unwrap();
 
-        // Describe table should not have managed_versioning set
-        let mut describe_req = DescribeTableRequest::new();
-        describe_req.id = Some(vec!["test_table".to_string()]);
-        let describe_resp = namespace.describe_table(describe_req).await.unwrap();
+            // Decode IPC and verify
+            let cursor = Cursor::new(bytes.to_vec());
+            let reader = FileReader::try_new(cursor, None).unwrap();
+            let batches: Vec<_> = reader.into_iter().map(|b| b.unwrap()).collect();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert_eq!(total_rows, 3);
+        }
 
-        // managed_versioning should be None when table_version_tracking_enabled=false
-        assert!(
-            describe_resp.managed_versioning.is_none(),
-            "managed_versioning should be None when table_version_tracking_enabled=false, got: {:?}",
-            describe_resp.managed_versioning
-        );
-    }
+        #[tokio::test]
+        async fn test_query_table_with_filter() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
+
+            let request = QueryTableRequest {
+                id: Some(table_id),
+                k: 10,
+                filter: Some("id <= 2".to_string()),
+                offset: None,
+                version: None,
+                ..Default::default()
+            };
 
-    #[tokio::test]
-    #[cfg(not(windows))]
-    async fn test_list_table_versions() {
-        use arrow::array::{Int32Array, RecordBatchIterator};
-        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
-        use arrow::record_batch::RecordBatch;
-        use lance::dataset::{Dataset, WriteMode, WriteParams};
-        use lance_namespace::models::{CreateNamespaceRequest, ListTableVersionsRequest};
+            let bytes = namespace.query_table(request).await.unwrap();
 
-        let temp_dir = TempStrDir::default();
-        let temp_path: &str = &temp_dir;
+            let cursor = Cursor::new(bytes.to_vec());
+            let reader = FileReader::try_new(cursor, None).unwrap();
+            let batches: Vec<_> = reader.into_iter().map(|b| b.unwrap()).collect();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert_eq!(total_rows, 2);
+        }
 
-        let namespace: Arc<dyn LanceNamespace> = Arc::new(
-            DirectoryNamespaceBuilder::new(temp_path)
-                .table_version_tracking_enabled(true)
-                .build()
-                .await
-                .unwrap(),
-        );
+        #[tokio::test]
+        async fn test_query_table_with_limit_and_offset() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
+
+            let request = QueryTableRequest {
+                id: Some(table_id),
+                k: 2,
+                filter: None,
+                offset: Some(1),
+                version: None,
+                ..Default::default()
+            };
 
-        // Create parent namespace first
-        let mut create_ns_req = CreateNamespaceRequest::new();
-        create_ns_req.id = Some(vec!["workspace".to_string()]);
-        namespace.create_namespace(create_ns_req).await.unwrap();
+            let bytes = namespace.query_table(request).await.unwrap();
 
-        // Create a table using write_into_namespace (version 1)
-        let table_id = vec!["workspace".to_string(), "test_table".to_string()];
-        let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
-            "id",
-            DataType::Int32,
-            false,
-        )]));
-        let batch = RecordBatch::try_new(
-            arrow_schema.clone(),
-            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
-        )
-        .unwrap();
-        let batches = RecordBatchIterator::new(vec![Ok(batch.clone())], arrow_schema.clone());
-        let write_params = WriteParams {
-            mode: WriteMode::Create,
-            ..Default::default()
-        };
-        let mut dataset = Dataset::write_into_namespace(
-            batches,
-            namespace.clone(),
-            table_id.clone(),
-            Some(write_params),
-        )
-        .await
-        .unwrap();
+            let cursor = Cursor::new(bytes.to_vec());
+            let reader = FileReader::try_new(cursor, None).unwrap();
+            let batches: Vec<_> = reader.into_iter().map(|b| b.unwrap()).collect();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert_eq!(total_rows, 2);
+        }
 
-        // Append to create version 2
-        let batch2 = RecordBatch::try_new(
-            arrow_schema.clone(),
-            vec![Arc::new(Int32Array::from(vec![100, 200]))],
-        )
-        .unwrap();
-        let batches = RecordBatchIterator::new(vec![Ok(batch2)], arrow_schema.clone());
-        dataset.append(batches, None).await.unwrap();
+        #[tokio::test]
+        async fn test_query_table_no_limit() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
+
+            // k=0 means no limit
+            let request = QueryTableRequest {
+                id: Some(table_id),
+                k: 0,
+                filter: None,
+                offset: None,
+                version: None,
+                ..Default::default()
+            };
 
-        // Append to create version 3
-        let batch3 = RecordBatch::try_new(
-            arrow_schema.clone(),
-            vec![Arc::new(Int32Array::from(vec![300, 400]))],
-        )
-        .unwrap();
-        let batches = RecordBatchIterator::new(vec![Ok(batch3)], arrow_schema);
-        dataset.append(batches, None).await.unwrap();
+            let bytes = namespace.query_table(request).await.unwrap();
 
-        // List versions - should have versions 1, 2, and 3
-        let mut list_req = ListTableVersionsRequest::new();
-        list_req.id = Some(table_id.clone());
-        let list_resp = namespace.list_table_versions(list_req).await.unwrap();
+            let cursor = Cursor::new(bytes.to_vec());
+            let reader = FileReader::try_new(cursor, None).unwrap();
+            let batches: Vec<_> = reader.into_iter().map(|b| b.unwrap()).collect();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert_eq!(total_rows, 3);
+        }
 
-        assert_eq!(
-            list_resp.versions.len(),
-            3,
-            "Should have 3 versions, got: {:?}",
-            list_resp.versions
-        );
+        #[tokio::test]
+        async fn test_query_table_with_columns() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
 
-        // Verify each version
-        for expected_version in 1..=3 {
-            let version = list_resp
-                .versions
-                .iter()
-                .find(|v| v.version == expected_version)
-                .unwrap_or_else(|| panic!("Expected version {}", expected_version));
+            let columns = Box::new(lance_namespace::models::QueryTableRequestColumns {
+                column_names: Some(vec!["id".to_string()]),
+                column_aliases: None,
+            });
 
-            assert!(
-                !version.manifest_path.is_empty(),
-                "manifest_path should be set for version {}",
-                expected_version
-            );
-            assert!(
-                version.manifest_path.contains(".manifest"),
-                "manifest_path should contain .manifest for version {}",
-                expected_version
-            );
-            assert!(
-                version.manifest_size.is_some(),
-                "manifest_size should be set for version {}",
-                expected_version
-            );
-            assert!(
-                version.manifest_size.unwrap() > 0,
-                "manifest_size should be > 0 for version {}",
-                expected_version
-            );
-            assert!(
-                version.timestamp_millis.is_some(),
-                "timestamp_millis should be set for version {}",
-                expected_version
-            );
+            let request = QueryTableRequest {
+                id: Some(table_id),
+                k: 10,
+                filter: None,
+                offset: None,
+                version: None,
+                columns: Some(columns),
+                ..Default::default()
+            };
+
+            let bytes = namespace.query_table(request).await.unwrap();
+
+            let cursor = Cursor::new(bytes.to_vec());
+            let reader = FileReader::try_new(cursor, None).unwrap();
+            let schema = reader.schema();
+            assert_eq!(schema.fields().len(), 1);
+            assert_eq!(schema.field(0).name(), "id");
+            let batches: Vec<_> = reader.into_iter().map(|b| b.unwrap()).collect();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert_eq!(total_rows, 3);
         }
-    }
 
-    #[tokio::test]
-    #[cfg(not(windows))]
-    async fn test_describe_table_version() {
-        use arrow::array::{Int32Array, RecordBatchIterator};
-        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
-        use arrow::record_batch::RecordBatch;
-        use lance::dataset::{Dataset, WriteMode, WriteParams};
-        use lance_namespace::models::{CreateNamespaceRequest, DescribeTableVersionRequest};
+        #[tokio::test]
+        async fn test_count_table_rows_with_version() {
+            use arrow::array::{Int32Array, StringArray};
+            use arrow::ipc::writer::StreamWriter;
 
-        let temp_dir = TempStrDir::default();
-        let temp_path: &str = &temp_dir;
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
 
-        let namespace: Arc<dyn LanceNamespace> = Arc::new(
-            DirectoryNamespaceBuilder::new(temp_path)
-                .table_version_tracking_enabled(true)
-                .build()
-                .await
-                .unwrap(),
-        );
+            // Insert more data to create version 2
+            let schema = create_test_schema();
+            let arrow_schema = convert_json_arrow_schema(&schema).unwrap();
+            let arrow_schema = Arc::new(arrow_schema);
 
-        // Create parent namespace first
-        let mut create_ns_req = CreateNamespaceRequest::new();
-        create_ns_req.id = Some(vec!["workspace".to_string()]);
-        namespace.create_namespace(create_ns_req).await.unwrap();
+            let id_array = Int32Array::from(vec![4, 5]);
+            let name_array = StringArray::from(vec!["Dave", "Eve"]);
+            let batch = arrow::record_batch::RecordBatch::try_new(
+                arrow_schema.clone(),
+                vec![Arc::new(id_array), Arc::new(name_array)],
+            )
+            .unwrap();
 
-        // Create a table using write_into_namespace (version 1)
-        let table_id = vec!["workspace".to_string(), "test_table".to_string()];
-        let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
-            "id",
-            DataType::Int32,
-            false,
-        )]));
-        let batch = RecordBatch::try_new(
-            arrow_schema.clone(),
-            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
-        )
-        .unwrap();
-        let batches = RecordBatchIterator::new(vec![Ok(batch)], arrow_schema.clone());
-        let write_params = WriteParams {
-            mode: WriteMode::Create,
-            ..Default::default()
-        };
-        let mut dataset = Dataset::write_into_namespace(
-            batches,
-            namespace.clone(),
-            table_id.clone(),
-            Some(write_params),
-        )
-        .await
-        .unwrap();
+            let mut buffer = Vec::new();
+            {
+                let mut writer = StreamWriter::try_new(&mut buffer, &arrow_schema).unwrap();
+                writer.write(&batch).unwrap();
+                writer.finish().unwrap();
+            }
 
-        // Append data to create version 2
-        let batch2 = RecordBatch::try_new(
-            arrow_schema.clone(),
-            vec![Arc::new(Int32Array::from(vec![100, 200]))],
-        )
-        .unwrap();
-        let batches = RecordBatchIterator::new(vec![Ok(batch2)], arrow_schema);
-        dataset.append(batches, None).await.unwrap();
+            let request = InsertIntoTableRequest {
+                id: Some(table_id.clone()),
+                mode: None,
+                ..Default::default()
+            };
+            namespace
+                .insert_into_table(request, Bytes::from(buffer))
+                .await
+                .unwrap();
 
-        // Describe version 1
-        let mut describe_req = DescribeTableVersionRequest::new();
-        describe_req.id = Some(table_id.clone());
-        describe_req.version = Some(1);
-        let describe_resp = namespace
-            .describe_table_version(describe_req)
-            .await
-            .unwrap();
+            // Version 1 should have 3 rows
+            let count_req = CountTableRowsRequest {
+                id: Some(table_id.clone()),
+                version: Some(1),
+                predicate: None,
+                ..Default::default()
+            };
+            let count = namespace.count_table_rows(count_req).await.unwrap();
+            assert_eq!(count, 3);
+
+            // Latest version should have 5 rows
+            let count_req = CountTableRowsRequest {
+                id: Some(table_id),
+                version: None,
+                predicate: None,
+                ..Default::default()
+            };
+            let count = namespace.count_table_rows(count_req).await.unwrap();
+            assert_eq!(count, 5);
+        }
 
-        let version = &describe_resp.version;
-        assert_eq!(version.version, 1);
-        assert!(version.timestamp_millis.is_some());
-        assert!(
-            !version.manifest_path.is_empty(),
-            "manifest_path should be set"
-        );
-        assert!(
-            version.manifest_path.contains(".manifest"),
-            "manifest_path should contain .manifest"
-        );
-        assert!(
-            version.manifest_size.is_some(),
-            "manifest_size should be set"
-        );
-        assert!(
-            version.manifest_size.unwrap() > 0,
-            "manifest_size should be > 0"
-        );
+        #[tokio::test]
+        async fn test_query_table_with_version() {
+            use arrow::array::{Int32Array, StringArray};
+            use arrow::ipc::writer::StreamWriter;
 
-        // Describe version 2
-        let mut describe_req = DescribeTableVersionRequest::new();
-        describe_req.id = Some(table_id.clone());
-        describe_req.version = Some(2);
-        let describe_resp = namespace
-            .describe_table_version(describe_req)
-            .await
-            .unwrap();
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
 
-        let version = &describe_resp.version;
-        assert_eq!(version.version, 2);
-        assert!(version.timestamp_millis.is_some());
-        assert!(
-            !version.manifest_path.is_empty(),
-            "manifest_path should be set"
-        );
-        assert!(
-            version.manifest_size.is_some(),
-            "manifest_size should be set"
-        );
-        assert!(
-            version.manifest_size.unwrap() > 0,
-            "manifest_size should be > 0"
-        );
-    }
+            // Insert more data to create version 2
+            let schema = create_test_schema();
+            let arrow_schema = convert_json_arrow_schema(&schema).unwrap();
+            let arrow_schema = Arc::new(arrow_schema);
 
-    #[tokio::test]
-    #[cfg(not(windows))]
-    async fn test_describe_table_version_latest() {
-        use arrow::array::{Int32Array, RecordBatchIterator};
-        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
-        use arrow::record_batch::RecordBatch;
-        use lance::dataset::{Dataset, WriteMode, WriteParams};
-        use lance_namespace::models::{CreateNamespaceRequest, DescribeTableVersionRequest};
+            let id_array = Int32Array::from(vec![4, 5]);
+            let name_array = StringArray::from(vec!["Dave", "Eve"]);
+            let batch = arrow::record_batch::RecordBatch::try_new(
+                arrow_schema.clone(),
+                vec![Arc::new(id_array), Arc::new(name_array)],
+            )
+            .unwrap();
 
-        let temp_dir = TempStrDir::default();
-        let temp_path: &str = &temp_dir;
+            let mut buffer = Vec::new();
+            {
+                let mut writer = StreamWriter::try_new(&mut buffer, &arrow_schema).unwrap();
+                writer.write(&batch).unwrap();
+                writer.finish().unwrap();
+            }
 
-        let namespace: Arc<dyn LanceNamespace> = Arc::new(
-            DirectoryNamespaceBuilder::new(temp_path)
-                .table_version_tracking_enabled(true)
-                .build()
+            let request = InsertIntoTableRequest {
+                id: Some(table_id.clone()),
+                mode: None,
+                ..Default::default()
+            };
+            namespace
+                .insert_into_table(request, Bytes::from(buffer))
                 .await
-                .unwrap(),
-        );
+                .unwrap();
 
-        // Create parent namespace first
-        let mut create_ns_req = CreateNamespaceRequest::new();
-        create_ns_req.id = Some(vec!["workspace".to_string()]);
-        namespace.create_namespace(create_ns_req).await.unwrap();
+            // Query version 1 should return 3 rows
+            let request = QueryTableRequest {
+                id: Some(table_id.clone()),
+                k: 100,
+                filter: None,
+                offset: None,
+                version: Some(1),
+                ..Default::default()
+            };
 
-        // Create a table using write_into_namespace (version 1)
-        let table_id = vec!["workspace".to_string(), "test_table".to_string()];
-        let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
-            "id",
-            DataType::Int32,
-            false,
-        )]));
-        let batch = RecordBatch::try_new(
-            arrow_schema.clone(),
-            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
-        )
-        .unwrap();
-        let batches = RecordBatchIterator::new(vec![Ok(batch)], arrow_schema.clone());
-        let write_params = WriteParams {
-            mode: WriteMode::Create,
-            ..Default::default()
-        };
-        let mut dataset = Dataset::write_into_namespace(
-            batches,
-            namespace.clone(),
-            table_id.clone(),
-            Some(write_params),
-        )
-        .await
-        .unwrap();
+            let bytes = namespace.query_table(request).await.unwrap();
+            let cursor = Cursor::new(bytes.to_vec());
+            let reader = FileReader::try_new(cursor, None).unwrap();
+            let batches: Vec<_> = reader.into_iter().map(|b| b.unwrap()).collect();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert_eq!(total_rows, 3);
+
+            // Query latest version should return 5 rows
+            let request = QueryTableRequest {
+                id: Some(table_id),
+                k: 100,
+                filter: None,
+                offset: None,
+                version: None,
+                ..Default::default()
+            };
 
-        // Append to create version 2
-        let batch2 = RecordBatch::try_new(
-            arrow_schema.clone(),
-            vec![Arc::new(Int32Array::from(vec![100, 200]))],
-        )
-        .unwrap();
-        let batches = RecordBatchIterator::new(vec![Ok(batch2)], arrow_schema.clone());
-        dataset.append(batches, None).await.unwrap();
+            let bytes = namespace.query_table(request).await.unwrap();
+            let cursor = Cursor::new(bytes.to_vec());
+            let reader = FileReader::try_new(cursor, None).unwrap();
+            let batches: Vec<_> = reader.into_iter().map(|b| b.unwrap()).collect();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert_eq!(total_rows, 5);
+        }
 
-        // Append to create version 3
-        let batch3 = RecordBatch::try_new(
-            arrow_schema.clone(),
-            vec![Arc::new(Int32Array::from(vec![300, 400]))],
-        )
-        .unwrap();
-        let batches = RecordBatchIterator::new(vec![Ok(batch3)], arrow_schema);
-        dataset.append(batches, None).await.unwrap();
+        /// Helper to create a namespace with a table that has a vector column for
+        /// vector search tests.
+        async fn create_ns_with_vector_table() -> (DirectoryNamespace, TempStdDir, Vec<String>) {
+            use arrow::array::{FixedSizeListArray, Float32Array, Int32Array};
+            use arrow::ipc::writer::StreamWriter;
+
+            let (namespace, temp_dir) = create_test_namespace().await;
+
+            // Build schema: id (int32), vector (fixed_size_list<float32>[4])
+            let arrow_schema = Arc::new(arrow::datatypes::Schema::new(vec![
+                arrow::datatypes::Field::new("id", arrow::datatypes::DataType::Int32, false),
+                arrow::datatypes::Field::new(
+                    "vector",
+                    arrow::datatypes::DataType::FixedSizeList(
+                        Arc::new(arrow::datatypes::Field::new(
+                            "item",
+                            arrow::datatypes::DataType::Float32,
+                            true,
+                        )),
+                        4,
+                    ),
+                    true,
+                ),
+            ]));
 
-        // Describe latest version (no version specified)
-        let mut describe_req = DescribeTableVersionRequest::new();
-        describe_req.id = Some(table_id.clone());
-        describe_req.version = None;
-        let describe_resp = namespace
-            .describe_table_version(describe_req)
-            .await
+            let id_array = Int32Array::from(vec![1, 2, 3]);
+            let values = Float32Array::from(vec![
+                1.0, 0.0, 0.0, 0.0, // vector for id=1
+                0.0, 1.0, 0.0, 0.0, // vector for id=2
+                0.0, 0.0, 1.0, 0.0, // vector for id=3
+            ]);
+            let vector_array = FixedSizeListArray::try_new(
+                Arc::new(arrow::datatypes::Field::new(
+                    "item",
+                    arrow::datatypes::DataType::Float32,
+                    true,
+                )),
+                4,
+                Arc::new(values),
+                None,
+            )
             .unwrap();
 
-        // Should return version 3 as it's the latest
-        assert_eq!(describe_resp.version.version, 3);
-    }
+            let batch = arrow::record_batch::RecordBatch::try_new(
+                arrow_schema.clone(),
+                vec![Arc::new(id_array), Arc::new(vector_array)],
+            )
+            .unwrap();
 
-    #[tokio::test]
-    #[cfg(not(windows))]
-    async fn test_create_table_version() {
-        use futures::TryStreamExt;
-        use lance::dataset::builder::DatasetBuilder;
-        use lance_namespace::models::CreateTableVersionRequest;
+            let mut buffer = Vec::new();
+            {
+                let mut writer = StreamWriter::try_new(&mut buffer, &arrow_schema).unwrap();
+                writer.write(&batch).unwrap();
+                writer.finish().unwrap();
+            }
 
-        let temp_dir = TempStrDir::default();
-        let temp_path: &str = &temp_dir;
+            // Write as a Lance dataset directly
+            let table_name = "vector_table";
+            let table_uri = format!("{}/{}.lance", temp_dir.to_str().unwrap(), table_name);
+            let reader = arrow::record_batch::RecordBatchIterator::new(
+                vec![Ok(batch)],
+                arrow_schema.clone(),
+            );
+            Dataset::write(reader, &table_uri, None).await.unwrap();
 
-        let namespace: Arc<dyn LanceNamespace> = Arc::new(
-            DirectoryNamespaceBuilder::new(temp_path)
-                .table_version_tracking_enabled(true)
-                .build()
-                .await
-                .unwrap(),
-        );
+            let table_id = vec![table_name.to_string()];
+            (namespace, temp_dir, table_id)
+        }
 
-        // Create a table
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-        let mut create_req = CreateTableRequest::new();
-        create_req.id = Some(vec!["test_table".to_string()]);
-        namespace
-            .create_table(create_req, bytes::Bytes::from(ipc_data))
-            .await
-            .unwrap();
+        #[tokio::test]
+        async fn test_query_table_vector_search() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_vector_table().await;
+
+            let vector = Box::new(lance_namespace::models::QueryTableRequestVector {
+                single_vector: Some(vec![1.0, 0.0, 0.0, 0.0]),
+                multi_vector: None,
+            });
+
+            let request = QueryTableRequest {
+                id: Some(table_id),
+                k: 2,
+                vector,
+                filter: None,
+                offset: None,
+                version: None,
+                ..Default::default()
+            };
+
+            let bytes = namespace.query_table(request).await.unwrap();
 
-        // Open the dataset using from_namespace to get proper object_store and paths
-        let table_id = vec!["test_table".to_string()];
-        let dataset = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone())
-            .await
-            .unwrap()
-            .load()
-            .await
-            .unwrap();
+            let cursor = Cursor::new(bytes.to_vec());
+            let reader = FileReader::try_new(cursor, None).unwrap();
+            let batches: Vec<_> = reader.into_iter().map(|b| b.unwrap()).collect();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert_eq!(total_rows, 2);
+        }
 
-        // Use dataset's object_store to find and copy the manifest
-        let versions_path = dataset.versions_dir();
-        let manifest_metas: Vec<_> = dataset
-            .object_store()
-            .inner
-            .list(Some(&versions_path))
-            .try_collect()
-            .await
-            .unwrap();
+        #[tokio::test]
+        async fn test_query_table_vector_search_with_distance_type() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_vector_table().await;
 
-        let manifest_meta = manifest_metas
-            .iter()
-            .find(|m| {
-                m.location
-                    .filename()
-                    .map(|f| f.ends_with(".manifest"))
-                    .unwrap_or(false)
-            })
-            .expect("No manifest file found");
+            let vector = Box::new(lance_namespace::models::QueryTableRequestVector {
+                single_vector: Some(vec![1.0, 0.0, 0.0, 0.0]),
+                multi_vector: None,
+            });
 
-        // Read the existing manifest data
-        let manifest_data = dataset
-            .object_store()
-            .inner
-            .get(&manifest_meta.location)
-            .await
-            .unwrap()
-            .bytes()
-            .await
-            .unwrap();
+            let request = QueryTableRequest {
+                id: Some(table_id),
+                k: 3,
+                vector,
+                filter: None,
+                offset: None,
+                version: None,
+                distance_type: Some("cosine".to_string()),
+                ..Default::default()
+            };
 
-        // Write to a staging location using the dataset's object_store
-        let staging_path = dataset.versions_dir().child("staging_manifest");
-        dataset
-            .object_store()
-            .inner
-            .put(&staging_path, manifest_data.into())
-            .await
-            .unwrap();
+            let bytes = namespace.query_table(request).await.unwrap();
 
-        // Create version 2 from staging manifest
-        // Use the same naming scheme as the existing dataset (V2)
-        let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string());
-        create_version_req.id = Some(table_id.clone());
-        create_version_req.naming_scheme = Some("V2".to_string());
+            let cursor = Cursor::new(bytes.to_vec());
+            let reader = FileReader::try_new(cursor, None).unwrap();
+            let batches: Vec<_> = reader.into_iter().map(|b| b.unwrap()).collect();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert_eq!(total_rows, 3);
+        }
 
-        let result = namespace.create_table_version(create_version_req).await;
-        assert!(
-            result.is_ok(),
-            "create_table_version should succeed: {:?}",
-            result
-        );
+        #[tokio::test]
+        async fn test_query_table_vector_search_with_filter() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_vector_table().await;
 
-        // Verify version 2 was created at the path returned in the response
-        let response = result.unwrap();
-        let version_info = response
-            .version
-            .expect("response should contain version info");
-        let version_2_path = Path::from(version_info.manifest_path);
-        let head_result = dataset.object_store().inner.head(&version_2_path).await;
-        assert!(
-            head_result.is_ok(),
-            "Version 2 manifest should exist at {}",
-            version_2_path
-        );
+            let vector = Box::new(lance_namespace::models::QueryTableRequestVector {
+                single_vector: Some(vec![1.0, 0.0, 0.0, 0.0]),
+                multi_vector: None,
+            });
 
-        // Verify the staging file has been deleted
-        let staging_head_result = dataset.object_store().inner.head(&staging_path).await;
-        assert!(
-            staging_head_result.is_err(),
-            "Staging manifest should have been deleted after create_table_version"
-        );
-    }
+            let request = QueryTableRequest {
+                id: Some(table_id),
+                k: 10,
+                vector,
+                filter: Some("id <= 2".to_string()),
+                offset: None,
+                version: None,
+                ..Default::default()
+            };
 
-    #[tokio::test]
-    #[cfg(not(windows))]
-    async fn test_create_table_version_conflict() {
-        // create_table_version should fail if the version already exists.
-        // Each version always writes to a new file location.
-        use futures::TryStreamExt;
-        use lance::dataset::builder::DatasetBuilder;
-        use lance_namespace::models::CreateTableVersionRequest;
+            let bytes = namespace.query_table(request).await.unwrap();
 
-        let temp_dir = TempStrDir::default();
-        let temp_path: &str = &temp_dir;
+            let cursor = Cursor::new(bytes.to_vec());
+            let reader = FileReader::try_new(cursor, None).unwrap();
+            let batches: Vec<_> = reader.into_iter().map(|b| b.unwrap()).collect();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert!(total_rows <= 2);
+        }
 
-        let namespace: Arc<dyn LanceNamespace> = Arc::new(
-            DirectoryNamespaceBuilder::new(temp_path)
-                .table_version_tracking_enabled(true)
-                .build()
-                .await
-                .unwrap(),
-        );
+        #[tokio::test]
+        async fn test_query_table_vector_search_with_nprobes_and_refine() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_vector_table().await;
 
-        // Create a table
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-        let mut create_req = CreateTableRequest::new();
-        create_req.id = Some(vec!["test_table".to_string()]);
-        namespace
-            .create_table(create_req, bytes::Bytes::from(ipc_data))
-            .await
-            .unwrap();
+            let vector = Box::new(lance_namespace::models::QueryTableRequestVector {
+                single_vector: Some(vec![0.0, 1.0, 0.0, 0.0]),
+                multi_vector: None,
+            });
 
-        // Open the dataset using from_namespace to get proper object_store and paths
-        let table_id = vec!["test_table".to_string()];
-        let dataset = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone())
-            .await
-            .unwrap()
-            .load()
-            .await
-            .unwrap();
+            let request = QueryTableRequest {
+                id: Some(table_id),
+                k: 2,
+                vector,
+                filter: None,
+                offset: None,
+                version: None,
+                nprobes: Some(1),
+                refine_factor: Some(1),
+                prefilter: Some(true),
+                ..Default::default()
+            };
 
-        // Use dataset's object_store to find and copy the manifest
-        let versions_path = dataset.versions_dir();
-        let manifest_metas: Vec<_> = dataset
-            .object_store()
-            .inner
-            .list(Some(&versions_path))
-            .try_collect()
-            .await
-            .unwrap();
+            let bytes = namespace.query_table(request).await.unwrap();
 
-        let manifest_meta = manifest_metas
-            .iter()
-            .find(|m| {
-                m.location
-                    .filename()
-                    .map(|f| f.ends_with(".manifest"))
-                    .unwrap_or(false)
-            })
-            .expect("No manifest file found");
+            let cursor = Cursor::new(bytes.to_vec());
+            let reader = FileReader::try_new(cursor, None).unwrap();
+            let batches: Vec<_> = reader.into_iter().map(|b| b.unwrap()).collect();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert_eq!(total_rows, 2);
+        }
 
-        // Read the existing manifest data
-        let manifest_data = dataset
-            .object_store()
-            .inner
-            .get(&manifest_meta.location)
-            .await
-            .unwrap()
-            .bytes()
-            .await
-            .unwrap();
+        #[tokio::test]
+        async fn test_namespace_id() {
+            let (namespace, _temp_dir) = create_test_namespace().await;
+            let id = namespace.namespace_id();
+            assert!(id.contains("DirectoryNamespace"));
+            assert!(id.contains("root"));
+        }
 
-        // Write to a staging location using the dataset's object_store
-        let staging_path = dataset.versions_dir().child("staging_manifest");
-        dataset
-            .object_store()
-            .inner
-            .put(&staging_path, manifest_data.into())
-            .await
-            .unwrap();
+        #[tokio::test]
+        async fn test_query_table_empty_table() {
+            let (namespace, _temp_dir) = create_test_namespace().await;
 
-        // First create version 2 (should succeed)
-        let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string());
-        create_version_req.id = Some(table_id.clone());
-        create_version_req.naming_scheme = Some("V2".to_string());
-        let first_result = namespace.create_table_version(create_version_req).await;
-        assert!(
-            first_result.is_ok(),
-            "First create_table_version for version 2 should succeed: {:?}",
-            first_result
-        );
+            // Create table with empty IPC data (schema only, no rows)
+            let schema = create_test_schema();
+            let ipc_data = create_test_ipc_data(&schema);
+            let mut create_request = CreateTableRequest::new();
+            create_request.id = Some(vec!["empty_table".to_string()]);
+            namespace
+                .create_table(create_request, bytes::Bytes::from(ipc_data))
+                .await
+                .unwrap();
 
-        // Get the path from the response for verification
-        let version_2_path = Path::from(
-            first_result
-                .unwrap()
-                .version
-                .expect("response should contain version info")
-                .manifest_path,
-        );
+            // Query the empty table — should hit the "no batches" else branch
+            let vector = Box::new(lance_namespace::models::QueryTableRequestVector {
+                single_vector: None,
+                multi_vector: None,
+            });
+            let request = QueryTableRequest {
+                id: Some(vec!["empty_table".to_string()]),
+                k: 10,
+                vector,
+                ..Default::default()
+            };
+            let bytes = namespace.query_table(request).await.unwrap();
 
-        // Create version 2 again (should fail - conflict)
-        let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string());
-        create_version_req.id = Some(table_id.clone());
-        create_version_req.naming_scheme = Some("V2".to_string());
+            let cursor = Cursor::new(bytes.to_vec());
+            let reader = FileReader::try_new(cursor, None).unwrap();
+            let batches: Vec<_> = reader.collect::<std::result::Result<Vec<_>, _>>().unwrap();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert_eq!(total_rows, 0, "empty table should yield no rows");
+        }
 
-        let result = namespace.create_table_version(create_version_req).await;
-        assert!(
-            result.is_err(),
-            "create_table_version should fail for existing version"
-        );
+        #[tokio::test]
+        async fn test_query_table_with_plain_filter_no_vector() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
 
-        // Verify version 2 still exists using the dataset's object_store
-        let head_result = dataset.object_store().inner.head(&version_2_path).await;
-        assert!(
-            head_result.is_ok(),
-            "Version 2 manifest should still exist at {}",
-            version_2_path
-        );
+            // Query with filter but no vector (plain scan path + filter)
+            let vector = Box::new(lance_namespace::models::QueryTableRequestVector {
+                single_vector: None,
+                multi_vector: None,
+            });
+            let request = QueryTableRequest {
+                id: Some(table_id),
+                k: 0,
+                vector,
+                filter: Some("id > 1".to_string()),
+                ..Default::default()
+            };
+            let bytes = namespace.query_table(request).await.unwrap();
+
+            let cursor = Cursor::new(bytes.to_vec());
+            let reader = FileReader::try_new(cursor, None).unwrap();
+            let batches: Vec<_> = reader.into_iter().map(|b| b.unwrap()).collect();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert!(total_rows > 0);
+            assert!(total_rows < 3);
+        }
     }
 
-    #[tokio::test]
-    async fn test_create_table_version_table_not_found() {
+    /// Tests for multi-table transaction support via table_version_storage_enabled.
+    mod multi_table_transactions {
+        use super::*;
+        use futures::TryStreamExt;
+        use lance::dataset::builder::DatasetBuilder;
         use lance_namespace::models::CreateTableVersionRequest;
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+        /// Helper to create a namespace with table_version_storage_enabled enabled
+        async fn create_managed_namespace(temp_path: &str) -> Arc<DirectoryNamespace> {
+            Arc::new(
+                DirectoryNamespaceBuilder::new(temp_path)
+                    .table_version_tracking_enabled(true)
+                    .table_version_storage_enabled(true)
+                    .manifest_enabled(true)
+                    .build()
+                    .await
+                    .unwrap(),
+            )
+        }
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_path)
-            .table_version_tracking_enabled(true)
-            .build()
-            .await
-            .unwrap();
+        /// Helper to create a table and get its staging manifest path
+        async fn create_table_and_get_staging(
+            namespace: Arc<dyn LanceNamespace>,
+            table_name: &str,
+        ) -> (Vec<String>, object_store::path::Path) {
+            let schema = create_test_schema();
+            let ipc_data = create_test_ipc_data(&schema);
+            let mut create_req = CreateTableRequest::new();
+            create_req.id = Some(vec![table_name.to_string()]);
+            namespace
+                .create_table(create_req, bytes::Bytes::from(ipc_data))
+                .await
+                .unwrap();
+
+            let table_id = vec![table_name.to_string()];
+            let dataset = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone())
+                .await
+                .unwrap()
+                .load()
+                .await
+                .unwrap();
+
+            // Find existing manifest and create a staging copy
+            let versions_path = dataset.versions_dir();
+            let manifest_metas: Vec<_> = dataset
+                .object_store()
+                .inner
+                .list(Some(&versions_path))
+                .try_collect()
+                .await
+                .unwrap();
+
+            let manifest_meta = manifest_metas
+                .iter()
+                .find(|m| {
+                    m.location
+                        .filename()
+                        .map(|f| f.ends_with(".manifest"))
+                        .unwrap_or(false)
+                })
+                .expect("No manifest file found");
+
+            let manifest_data = dataset
+                .object_store()
+                .inner
+                .get(&manifest_meta.location)
+                .await
+                .unwrap()
+                .bytes()
+                .await
+                .unwrap();
+
+            let staging_path = dataset
+                .versions_dir()
+                .child(format!("staging_{}", table_name));
+            dataset
+                .object_store()
+                .inner
+                .put(&staging_path, manifest_data.into())
+                .await
+                .unwrap();
 
-        // Try to create version for non-existent table
-        let mut create_version_req =
-            CreateTableVersionRequest::new(1, "/some/staging/path".to_string());
-        create_version_req.id = Some(vec!["non_existent_table".to_string()]);
+            (table_id, staging_path)
+        }
 
-        let result = namespace.create_table_version(create_version_req).await;
-        assert!(
-            result.is_err(),
-            "create_table_version should fail for non-existent table"
-        );
-        let err_msg = result.unwrap_err().to_string();
-        assert!(
-            err_msg.contains("does not exist"),
-            "Error should mention table does not exist, got: {}",
-            err_msg
-        );
-    }
+        #[tokio::test]
+        async fn test_table_version_storage_enabled_requires_manifest() {
+            // table_version_storage_enabled=true requires manifest_enabled=true
+            let temp_dir = TempStdDir::default();
+            let temp_path = temp_dir.to_str().unwrap();
 
-    /// End-to-end integration test module for table version tracking.
-    mod e2e_table_version_tracking {
-        use super::*;
-        use std::sync::atomic::{AtomicUsize, Ordering};
+            let result = DirectoryNamespaceBuilder::new(temp_path)
+                .table_version_storage_enabled(true)
+                .manifest_enabled(false)
+                .build()
+                .await;
 
-        /// Tracking wrapper around a namespace that counts method invocations.
-        struct TrackingNamespace {
-            inner: DirectoryNamespace,
-            create_table_version_count: AtomicUsize,
-            describe_table_version_count: AtomicUsize,
-            list_table_versions_count: AtomicUsize,
+            assert!(
+                result.is_err(),
+                "Should fail when table_version_storage_enabled=true but manifest_enabled=false"
+            );
         }
 
-        impl TrackingNamespace {
-            fn new(inner: DirectoryNamespace) -> Self {
-                Self {
-                    inner,
-                    create_table_version_count: AtomicUsize::new(0),
-                    describe_table_version_count: AtomicUsize::new(0),
-                    list_table_versions_count: AtomicUsize::new(0),
-                }
-            }
+        #[tokio::test]
+        #[cfg(not(windows))]
+        async fn test_create_table_version_records_in_manifest() {
+            // When table_version_storage_enabled is enabled, single create_table_version
+            // should also record the version in __manifest
+            let temp_dir = TempStrDir::default();
+            let temp_path: &str = &temp_dir;
+
+            let namespace = create_managed_namespace(temp_path).await;
+            let ns: Arc<dyn LanceNamespace> = namespace.clone();
+
+            let (table_id, staging_path) =
+                create_table_and_get_staging(ns.clone(), "table_managed").await;
+
+            // Create version 2
+            let mut create_req = CreateTableVersionRequest::new(2, staging_path.to_string());
+            create_req.id = Some(table_id.clone());
+            create_req.naming_scheme = Some("V2".to_string());
+            let response = namespace.create_table_version(create_req).await.unwrap();
+
+            assert!(response.version.is_some());
+            let version = response.version.unwrap();
+            assert_eq!(version.version, 2);
+
+            // Verify the version is recorded in __manifest by querying it
+            let manifest_ns = namespace.manifest_ns.as_ref().unwrap();
+            let table_id_str = manifest::ManifestNamespace::str_object_id(&table_id);
+            let versions = manifest_ns
+                .query_table_versions(&table_id_str, false, None)
+                .await
+                .unwrap();
 
-            fn create_table_version_calls(&self) -> usize {
-                self.create_table_version_count.load(Ordering::SeqCst)
-            }
+            assert!(
+                !versions.is_empty(),
+                "Version should be recorded in __manifest"
+            );
+            let (ver, _path) = &versions[0];
+            assert_eq!(*ver, 2, "Recorded version should be 2");
+        }
+    }
 
-            fn describe_table_version_calls(&self) -> usize {
-                self.describe_table_version_count.load(Ordering::SeqCst)
-            }
+    #[tokio::test]
+    async fn test_list_all_tables() {
+        use lance_namespace::models::ListTablesRequest;
 
-            fn list_table_versions_calls(&self) -> usize {
-                self.list_table_versions_count.load(Ordering::SeqCst)
-            }
-        }
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "alpha").await;
+        create_scalar_table(&namespace, "beta").await;
 
-        impl std::fmt::Debug for TrackingNamespace {
-            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-                f.debug_struct("TrackingNamespace")
-                    .field(
-                        "create_table_version_calls",
-                        &self.create_table_version_calls(),
-                    )
-                    .finish()
-            }
-        }
+        let request = ListTablesRequest {
+            id: Some(vec![]),
+            page_token: None,
+            limit: None,
+            ..Default::default()
+        };
+        let response = namespace.list_all_tables(request).await.unwrap();
+        let mut tables = response.tables;
+        tables.sort();
+        assert_eq!(tables, vec!["alpha", "beta"]);
+    }
 
-        #[async_trait]
-        impl LanceNamespace for TrackingNamespace {
-            async fn create_namespace(
-                &self,
-                request: CreateNamespaceRequest,
-            ) -> Result<CreateNamespaceResponse> {
-                self.inner.create_namespace(request).await
-            }
+    #[tokio::test]
+    async fn test_restore_table() {
+        use lance_namespace::models::RestoreTableRequest;
 
-            async fn describe_namespace(
-                &self,
-                request: DescribeNamespaceRequest,
-            ) -> Result<DescribeNamespaceResponse> {
-                self.inner.describe_namespace(request).await
-            }
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
 
-            async fn namespace_exists(&self, request: NamespaceExistsRequest) -> Result<()> {
-                self.inner.namespace_exists(request).await
-            }
+        // Create a second version by creating a scalar index (this adds a new version)
+        create_scalar_index(&namespace, "users", "users_id_idx").await;
 
-            async fn list_namespaces(
-                &self,
-                request: ListNamespacesRequest,
-            ) -> Result<ListNamespacesResponse> {
-                self.inner.list_namespaces(request).await
-            }
+        let dataset = open_dataset(&namespace, "users").await;
+        let current_version = dataset.version().version;
+        assert!(current_version >= 2, "Should have at least 2 versions");
 
-            async fn drop_namespace(
-                &self,
-                request: DropNamespaceRequest,
-            ) -> Result<DropNamespaceResponse> {
-                self.inner.drop_namespace(request).await
-            }
+        // Restore to version 1
+        let mut restore_req = RestoreTableRequest::new(1);
+        restore_req.id = Some(vec!["users".to_string()]);
+        let response = namespace.restore_table(restore_req).await.unwrap();
 
-            async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
-                self.inner.list_tables(request).await
-            }
+        // transaction_id should be present (the restore operation)
+        assert!(
+            response.transaction_id.is_some(),
+            "restore_table should return a transaction_id"
+        );
 
-            async fn describe_table(
-                &self,
-                request: DescribeTableRequest,
-            ) -> Result<DescribeTableResponse> {
-                self.inner.describe_table(request).await
-            }
+        // Verify the dataset now has a new version (restore creates a new version)
+        let dataset_after = open_dataset(&namespace, "users").await;
+        assert!(
+            dataset_after.version().version > current_version,
+            "Restore should create a new version"
+        );
+    }
 
-            async fn table_exists(&self, request: TableExistsRequest) -> Result<()> {
-                self.inner.table_exists(request).await
-            }
+    #[tokio::test]
+    async fn test_update_table_schema_metadata() {
+        use lance_namespace::models::UpdateTableSchemaMetadataRequest;
 
-            async fn drop_table(&self, request: DropTableRequest) -> Result<DropTableResponse> {
-                self.inner.drop_table(request).await
-            }
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "products").await;
 
-            async fn create_table(
-                &self,
-                request: CreateTableRequest,
-                request_data: Bytes,
-            ) -> Result<CreateTableResponse> {
-                self.inner.create_table(request, request_data).await
-            }
+        let mut metadata = HashMap::new();
+        metadata.insert("owner".to_string(), "team_a".to_string());
+        metadata.insert("version".to_string(), "1.0".to_string());
 
-            async fn declare_table(
-                &self,
-                request: DeclareTableRequest,
-            ) -> Result<DeclareTableResponse> {
-                self.inner.declare_table(request).await
-            }
+        let mut req = UpdateTableSchemaMetadataRequest::new();
+        req.id = Some(vec!["products".to_string()]);
+        req.metadata = Some(metadata.clone());
 
-            async fn list_table_versions(
-                &self,
-                request: ListTableVersionsRequest,
-            ) -> Result<ListTableVersionsResponse> {
-                self.list_table_versions_count
-                    .fetch_add(1, Ordering::SeqCst);
-                self.inner.list_table_versions(request).await
-            }
+        let response = namespace.update_table_schema_metadata(req).await.unwrap();
 
-            async fn create_table_version(
-                &self,
-                request: CreateTableVersionRequest,
-            ) -> Result<CreateTableVersionResponse> {
-                self.create_table_version_count
-                    .fetch_add(1, Ordering::SeqCst);
-                self.inner.create_table_version(request).await
-            }
+        assert!(response.metadata.is_some());
+        let returned = response.metadata.unwrap();
+        assert_eq!(returned.get("owner"), Some(&"team_a".to_string()));
+        assert_eq!(returned.get("version"), Some(&"1.0".to_string()));
+        assert!(
+            response.transaction_id.is_some(),
+            "update_table_schema_metadata should return a transaction_id"
+        );
+    }
 
-            async fn describe_table_version(
-                &self,
-                request: DescribeTableVersionRequest,
-            ) -> Result<DescribeTableVersionResponse> {
-                self.describe_table_version_count
-                    .fetch_add(1, Ordering::SeqCst);
-                self.inner.describe_table_version(request).await
-            }
+    #[tokio::test]
+    async fn test_get_table_stats() {
+        use lance_namespace::models::GetTableStatsRequest;
 
-            async fn batch_delete_table_versions(
-                &self,
-                request: BatchDeleteTableVersionsRequest,
-            ) -> Result<BatchDeleteTableVersionsResponse> {
-                self.inner.batch_delete_table_versions(request).await
-            }
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "items").await;
+        create_scalar_index(&namespace, "items", "items_id_idx").await;
 
-            fn namespace_id(&self) -> String {
-                self.inner.namespace_id()
-            }
-        }
+        let mut req = GetTableStatsRequest::new();
+        req.id = Some(vec!["items".to_string()]);
 
-        #[tokio::test]
-        async fn test_describe_table_returns_managed_versioning() {
-            use lance_namespace::models::{CreateNamespaceRequest, DescribeTableRequest};
+        let response = namespace.get_table_stats(req).await.unwrap();
+        assert_eq!(response.num_rows, 3);
+        assert_eq!(response.num_indices, 1);
+    }
 
-            let temp_dir = TempStdDir::default();
-            let temp_path = temp_dir.to_str().unwrap();
+    #[tokio::test]
+    async fn test_explain_table_query_plan() {
+        use lance_namespace::models::QueryTableRequestVector;
+        use lance_namespace::models::{ExplainTableQueryPlanRequest, QueryTableRequest};
 
-            // Create namespace with table_version_tracking_enabled and manifest_enabled
-            let ns = DirectoryNamespaceBuilder::new(temp_path)
-                .table_version_tracking_enabled(true)
-                .manifest_enabled(true)
-                .build()
-                .await
-                .unwrap();
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "catalog").await;
+
+        let mut query = QueryTableRequest::new(1, QueryTableRequestVector::new());
+        query.filter = Some("id > 1".to_string());
+        query.columns = Some(Box::new(QueryTableRequestColumns {
+            column_names: Some(vec!["id".to_string(), "name".to_string()]),
+            column_aliases: None,
+        }));
+        query.with_row_id = Some(true);
+
+        let mut req = ExplainTableQueryPlanRequest::new(query);
+        req.id = Some(vec!["catalog".to_string()]);
+
+        let plan_str = namespace.explain_table_query_plan(req).await.unwrap();
+        assert_plan_contains_all(
+            &plan_str,
+            &[
+                "ProjectionExec: expr=[id@0 as id, name@2 as name",
+                "Take: columns=\"id, _rowid, (name)\"",
+                "LanceRead: uri=",
+                "projection=[id]",
+                "row_id=true, row_addr=false",
+                "full_filter=id > Int32(1)",
+                "refine_filter=id > Int32(1)",
+            ],
+            "Filtered explain plan should preserve late materialization and filter pushdown",
+        );
+    }
 
-            // Create parent namespace
-            let mut create_ns_req = CreateNamespaceRequest::new();
-            create_ns_req.id = Some(vec!["workspace".to_string()]);
-            ns.create_namespace(create_ns_req).await.unwrap();
+    #[tokio::test]
+    async fn test_analyze_table_query_plan() {
+        use lance_namespace::models::AnalyzeTableQueryPlanRequest;
+        use lance_namespace::models::QueryTableRequestVector;
 
-            // Create a table with multi-level ID (namespace + table)
-            let schema = create_test_schema();
-            let ipc_data = create_test_ipc_data(&schema);
-            let mut create_req = CreateTableRequest::new();
-            create_req.id = Some(vec!["workspace".to_string(), "test_table".to_string()]);
-            ns.create_table(create_req, bytes::Bytes::from(ipc_data))
-                .await
-                .unwrap();
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "catalog").await;
+
+        let mut req = AnalyzeTableQueryPlanRequest::new(1, QueryTableRequestVector::new());
+        req.id = Some(vec!["catalog".to_string()]);
+        req.filter = Some("id > 0".to_string());
+        req.columns = Some(Box::new(QueryTableRequestColumns {
+            column_names: Some(vec!["id".to_string(), "name".to_string()]),
+            column_aliases: None,
+        }));
+        req.with_row_id = Some(true);
+
+        let analysis_str = namespace.analyze_table_query_plan(req).await.unwrap();
+        assert_plan_contains_all(
+            &analysis_str,
+            &[
+                "AnalyzeExec verbose=true",
+                "ProjectionExec: elapsed=",
+                "expr=[id@0 as id, name@2 as name",
+                "Take: elapsed=",
+                "columns=\"id, _rowid, (name)\"",
+                "CoalesceBatchesExec: elapsed=",
+                "LanceRead: elapsed=",
+                "projection=[id]",
+                "row_id=true, row_addr=false",
+                "full_filter=id > Int32(0)",
+                "refine_filter=id > Int32(0)",
+                "metrics=[output_rows=",
+            ],
+            "Filtered analyze plan should preserve late materialization and filter pushdown",
+        );
+    }
 
-            // Describe table should return managed_versioning=true
-            let mut describe_req = DescribeTableRequest::new();
-            describe_req.id = Some(vec!["workspace".to_string(), "test_table".to_string()]);
-            let describe_resp = ns.describe_table(describe_req).await.unwrap();
+    #[tokio::test]
+    #[cfg_attr(
+        windows,
+        ignore = "TODO: https://github.com/lance-format/lance/issues/6557"
+    )]
+    async fn test_dir_listing_no_extra_calls_without_migration() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let root_uri = file_object_store_uri(temp_path);
+        let listing_count = Arc::new(AtomicUsize::new(0));
+        let session = build_listing_counting_session(listing_count.clone());
 
-            // managed_versioning should be true
-            assert_eq!(
-                describe_resp.managed_versioning,
-                Some(true),
-                "managed_versioning should be true when table_version_tracking_enabled=true"
-            );
-        }
+        // Create a table using dir-listing-only namespace
+        let dir_only_ns = DirectoryNamespaceBuilder::new(root_uri.clone())
+            .session(session.clone())
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
 
-        #[tokio::test]
-        #[cfg(not(windows))]
-        async fn test_external_manifest_store_invokes_namespace_apis() {
-            use arrow::array::{Int32Array, StringArray};
-            use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
-            use arrow::record_batch::RecordBatch;
-            use lance::Dataset;
-            use lance::dataset::builder::DatasetBuilder;
-            use lance::dataset::{WriteMode, WriteParams};
-            use lance_namespace::models::CreateNamespaceRequest;
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["test_table".to_string()]);
+        dir_only_ns
+            .create_table(create_req, Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        // Build a namespace with both enabled but migration disabled (default)
+        let hybrid_ns = DirectoryNamespaceBuilder::new(root_uri)
+            .session(session)
+            .manifest_enabled(true)
+            .dir_listing_enabled(true)
+            .dir_listing_to_manifest_migration_enabled(false)
+            .build()
+            .await
+            .unwrap();
 
-            let temp_dir = TempStdDir::default();
-            let temp_path = temp_dir.to_str().unwrap();
+        // Reset counter before the operation we want to measure
+        listing_count.store(0, Ordering::SeqCst);
 
-            // Create namespace with table_version_tracking_enabled and manifest_enabled
-            let inner_ns = DirectoryNamespaceBuilder::new(temp_path)
-                .table_version_tracking_enabled(true)
-                .manifest_enabled(true)
-                .build()
-                .await
-                .unwrap();
+        // table_exists should use dir listing directly, making only 1 listing call
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["test_table".to_string()]);
+        hybrid_ns.table_exists(exists_req).await.unwrap();
 
-            let tracking_ns = Arc::new(TrackingNamespace::new(inner_ns));
-            let ns: Arc<dyn LanceNamespace> = tracking_ns.clone();
+        let count = listing_count.load(Ordering::SeqCst);
+        assert_eq!(
+            count, 1,
+            "Expected exactly 1 listing call for table_exists \
+             without migration mode, but got {}",
+            count
+        );
 
-            // Create parent namespace
-            let mut create_ns_req = CreateNamespaceRequest::new();
-            create_ns_req.id = Some(vec!["workspace".to_string()]);
-            ns.create_namespace(create_ns_req).await.unwrap();
+        // Reset and test describe_table
+        listing_count.store(0, Ordering::SeqCst);
 
-            // Create a table with multi-level ID (namespace + table)
-            let table_id = vec!["workspace".to_string(), "test_table".to_string()];
+        let mut describe_req = DescribeTableRequest::new();
+        describe_req.id = Some(vec!["test_table".to_string()]);
+        hybrid_ns.describe_table(describe_req).await.unwrap();
 
-            // Create some initial data
-            let arrow_schema = Arc::new(ArrowSchema::new(vec![
-                Field::new("id", DataType::Int32, false),
-                Field::new("name", DataType::Utf8, true),
-            ]));
-            let batch = RecordBatch::try_new(
-                arrow_schema.clone(),
-                vec![
-                    Arc::new(Int32Array::from(vec![1, 2, 3])),
-                    Arc::new(StringArray::from(vec!["a", "b", "c"])),
-                ],
-            )
-            .unwrap();
+        let count = listing_count.load(Ordering::SeqCst);
+        assert_eq!(
+            count, 1,
+            "Expected exactly 1 listing call for describe_table \
+             without migration mode, but got {}",
+            count
+        );
+    }
 
-            // Create a table using write_into_namespace
-            let batches = RecordBatchIterator::new(vec![Ok(batch.clone())], arrow_schema.clone());
-            let write_params = WriteParams {
-                mode: WriteMode::Create,
-                ..Default::default()
-            };
-            let mut dataset = Dataset::write_into_namespace(
-                batches,
-                ns.clone(),
-                table_id.clone(),
-                Some(write_params),
-            )
+    #[tokio::test]
+    #[cfg_attr(
+        windows,
+        ignore = "TODO: https://github.com/lance-format/lance/issues/6557"
+    )]
+    async fn test_dir_listing_extra_calls_with_migration() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let root_uri = file_object_store_uri(temp_path);
+        let listing_count = Arc::new(AtomicUsize::new(0));
+        let session = build_listing_counting_session(listing_count.clone());
+
+        // Create a table using dir-listing-only namespace so it exists physically but is absent from __manifest.
+        let dir_only_ns = DirectoryNamespaceBuilder::new(root_uri.clone())
+            .session(session.clone())
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
+            .build()
             .await
             .unwrap();
-            assert_eq!(dataset.version().version, 1);
-
-            // Verify create_table_version was called once during initial write_into_namespace
-            assert_eq!(
-                tracking_ns.create_table_version_calls(),
-                1,
-                "create_table_version should have been called once during initial write_into_namespace"
-            );
 
-            // Append data - this should call create_table_version again
-            let append_batch = RecordBatch::try_new(
-                arrow_schema.clone(),
-                vec![
-                    Arc::new(Int32Array::from(vec![4, 5, 6])),
-                    Arc::new(StringArray::from(vec!["d", "e", "f"])),
-                ],
-            )
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec!["test_table".to_string()]);
+        dir_only_ns
+            .create_table(create_req, Bytes::from(ipc_data))
+            .await
             .unwrap();
-            let append_batches = RecordBatchIterator::new(vec![Ok(append_batch)], arrow_schema);
-            dataset.append(append_batches, None).await.unwrap();
 
-            assert_eq!(
-                tracking_ns.create_table_version_calls(),
-                2,
-                "create_table_version should have been called twice (once for create, once for append)"
-            );
+        let hybrid_ns = DirectoryNamespaceBuilder::new(root_uri)
+            .session(session)
+            .manifest_enabled(true)
+            .dir_listing_enabled(true)
+            .dir_listing_to_manifest_migration_enabled(true)
+            .build()
+            .await
+            .unwrap();
 
-            // checkout_latest should call list_table_versions exactly once
-            let initial_list_calls = tracking_ns.list_table_versions_calls();
-            let latest_dataset = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
-                .await
-                .unwrap()
-                .load()
-                .await
-                .unwrap();
-            assert_eq!(latest_dataset.version().version, 2);
-            assert_eq!(
-                tracking_ns.list_table_versions_calls(),
-                initial_list_calls + 1,
-                "list_table_versions should have been called exactly once during checkout_latest"
-            );
+        // table_exists first checks __manifest (one list on __manifest/_versions),
+        // then falls back to the table directory (one list_with_delimiter on test_table.lance).
+        listing_count.store(0, Ordering::SeqCst);
 
-            // checkout to specific version should call describe_table_version exactly once
-            let initial_describe_calls = tracking_ns.describe_table_version_calls();
-            let v1_dataset = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
-                .await
-                .unwrap()
-                .with_version(1)
-                .load()
-                .await
-                .unwrap();
-            assert_eq!(v1_dataset.version().version, 1);
-            assert_eq!(
-                tracking_ns.describe_table_version_calls(),
-                initial_describe_calls + 1,
-                "describe_table_version should have been called exactly once during checkout to version 1"
-            );
-        }
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["test_table".to_string()]);
+        hybrid_ns.table_exists(exists_req).await.unwrap();
 
-        #[tokio::test]
-        #[cfg(not(windows))]
-        async fn test_dataset_commit_with_external_manifest_store() {
-            use arrow::array::{Int32Array, StringArray};
-            use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
-            use arrow::record_batch::RecordBatch;
-            use futures::TryStreamExt;
-            use lance::dataset::{Dataset, WriteMode, WriteParams};
-            use lance_namespace::models::CreateNamespaceRequest;
-            use lance_table::io::commit::ManifestNamingScheme;
+        let count = listing_count.load(Ordering::SeqCst);
+        assert_eq!(
+            count, 2,
+            "Expected exactly 2 listing calls for table_exists with migration mode \
+             (manifest reload + table directory fallback), but got {}",
+            count
+        );
 
-            let temp_dir = TempStdDir::default();
-            let temp_path = temp_dir.to_str().unwrap();
+        // describe_table follows the same path when the table is not yet registered in __manifest.
+        listing_count.store(0, Ordering::SeqCst);
 
-            // Create namespace with table_version_tracking_enabled and manifest_enabled
-            let inner_ns = DirectoryNamespaceBuilder::new(temp_path)
-                .table_version_tracking_enabled(true)
-                .manifest_enabled(true)
-                .build()
-                .await
-                .unwrap();
+        let mut describe_req = DescribeTableRequest::new();
+        describe_req.id = Some(vec!["test_table".to_string()]);
+        hybrid_ns.describe_table(describe_req).await.unwrap();
 
-            let tracking_ns: Arc<dyn LanceNamespace> = Arc::new(TrackingNamespace::new(inner_ns));
+        let count = listing_count.load(Ordering::SeqCst);
+        assert_eq!(
+            count, 2,
+            "Expected exactly 2 listing calls for describe_table with migration mode \
+             (manifest reload + table directory fallback), but got {}",
+            count
+        );
+    }
 
-            // Create parent namespace
-            let mut create_ns_req = CreateNamespaceRequest::new();
-            create_ns_req.id = Some(vec!["workspace".to_string()]);
-            tracking_ns.create_namespace(create_ns_req).await.unwrap();
+    #[tokio::test]
+    async fn test_migration_not_found_errors_include_table_id() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
 
-            // Create a table using write_into_namespace
-            let table_id = vec!["workspace".to_string(), "test_table".to_string()];
-            let arrow_schema = Arc::new(ArrowSchema::new(vec![
-                Field::new("id", DataType::Int32, false),
-                Field::new("name", DataType::Utf8, true),
-            ]));
-            let batch = RecordBatch::try_new(
-                arrow_schema.clone(),
-                vec![
-                    Arc::new(Int32Array::from(vec![1, 2, 3])),
-                    Arc::new(StringArray::from(vec!["a", "b", "c"])),
-                ],
-            )
-            .unwrap();
-            let batches = RecordBatchIterator::new(vec![Ok(batch)], arrow_schema.clone());
-            let write_params = WriteParams {
-                mode: WriteMode::Create,
-                ..Default::default()
-            };
-            let dataset = Dataset::write_into_namespace(
-                batches,
-                tracking_ns.clone(),
-                table_id.clone(),
-                Some(write_params),
-            )
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(true)
+            .dir_listing_enabled(true)
+            .dir_listing_to_manifest_migration_enabled(true)
+            .build()
             .await
             .unwrap();
-            assert_eq!(dataset.version().version, 1);
 
-            // Append data using write_into_namespace (APPEND mode)
-            let batch2 = RecordBatch::try_new(
-                arrow_schema.clone(),
-                vec![
-                    Arc::new(Int32Array::from(vec![4, 5, 6])),
-                    Arc::new(StringArray::from(vec!["d", "e", "f"])),
-                ],
-            )
-            .unwrap();
-            let batches = RecordBatchIterator::new(vec![Ok(batch2)], arrow_schema);
-            let write_params = WriteParams {
-                mode: WriteMode::Append,
-                ..Default::default()
-            };
-            Dataset::write_into_namespace(
-                batches,
-                tracking_ns.clone(),
-                table_id.clone(),
-                Some(write_params),
-            )
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["missing_table".to_string()]);
+        let err = namespace.table_exists(exists_req).await.unwrap_err();
+        assert!(matches!(err, Error::Namespace { .. }));
+        let err_msg = err.to_string();
+        assert!(err_msg.contains("Table not found"));
+        assert!(err_msg.contains("table id 'missing_table'"));
+
+        let mut describe_req = DescribeTableRequest::new();
+        describe_req.id = Some(vec!["missing_table".to_string()]);
+        let err = namespace.describe_table(describe_req).await.unwrap_err();
+        assert!(matches!(err, Error::Namespace { .. }));
+        let err_msg = err.to_string();
+        assert!(err_msg.contains("Table not found"));
+        assert!(err_msg.contains("table id 'missing_table'"));
+    }
+
+    #[tokio::test]
+    async fn test_manifest_not_found_errors_include_full_table_id() {
+        use lance_namespace::models::CreateNamespaceRequest;
+
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(true)
+            .dir_listing_enabled(true)
+            .build()
             .await
             .unwrap();
 
-            // Verify version 2 was created using the dataset's object_store
-            // List manifests in the versions directory to find the V2 named manifest
-            let manifest_metas: Vec<_> = dataset
-                .object_store()
-                .inner
-                .list(Some(&dataset.versions_dir()))
-                .try_collect()
-                .await
-                .unwrap();
-            let version_2_found = manifest_metas.iter().any(|m| {
-                m.location
-                    .filename()
-                    .map(|f| {
-                        f.ends_with(".manifest")
-                            && ManifestNamingScheme::V2.parse_version(f) == Some(2)
-                    })
-                    .unwrap_or(false)
-            });
-            assert!(
-                version_2_found,
-                "Version 2 manifest should exist in versions directory"
-            );
-        }
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["workspace".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
+
+        let missing_table_id = vec!["workspace".to_string(), "missing_table".to_string()];
+
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(missing_table_id.clone());
+        let err = namespace.table_exists(exists_req).await.unwrap_err();
+        assert!(matches!(err, Error::Namespace { .. }));
+        let err_msg = err.to_string();
+        assert!(err_msg.contains("Table not found"));
+        assert!(err_msg.contains("table id 'workspace$missing_table'"));
+
+        let mut describe_req = DescribeTableRequest::new();
+        describe_req.id = Some(missing_table_id);
+        let err = namespace.describe_table(describe_req).await.unwrap_err();
+        assert!(matches!(err, Error::Namespace { .. }));
+        let err_msg = err.to_string();
+        assert!(err_msg.contains("Table not found"));
+        assert!(err_msg.contains("table id 'workspace$missing_table'"));
     }
 }
diff --git a/rust/lance-namespace-impls/src/dir/manifest.rs b/rust/lance-namespace-impls/src/dir/manifest.rs
index 70ef7be7147..8a6a8ee0d85 100644
--- a/rust/lance-namespace-impls/src/dir/manifest.rs
+++ b/rust/lance-namespace-impls/src/dir/manifest.rs
@@ -6,6 +6,7 @@
 //! This module provides a namespace implementation that uses a manifest table
 //! to track tables and nested namespaces.
 
+use arrow::array::builder::{ListBuilder, StringBuilder};
 use arrow::array::{Array, RecordBatch, RecordBatchIterator, StringArray};
 use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
 use arrow_ipc::reader::StreamReader;
@@ -13,16 +14,19 @@ use async_trait::async_trait;
 use bytes::Bytes;
 use futures::{FutureExt, stream::StreamExt};
 use lance::dataset::optimize::{CompactionOptions, compact_files};
-use lance::dataset::{ReadParams, WriteParams, builder::DatasetBuilder};
+use lance::dataset::{
+    DeleteBuilder, MergeInsertBuilder, ReadParams, WhenMatched, WhenNotMatched, WriteParams,
+    builder::DatasetBuilder,
+};
+use lance::index::DatasetIndexExt;
 use lance::session::Session;
 use lance::{Dataset, dataset::scanner::Scanner};
 use lance_core::Error as LanceError;
 use lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY_POSITION;
-use lance_core::{Error, Result, box_error};
+use lance_core::{Error, Result};
 use lance_index::IndexType;
 use lance_index::optimize::OptimizeOptions;
 use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
-use lance_index::traits::DatasetIndexExt;
 use lance_io::object_store::{ObjectStore, ObjectStoreParams};
 use lance_namespace::LanceNamespace;
 use lance_namespace::error::NamespaceError;
@@ -30,10 +34,11 @@ use lance_namespace::models::{
     CreateNamespaceRequest, CreateNamespaceResponse, CreateTableRequest, CreateTableResponse,
     DeclareTableRequest, DeclareTableResponse, DeregisterTableRequest, DeregisterTableResponse,
     DescribeNamespaceRequest, DescribeNamespaceResponse, DescribeTableRequest,
-    DescribeTableResponse, DropNamespaceRequest, DropNamespaceResponse, DropTableRequest,
-    DropTableResponse, ListNamespacesRequest, ListNamespacesResponse, ListTablesRequest,
-    ListTablesResponse, NamespaceExistsRequest, RegisterTableRequest, RegisterTableResponse,
-    TableExistsRequest,
+    DescribeTableResponse, DescribeTableVersionResponse, DropNamespaceRequest,
+    DropNamespaceResponse, DropTableRequest, DropTableResponse, ListNamespacesRequest,
+    ListNamespacesResponse, ListTableVersionsResponse, ListTablesRequest, ListTablesResponse,
+    NamespaceExistsRequest, RegisterTableRequest, RegisterTableResponse, TableExistsRequest,
+    TableVersion,
 };
 use lance_namespace::schema::arrow_schema_to_json;
 use object_store::path::Path;
@@ -44,7 +49,7 @@ use std::{
     ops::{Deref, DerefMut},
     sync::Arc,
 };
-use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
+use tokio::sync::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};
 
 const MANIFEST_TABLE_NAME: &str = "__manifest";
 const DELIMITER: &str = "$";
@@ -56,12 +61,16 @@ const OBJECT_ID_INDEX_NAME: &str = "object_id_btree";
 const OBJECT_TYPE_INDEX_NAME: &str = "object_type_bitmap";
 /// LabelList index on the base_objects column for view dependencies
 const BASE_OBJECTS_INDEX_NAME: &str = "base_objects_label_list";
+/// Inline maintenance on the manifest table is expensive relative to a single-row mutation.
+/// Wait until enough fragments accumulate before compacting files or merging indices.
+const MANIFEST_INLINE_OPTIMIZATION_FRAGMENT_THRESHOLD: usize = 8;
 
 /// Object types that can be stored in the manifest
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum ObjectType {
     Namespace,
     Table,
+    TableVersion,
 }
 
 impl ObjectType {
@@ -69,6 +78,7 @@ impl ObjectType {
         match self {
             Self::Namespace => "namespace",
             Self::Table => "table",
+            Self::TableVersion => "table_version",
         }
     }
 
@@ -76,7 +86,11 @@ impl ObjectType {
         match s {
             "namespace" => Ok(Self::Namespace),
             "table" => Ok(Self::Table),
-            _ => Err(Error::io(format!("Invalid object type: {}", s))),
+            "table_version" => Ok(Self::TableVersion),
+            _ => Err(NamespaceError::Internal {
+                message: format!("Invalid object type: {}", s),
+            }
+            .into()),
         }
     }
 }
@@ -89,6 +103,22 @@ pub struct TableInfo {
     pub location: String,
 }
 
+/// An entry to be inserted into the manifest table.
+///
+/// This struct makes the meaning of each field explicit, replacing the
+/// previous tuple-based API `(String, ObjectType, Option<String>, Option<String>)`.
+#[derive(Debug, Clone)]
+pub struct ManifestEntry {
+    /// The unique object identifier (e.g., table name or version object_id)
+    pub object_id: String,
+    /// The type of the object (Namespace, Table, or TableVersion)
+    pub object_type: ObjectType,
+    /// The storage location (e.g., directory name for tables)
+    pub location: Option<String>,
+    /// Additional metadata serialized as JSON
+    pub metadata: Option<String>,
+}
+
 /// Information about a namespace stored in the manifest
 #[derive(Debug, Clone)]
 pub struct NamespaceInfo {
@@ -151,10 +181,9 @@ impl DatasetConsistencyWrapper {
             current_version
         );
         let latest_version = read_guard.latest_version_id().await.map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to get latest version: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to get latest version: {:?}", e),
+            })
         })?;
         log::debug!(
             "Reload got latest_version={} for uri={}, current_version={}",
@@ -175,18 +204,16 @@ impl DatasetConsistencyWrapper {
 
         // Double-check after acquiring write lock (someone else might have reloaded)
         let latest_version = write_guard.latest_version_id().await.map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to get latest version: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to get latest version: {:?}", e),
+            })
         })?;
 
         if latest_version != write_guard.version().version {
             write_guard.checkout_latest().await.map_err(|e| {
-                Error::io_source(box_error(std::io::Error::other(format!(
-                    "Failed to checkout latest: {}",
-                    e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to checkout latest: {:?}", e),
+                })
             })?;
         }
 
@@ -230,11 +257,8 @@ impl DerefMut for DatasetWriteGuard<'_> {
 pub struct ManifestNamespace {
     root: String,
     storage_options: Option<HashMap<String, String>>,
-    #[allow(dead_code)]
     session: Option<Arc<Session>>,
-    #[allow(dead_code)]
     object_store: Arc<ObjectStore>,
-    #[allow(dead_code)]
     base_path: Path,
     manifest_dataset: DatasetConsistencyWrapper,
     /// Whether directory listing is enabled in dual mode
@@ -247,6 +271,9 @@ pub struct ManifestNamespace {
     /// Number of retries for commit operations on the manifest table.
     /// If None, defaults to [`lance_table::io::commit::CommitConfig`] default (20).
     commit_retries: Option<u32>,
+    /// Serialize manifest mutations within a single namespace instance so concurrent
+    /// create/drop calls do not compete with each other on the same in-memory snapshot.
+    manifest_mutation_lock: Arc<Mutex<()>>,
 }
 
 impl std::fmt::Debug for ManifestNamespace {
@@ -266,15 +293,15 @@ impl std::fmt::Debug for ManifestNamespace {
 /// Convert a Lance commit error to an appropriate namespace error.
 ///
 /// Maps lance commit errors to namespace errors:
-/// - `CommitConflict`: version collision retries exhausted -> Throttled (safe to retry)
+/// - `CommitConflict`: version collision retries exhausted -> Throttling (safe to retry)
 /// - `TooMuchWriteContention`: RetryableCommitConflict (semantic conflict) retries exhausted -> ConcurrentModification
 /// - `IncompatibleTransaction`: incompatible concurrent change -> ConcurrentModification
 /// - Errors containing "matched/duplicate/already exists": ConcurrentModification (from WhenMatched::Fail)
 /// - Other errors: IO error with the operation description
 fn convert_lance_commit_error(e: &LanceError, operation: &str, object_id: Option<&str>) -> Error {
     match e {
-        // CommitConflict: version collision retries exhausted -> Throttled (safe to retry)
-        LanceError::CommitConflict { .. } => NamespaceError::Throttled {
+        // CommitConflict: version collision retries exhausted -> Throttling (safe to retry)
+        LanceError::CommitConflict { .. } => NamespaceError::Throttling {
             message: format!("Too many concurrent writes, please retry later: {:?}", e),
         }
         .into(),
@@ -314,10 +341,9 @@ fn convert_lance_commit_error(e: &LanceError, operation: &str, object_id: Option
                 };
                 return NamespaceError::ConcurrentModification { message }.into();
             }
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "{}: {:?}",
-                operation, e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("{}: {:?}", operation, e),
+            })
         }
     }
 }
@@ -334,10 +360,15 @@ impl ManifestNamespace {
         dir_listing_enabled: bool,
         inline_optimization_enabled: bool,
         commit_retries: Option<u32>,
+        table_version_storage_enabled: bool,
     ) -> Result<Self> {
-        let manifest_dataset =
-            Self::ensure_manifest_table_up_to_date(&root, &storage_options, session.clone())
-                .await?;
+        let manifest_dataset = Self::ensure_manifest_table_up_to_date(
+            &root,
+            &storage_options,
+            session.clone(),
+            table_version_storage_enabled,
+        )
+        .await?;
 
         Ok(Self {
             root,
@@ -349,6 +380,7 @@ impl ManifestNamespace {
             dir_listing_enabled,
             inline_optimization_enabled,
             commit_retries,
+            manifest_mutation_lock: Arc::new(Mutex::new(())),
         })
     }
 
@@ -379,30 +411,62 @@ impl ManifestNamespace {
         }
     }
 
-    /// Split an object ID (table_id as vec of strings) into namespace and table name
-    fn split_object_id(table_id: &[String]) -> (Vec<String>, String) {
-        if table_id.len() == 1 {
-            (vec![], table_id[0].clone())
+    /// Split an object ID (vec of strings) into namespace and table name
+    pub fn split_object_id(object_id: &[String]) -> (Vec<String>, String) {
+        if object_id.len() == 1 {
+            (vec![], object_id[0].clone())
         } else {
             (
-                table_id[..table_id.len() - 1].to_vec(),
-                table_id[table_id.len() - 1].clone(),
+                object_id[..object_id.len() - 1].to_vec(),
+                object_id[object_id.len() - 1].clone(),
             )
         }
     }
 
-    /// Convert a table ID (vec of strings) to an object_id string
-    fn str_object_id(table_id: &[String]) -> String {
-        table_id.join(DELIMITER)
+    /// Convert an ID (vec of strings) to an object_id string
+    pub fn str_object_id(object_id: &[String]) -> String {
+        object_id.join(DELIMITER)
+    }
+
+    fn format_table_id(table_id: &[String]) -> String {
+        format!("table id '{}'", Self::str_object_id(table_id))
+    }
+
+    /// Format a version number as a zero-padded lexicographically sortable string.
+    ///
+    /// Versions are stored as 20-digit zero-padded integers (e.g., `00000000000000000001`
+    /// for version 1) so that string-based range queries and sorting work correctly.
+    pub fn format_table_version(version: i64) -> String {
+        format!("{:020}", version)
+    }
+
+    /// Build the object_id for a table version entry.
+    ///
+    /// Format: `{table_object_id}${zero_padded_version}`
+    pub fn build_version_object_id(table_object_id: &str, version: i64) -> String {
+        format!(
+            "{}{}{}",
+            table_object_id,
+            DELIMITER,
+            Self::format_table_version(version)
+        )
+    }
+
+    /// Parse a version number from the version suffix of a table version object_id.
+    ///
+    /// The object_id is formatted as `{table_id}${zero_padded_version}`.
+    pub fn parse_version_from_object_id(object_id: &str) -> Option<i64> {
+        let (_namespace, name) = Self::parse_object_id(object_id);
+        name.parse::<i64>().ok()
     }
 
-    /// Generate a new directory name in format: <hash>_<object_id>
+    /// Generate a new directory name in format: `<hash>_<object_id>`
     /// The hash is used to (1) optimize object store throughput,
     /// (2) have high enough entropy in a short period of time to prevent issues like
     /// failed table creation, delete and create new table of the same name, etc.
     /// The object_id is added after the hash to ensure
     /// dir name uniqueness and make debugging easier.
-    fn generate_dir_name(object_id: &str) -> String {
+    pub fn generate_dir_name(object_id: &str) -> String {
         // Generate a random number for uniqueness
         let random_num: u64 = rand::random();
 
@@ -428,16 +492,20 @@ impl ManifestNamespace {
             base_url.set_path(&format!("{}/", base_url.path()));
         }
 
-        let full_url = base_url.join(relative_location).map_err(|e| {
-            Error::invalid_input_source(
-                format!(
+        let mut full_url = base_url.join(relative_location).map_err(|e| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: format!(
                     "Failed to join URI '{}' with '{}': {:?}",
                     root, relative_location, e
-                )
-                .into(),
-            )
+                ),
+            })
         })?;
 
+        // Clear any query string to avoid trailing "?" in the URL.
+        // Use set_query(None) instead of set_query("") because the latter
+        // would still add a trailing '?' to the URL when serialized.
+        full_url.set_query(None);
+
         Ok(full_url.to_string())
     }
 
@@ -558,6 +626,13 @@ impl ManifestNamespace {
             }
         }
 
+        let should_compact_and_optimize =
+            dataset.count_fragments() >= MANIFEST_INLINE_OPTIMIZATION_FRAGMENT_THRESHOLD;
+
+        if !should_compact_and_optimize {
+            return Ok(());
+        }
+
         // Step 2: Run file compaction
         log::debug!("Running file compaction on __manifest table");
         match compact_files(dataset, CompactionOptions::default(), None).await {
@@ -627,19 +702,17 @@ impl ManifestNamespace {
     /// Helper to execute a scanner and collect results into a Vec
     async fn execute_scanner(scanner: Scanner) -> Result<Vec<RecordBatch>> {
         let mut stream = scanner.try_into_stream().await.map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to create stream: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to create stream: {:?}", e),
+            })
         })?;
 
         let mut batches = Vec::new();
         while let Some(batch) = stream.next().await {
             batches.push(batch.map_err(|e| {
-                Error::io_source(box_error(std::io::Error::other(format!(
-                    "Failed to read batch: {}",
-                    e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to read batch: {:?}", e),
+                })
             })?);
         }
 
@@ -648,44 +721,48 @@ impl ManifestNamespace {
 
     /// Helper to get a string column from a record batch
     fn get_string_column<'a>(batch: &'a RecordBatch, column_name: &str) -> Result<&'a StringArray> {
-        let column = batch
-            .column_by_name(column_name)
-            .ok_or_else(|| Error::io(format!("Column '{}' not found", column_name)))?;
+        let column = batch.column_by_name(column_name).ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Column '{}' not found", column_name),
+            })
+        })?;
         column
             .as_any()
             .downcast_ref::<StringArray>()
-            .ok_or_else(|| Error::io(format!("Column '{}' is not a string array", column_name)))
+            .ok_or_else(|| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Column '{}' is not a string array", column_name),
+                })
+            })
     }
 
     /// Check if the manifest contains an object with the given ID
     async fn manifest_contains_object(&self, object_id: &str) -> Result<bool> {
-        let filter = format!("object_id = '{}'", object_id);
+        let escaped_id = object_id.replace('\'', "''");
+        let filter = format!("object_id = '{}'", escaped_id);
 
         let dataset_guard = self.manifest_dataset.get().await?;
         let mut scanner = dataset_guard.scan();
 
         scanner.filter(&filter).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to filter: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
         })?;
 
         // Project no columns and enable row IDs for count_rows to work
         scanner.project::<&str>(&[]).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to project: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to project: {:?}", e),
+            })
         })?;
 
         scanner.with_row_id();
 
         let count = scanner.count_rows().await.map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to count rows: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to count rows: {:?}", e),
+            })
         })?;
 
         Ok(count > 0)
@@ -693,19 +770,18 @@ impl ManifestNamespace {
 
     /// Query the manifest for a table with the given object ID
     async fn query_manifest_for_table(&self, object_id: &str) -> Result<Option<TableInfo>> {
-        let filter = format!("object_id = '{}' AND object_type = 'table'", object_id);
+        let escaped_id = object_id.replace('\'', "''");
+        let filter = format!("object_id = '{}' AND object_type = 'table'", escaped_id);
         let mut scanner = self.manifest_scanner().await?;
         scanner.filter(&filter).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to filter: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
         })?;
         scanner.project(&["object_id", "location"]).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to project: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to project: {:?}", e),
+            })
         })?;
         let batches = Self::execute_scanner(scanner).await?;
 
@@ -719,10 +795,13 @@ impl ManifestNamespace {
 
             total_rows += batch.num_rows();
             if total_rows > 1 {
-                return Err(Error::io(format!(
-                    "Expected exactly 1 table with id '{}', found {}",
-                    object_id, total_rows
-                )));
+                return Err(NamespaceError::Internal {
+                    message: format!(
+                        "Expected exactly 1 table with id '{}', found {}",
+                        object_id, total_rows
+                    ),
+                }
+                .into());
             }
 
             let object_id_array = Self::get_string_column(&batch, "object_id")?;
@@ -745,16 +824,14 @@ impl ManifestNamespace {
         let filter = "object_type = 'table' AND NOT contains(object_id, '$')";
         let mut scanner = self.manifest_scanner().await?;
         scanner.filter(filter).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to filter: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
         })?;
         scanner.project(&["location"]).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to project: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to project: {:?}", e),
+            })
         })?;
 
         let batches = Self::execute_scanner(scanner).await?;
@@ -780,24 +857,39 @@ impl ManifestNamespace {
         object_type: ObjectType,
         location: Option<String>,
     ) -> Result<()> {
-        self.insert_into_manifest_with_metadata(object_id, object_type, location, None, None)
-            .await
+        self.insert_into_manifest_with_metadata(
+            vec![ManifestEntry {
+                object_id,
+                object_type,
+                location,
+                metadata: None,
+            }],
+            None,
+        )
+        .await
     }
 
-    /// Insert an entry into the manifest table with metadata and base_objects
-    async fn insert_into_manifest_with_metadata(
+    /// Insert one or more entries into the manifest table with metadata and base_objects.
+    ///
+    /// This is the unified entry point for both single and batch inserts.
+    /// Uses a single MergeInsert operation to insert all entries at once.
+    /// If any entry already exists (matching object_id), the entire batch fails.
+    pub async fn insert_into_manifest_with_metadata(
         &self,
-        object_id: String,
-        object_type: ObjectType,
-        location: Option<String>,
-        metadata: Option<String>,
+        entries: Vec<ManifestEntry>,
         base_objects: Option<Vec<String>>,
     ) -> Result<()> {
-        use arrow::array::builder::{ListBuilder, StringBuilder};
+        if entries.is_empty() {
+            return Ok(());
+        }
 
         let schema = Self::manifest_schema();
 
-        // Create base_objects array from the provided list
+        let mut object_ids = Vec::with_capacity(entries.len());
+        let mut object_types = Vec::with_capacity(entries.len());
+        let mut locations: Vec<Option<String>> = Vec::with_capacity(entries.len());
+        let mut metadatas: Vec<Option<String>> = Vec::with_capacity(entries.len());
+
         let string_builder = StringBuilder::new();
         let mut list_builder = ListBuilder::new(string_builder).with_field(Arc::new(Field::new(
             "object_id",
@@ -805,64 +897,81 @@ impl ManifestNamespace {
             true,
         )));
 
-        match base_objects {
-            Some(objects) => {
-                for obj in objects {
-                    list_builder.values().append_value(obj);
+        for (i, entry) in entries.iter().enumerate() {
+            object_ids.push(entry.object_id.as_str());
+            object_types.push(entry.object_type.as_str());
+            locations.push(entry.location.clone());
+            metadatas.push(entry.metadata.clone());
+
+            // Only the first entry gets the base_objects (for single-entry inserts
+            // with base_objects like view creation); batch entries use null.
+            if i == 0 {
+                match &base_objects {
+                    Some(objects) => {
+                        for obj in objects {
+                            list_builder.values().append_value(obj);
+                        }
+                        list_builder.append(true);
+                    }
+                    None => {
+                        list_builder.append_null();
+                    }
                 }
-                list_builder.append(true);
-            }
-            None => {
+            } else {
                 list_builder.append_null();
             }
         }
 
         let base_objects_array = list_builder.finish();
 
-        // Create arrays with optional values
-        let location_array = match location {
-            Some(loc) => Arc::new(StringArray::from(vec![Some(loc)])),
-            None => Arc::new(StringArray::from(vec![None::<String>])),
-        };
+        let location_array: Arc<dyn Array> = Arc::new(StringArray::from(
+            locations.iter().map(|l| l.as_deref()).collect::<Vec<_>>(),
+        ));
 
-        let metadata_array = match metadata {
-            Some(meta) => Arc::new(StringArray::from(vec![Some(meta)])),
-            None => Arc::new(StringArray::from(vec![None::<String>])),
-        };
+        let metadata_array: Arc<dyn Array> = Arc::new(StringArray::from(
+            metadatas.iter().map(|m| m.as_deref()).collect::<Vec<_>>(),
+        ));
 
         let batch = RecordBatch::try_new(
             schema.clone(),
             vec![
-                Arc::new(StringArray::from(vec![object_id.as_str()])),
-                Arc::new(StringArray::from(vec![object_type.as_str()])),
+                Arc::new(StringArray::from(object_ids)),
+                Arc::new(StringArray::from(object_types.to_vec())),
                 location_array,
                 metadata_array,
                 Arc::new(base_objects_array),
             ],
         )
-        .map_err(|e| Error::io(format!("Failed to create manifest entry: {}", e)))?;
+        .map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to create manifest entries: {:?}", e),
+            })
+        })?;
 
         let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
 
         // Use MergeInsert to ensure uniqueness on object_id
+        let _mutation_guard = self.manifest_mutation_lock.lock().await;
         let dataset_guard = self.manifest_dataset.get().await?;
         let dataset_arc = Arc::new(dataset_guard.clone());
         drop(dataset_guard); // Drop read guard before merge insert
 
         let mut merge_builder =
-            lance::dataset::MergeInsertBuilder::try_new(dataset_arc, vec!["object_id".to_string()])
-                .map_err(|e| {
-                    Error::io_source(box_error(std::io::Error::other(format!(
-                        "Failed to create merge builder: {}",
-                        e
-                    ))))
-                })?;
-
-        merge_builder.when_matched(lance::dataset::WhenMatched::Fail);
-        merge_builder.when_not_matched(lance::dataset::WhenNotMatched::InsertAll);
-        // conflict_retries=0: no outer loop retry on semantic conflicts (handled by caller)
-        // commit_retries: inner retry for manifest version conflicts (uses lance default if not set)
-        merge_builder.conflict_retries(0);
+            MergeInsertBuilder::try_new(dataset_arc, vec!["object_id".to_string()]).map_err(
+                |e| {
+                    lance_core::Error::from(NamespaceError::Internal {
+                        message: format!("Failed to create merge builder: {:?}", e),
+                    })
+                },
+            )?;
+        merge_builder.when_matched(WhenMatched::Fail);
+        merge_builder.when_not_matched(WhenNotMatched::InsertAll);
+        // Use conflict_retries to handle cross-process races on manifest mutations.
+        // When two processes concurrently insert the same object_id, the second one
+        // hits a commit conflict. With conflict_retries > 0, the retry re-evaluates
+        // the full MergeInsert plan against the latest data, where the join detects
+        // the existing row and WhenMatched::Fail fires, producing a clear error.
+        merge_builder.conflict_retries(5);
         // TODO: after BTREE index creation on object_id, has_scalar_index=true causes
         // MergeInsert to use V1 path which lacks bloom filters for conflict detection. This
         // results in (Some, None) filter mismatch when rebasing against V2 operations.
@@ -875,15 +984,14 @@ impl ManifestNamespace {
         let (new_dataset_arc, _merge_stats) = merge_builder
             .try_build()
             .map_err(|e| {
-                Error::io_source(box_error(std::io::Error::other(format!(
-                    "Failed to build merge: {}",
-                    e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to build merge: {:?}", e),
+                })
             })?
             .execute_reader(Box::new(reader))
             .await
             .map_err(|e| {
-                convert_lance_commit_error(&e, "Failed to execute merge", Some(&object_id))
+                convert_lance_commit_error(&e, "Failed to execute merge insert into manifest", None)
             })?;
 
         let new_dataset = Arc::try_unwrap(new_dataset_arc).unwrap_or_else(|arc| (*arc).clone());
@@ -905,11 +1013,12 @@ impl ManifestNamespace {
         let predicate = format!("object_id = '{}'", object_id);
 
         // Get dataset and use DeleteBuilder with configured retries
+        let _mutation_guard = self.manifest_mutation_lock.lock().await;
         let dataset_guard = self.manifest_dataset.get().await?;
         let dataset = Arc::new(dataset_guard.clone());
         drop(dataset_guard); // Drop read guard before delete
 
-        let new_dataset = lance::dataset::DeleteBuilder::new(dataset, &predicate)
+        let new_dataset = DeleteBuilder::new(dataset, &predicate)
             .execute()
             .await
             .map_err(|e| convert_lance_commit_error(&e, "Failed to delete", None))?;
@@ -932,11 +1041,417 @@ impl ManifestNamespace {
         Ok(())
     }
 
+    /// Query the manifest for all versions of a table, sorted by version.
+    ///
+    /// Returns a list of (version, metadata_json_string) tuples where metadata_json_string
+    /// contains the full metadata JSON stored in the manifest (manifest_path, manifest_size,
+    /// e_tag, naming_scheme).
+    ///
+    /// **Known limitation**: All matching rows are loaded into memory, sorted in Rust,
+    /// and then truncated. For tables with a very large number of versions this may be
+    /// expensive. Pushing sort/limit into the scan is not yet supported by Lance.
+    pub async fn query_table_versions(
+        &self,
+        object_id: &str,
+        descending: bool,
+        limit: Option<i32>,
+    ) -> Result<Vec<(i64, String)>> {
+        let escaped_id = object_id.replace('\'', "''");
+        // table_version object_ids are formatted as "{object_id}${zero_padded_version}"
+        let filter = format!(
+            "object_type = 'table_version' AND starts_with(object_id, '{}{}')",
+            escaped_id, DELIMITER
+        );
+        let mut scanner = self.manifest_scanner().await?;
+        scanner.filter(&filter).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
+        })?;
+        scanner.project(&["object_id", "metadata"]).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to project: {:?}", e),
+            })
+        })?;
+        let batches = Self::execute_scanner(scanner).await?;
+
+        let mut versions: Vec<(i64, String)> = Vec::new();
+        for batch in batches {
+            if batch.num_rows() == 0 {
+                continue;
+            }
+            let object_id_array = Self::get_string_column(&batch, "object_id")?;
+            let metadata_array = Self::get_string_column(&batch, "metadata")?;
+            for i in 0..batch.num_rows() {
+                let oid = object_id_array.value(i);
+                // Parse version from object_id
+                if let Some(version) = Self::parse_version_from_object_id(oid) {
+                    let metadata_str = metadata_array.value(i).to_string();
+                    versions.push((version, metadata_str));
+                }
+            }
+        }
+
+        if descending {
+            versions.sort_by(|a, b| b.0.cmp(&a.0));
+        } else {
+            versions.sort_by(|a, b| a.0.cmp(&b.0));
+        }
+
+        if let Some(limit) = limit {
+            versions.truncate(limit as usize);
+        }
+
+        Ok(versions)
+    }
+
+    /// Query the manifest for a specific version of a table.
+    ///
+    /// Returns the full metadata JSON string if found, which contains
+    /// manifest_path, manifest_size, e_tag, and naming_scheme.
+    ///
+    pub async fn query_table_version(
+        &self,
+        object_id: &str,
+        version: i64,
+    ) -> Result<Option<String>> {
+        let version_object_id = Self::build_version_object_id(object_id, version);
+        self.query_table_version_by_object_id(&version_object_id)
+            .await
+    }
+
+    /// Query a specific table version by its exact object_id.
+    async fn query_table_version_by_object_id(
+        &self,
+        version_object_id: &str,
+    ) -> Result<Option<String>> {
+        let escaped_id = version_object_id.replace('\'', "''");
+        let filter = format!(
+            "object_id = '{}' AND object_type = 'table_version'",
+            escaped_id
+        );
+        let mut scanner = self.manifest_scanner().await?;
+        scanner.filter(&filter).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
+        })?;
+        scanner.project(&["metadata"]).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to project: {:?}", e),
+            })
+        })?;
+        let batches = Self::execute_scanner(scanner).await?;
+
+        for batch in batches {
+            if batch.num_rows() == 0 {
+                continue;
+            }
+            let metadata_array = Self::get_string_column(&batch, "metadata")?;
+            return Ok(Some(metadata_array.value(0).to_string()));
+        }
+
+        Ok(None)
+    }
+
+    /// Delete table version entries from the manifest for a given table and version ranges.
+    ///
+    /// Each range is (start_version, end_version) inclusive. Deletes all matching
+    /// `object_type = 'table_version'` entries whose object_id matches
+    /// `{object_id}${zero_padded_version}`.
+    ///
+    /// Builds a single filter expression covering all version ranges and executes
+    /// one bulk delete operation instead of deleting versions one at a time.
+    pub async fn delete_table_versions(
+        &self,
+        object_id: &str,
+        ranges: &[(i64, i64)],
+    ) -> Result<i64> {
+        if ranges.is_empty() {
+            return Ok(0);
+        }
+
+        // Collect all object_ids to delete (both new zero-padded and legacy formats)
+        let mut object_id_conditions: Vec<String> = Vec::new();
+        for (start, end) in ranges {
+            for version in *start..=*end {
+                let oid = Self::build_version_object_id(object_id, version);
+                let escaped = oid.replace('\'', "''");
+                object_id_conditions.push(format!("'{}'", escaped));
+            }
+        }
+
+        if object_id_conditions.is_empty() {
+            return Ok(0);
+        }
+
+        // First, count how many entries exist so we can report the deleted count
+        let in_list = object_id_conditions.join(", ");
+        let filter = format!(
+            "object_type = 'table_version' AND object_id IN ({})",
+            in_list
+        );
+
+        let mut scanner = self.manifest_scanner().await?;
+        scanner.filter(&filter).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
+        })?;
+        scanner.project(&["object_id"]).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to project: {:?}", e),
+            })
+        })?;
+        let batches = Self::execute_scanner(scanner).await?;
+        let deleted_count: i64 = batches.iter().map(|b| b.num_rows() as i64).sum();
+
+        if deleted_count == 0 {
+            return Ok(0);
+        }
+
+        // Execute a single bulk delete with the combined filter
+        let _mutation_guard = self.manifest_mutation_lock.lock().await;
+        let dataset_guard = self.manifest_dataset.get().await?;
+        let dataset = Arc::new(dataset_guard.clone());
+        drop(dataset_guard);
+
+        let new_dataset = DeleteBuilder::new(dataset, &filter)
+            .execute()
+            .await
+            .map_err(|e| {
+                convert_lance_commit_error(&e, "Failed to batch delete table versions", None)
+            })?;
+
+        self.manifest_dataset
+            .set_latest(
+                Arc::try_unwrap(new_dataset.new_dataset).unwrap_or_else(|arc| (*arc).clone()),
+            )
+            .await;
+
+        if let Err(e) = self.run_inline_optimization().await {
+            log::warn!(
+                "Unexpected failure when running inline optimization: {:?}",
+                e
+            );
+        }
+
+        Ok(deleted_count)
+    }
+
+    /// Atomically delete table version entries from the manifest by their object_ids.
+    ///
+    /// This method supports multi-table transactional deletion: all specified
+    /// object_ids (which may span multiple tables) are deleted in a single atomic
+    /// `DeleteBuilder` operation. Either all entries are removed or none are.
+    ///
+    /// Object IDs are formatted as `{table_id}${version}`.
+    pub async fn batch_delete_table_versions_by_object_ids(
+        &self,
+        object_ids: &[String],
+    ) -> Result<i64> {
+        if object_ids.is_empty() {
+            return Ok(0);
+        }
+
+        let in_list: String = object_ids
+            .iter()
+            .map(|oid| {
+                let escaped = oid.replace('\'', "''");
+                format!("'{}'", escaped)
+            })
+            .collect::<Vec<_>>()
+            .join(", ");
+
+        let filter = format!(
+            "object_type = 'table_version' AND object_id IN ({})",
+            in_list
+        );
+
+        // Count how many entries exist so we can report the deleted count
+        let mut scanner = self.manifest_scanner().await?;
+        scanner.filter(&filter).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
+        })?;
+        scanner.project(&["object_id"]).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to project: {:?}", e),
+            })
+        })?;
+        let batches = Self::execute_scanner(scanner).await?;
+        let deleted_count: i64 = batches.iter().map(|b| b.num_rows() as i64).sum();
+
+        if deleted_count == 0 {
+            return Ok(0);
+        }
+
+        // Execute a single atomic bulk delete covering all tables
+        let _mutation_guard = self.manifest_mutation_lock.lock().await;
+        let dataset_guard = self.manifest_dataset.get().await?;
+        let dataset = Arc::new(dataset_guard.clone());
+        drop(dataset_guard);
+
+        let new_dataset = DeleteBuilder::new(dataset, &filter)
+            .execute()
+            .await
+            .map_err(|e| {
+                convert_lance_commit_error(
+                    &e,
+                    "Failed to batch delete table versions across multiple tables",
+                    None,
+                )
+            })?;
+
+        self.manifest_dataset
+            .set_latest(
+                Arc::try_unwrap(new_dataset.new_dataset).unwrap_or_else(|arc| (*arc).clone()),
+            )
+            .await;
+
+        if let Err(e) = self.run_inline_optimization().await {
+            log::warn!(
+                "Unexpected failure when running inline optimization: {:?}",
+                e
+            );
+        }
+
+        Ok(deleted_count)
+    }
+
+    /// Set a property flag in the __manifest table's metadata key-value map.
+    ///
+    /// This uses `dataset.update_metadata()` to persist the flag in the
+    /// __manifest dataset's table metadata, rather than inserting a row.
+    /// If the property already exists with the same value, this is a no-op.
+    pub async fn set_property(&self, name: &str, value: &str) -> Result<()> {
+        let _mutation_guard = self.manifest_mutation_lock.lock().await;
+        let dataset_guard = self.manifest_dataset.get().await?;
+        if dataset_guard.metadata().get(name) == Some(&value.to_string()) {
+            return Ok(());
+        }
+        drop(dataset_guard);
+
+        let mut dataset_guard = self.manifest_dataset.get_mut().await?;
+        dataset_guard
+            .update_metadata([(name, value)])
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to set property '{}' in __manifest metadata: {}",
+                        name, e
+                    ),
+                })
+            })?;
+        Ok(())
+    }
+
+    /// Check if a property flag exists in the __manifest table's metadata key-value map.
+    pub async fn has_property(&self, name: &str) -> Result<bool> {
+        let dataset_guard = self.manifest_dataset.get().await?;
+        Ok(dataset_guard.metadata().contains_key(name))
+    }
+
+    /// Parse metadata JSON into a `TableVersion`.
+    ///
+    /// Returns `None` if metadata is invalid or missing required fields.
+    fn parse_table_version(version: i64, metadata_str: &str) -> Option<TableVersion> {
+        let meta: serde_json::Value = match serde_json::from_str(metadata_str) {
+            Ok(v) => v,
+            Err(e) => {
+                log::warn!(
+                    "Skipping version {} due to invalid metadata JSON: {}",
+                    version,
+                    e
+                );
+                return None;
+            }
+        };
+        let manifest_path = match meta.get("manifest_path").and_then(|v| v.as_str()) {
+            Some(p) => p.to_string(),
+            None => {
+                log::warn!(
+                    "Skipping version {} due to missing 'manifest_path' in metadata — \
+                     this may indicate data corruption",
+                    version
+                );
+                return None;
+            }
+        };
+        let manifest_size = meta.get("manifest_size").and_then(|v| v.as_i64());
+        let e_tag = meta
+            .get("e_tag")
+            .and_then(|v| v.as_str())
+            .map(|s| s.to_string());
+        Some(TableVersion {
+            version,
+            manifest_path,
+            manifest_size,
+            e_tag,
+            timestamp_millis: None,
+            metadata: None,
+        })
+    }
+
+    /// List table versions from the __manifest table.
+    ///
+    /// Queries the manifest for all versions of the given table and returns
+    /// them as a `ListTableVersionsResponse`.
+    pub async fn list_table_versions(
+        &self,
+        table_id: &[String],
+        descending: bool,
+        limit: Option<i32>,
+    ) -> Result<ListTableVersionsResponse> {
+        let object_id = Self::str_object_id(table_id);
+        let manifest_versions = self
+            .query_table_versions(&object_id, descending, limit)
+            .await?;
+
+        let table_versions: Vec<TableVersion> = manifest_versions
+            .into_iter()
+            .filter_map(|(version, metadata_str)| Self::parse_table_version(version, &metadata_str))
+            .collect();
+
+        Ok(ListTableVersionsResponse {
+            versions: table_versions,
+            page_token: None,
+        })
+    }
+
+    /// Describe a specific table version from the __manifest table.
+    ///
+    /// Queries the manifest for a specific version and returns it as a
+    /// `DescribeTableVersionResponse`. Returns an error if the version is not found.
+    pub async fn describe_table_version(
+        &self,
+        table_id: &[String],
+        version: i64,
+    ) -> Result<DescribeTableVersionResponse> {
+        let object_id = Self::str_object_id(table_id);
+        if let Some(metadata_str) = self.query_table_version(&object_id, version).await?
+            && let Some(tv) = Self::parse_table_version(version, &metadata_str)
+        {
+            return Ok(DescribeTableVersionResponse {
+                version: Box::new(tv),
+            });
+        }
+        Err(NamespaceError::TableVersionNotFound {
+            message: format!("version {} for table {:?}", version, table_id),
+        }
+        .into())
+    }
+
     /// Register a table in the manifest without creating the physical table (internal helper for migration)
     pub async fn register_table(&self, name: &str, location: String) -> Result<()> {
         let object_id = Self::build_object_id(&[], name);
         if self.manifest_contains_object(&object_id).await? {
-            return Err(Error::io(format!("Table '{}' already exists", name)));
+            return Err(NamespaceError::Internal {
+                message: format!("Table '{}' already exists", name),
+            }
+            .into());
         }
 
         self.insert_into_manifest(object_id, ObjectType::Table, Some(location))
@@ -949,9 +1464,10 @@ impl ManifestNamespace {
             let partial_path = &namespace_path[..i];
             let object_id = partial_path.join(DELIMITER);
             if !self.manifest_contains_object(&object_id).await? {
-                return Err(Error::namespace_source(
-                    format!("Parent namespace '{}' does not exist", object_id).into(),
-                ));
+                return Err(NamespaceError::NamespaceNotFound {
+                    message: format!("parent namespace '{}'", object_id),
+                }
+                .into());
             }
         }
         Ok(())
@@ -959,19 +1475,18 @@ impl ManifestNamespace {
 
     /// Query the manifest for a namespace with the given object ID
     async fn query_manifest_for_namespace(&self, object_id: &str) -> Result<Option<NamespaceInfo>> {
-        let filter = format!("object_id = '{}' AND object_type = 'namespace'", object_id);
+        let escaped_id = object_id.replace('\'', "''");
+        let filter = format!("object_id = '{}' AND object_type = 'namespace'", escaped_id);
         let mut scanner = self.manifest_scanner().await?;
         scanner.filter(&filter).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to filter: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
         })?;
         scanner.project(&["object_id", "metadata"]).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to project: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to project: {:?}", e),
+            })
         })?;
         let batches = Self::execute_scanner(scanner).await?;
 
@@ -985,10 +1500,13 @@ impl ManifestNamespace {
 
             total_rows += batch.num_rows();
             if total_rows > 1 {
-                return Err(Error::io(format!(
-                    "Expected exactly 1 namespace with id '{}', found {}",
-                    object_id, total_rows
-                )));
+                return Err(NamespaceError::Internal {
+                    message: format!(
+                        "Expected exactly 1 namespace with id '{}', found {}",
+                        object_id, total_rows
+                    ),
+                }
+                .into());
             }
 
             let object_id_array = Self::get_string_column(&batch, "object_id")?;
@@ -1000,10 +1518,13 @@ impl ManifestNamespace {
                 match serde_json::from_str::<HashMap<String, String>>(metadata_str) {
                     Ok(map) => Some(map),
                     Err(e) => {
-                        return Err(Error::io(format!(
-                            "Failed to deserialize metadata for namespace '{}': {}",
-                            object_id, e
-                        )));
+                        return Err(NamespaceError::Internal {
+                            message: format!(
+                                "Failed to deserialize metadata for namespace '{}': {}",
+                                object_id, e
+                            ),
+                        }
+                        .into());
                     }
                 }
             } else {
@@ -1027,10 +1548,12 @@ impl ManifestNamespace {
     /// 1. Try to load an existing manifest table
     /// 2. If it exists, check and migrate the schema if needed (e.g., add primary key metadata)
     /// 3. If it doesn't exist, create a new manifest table with the current schema
+    /// 4. Persist feature flags (e.g., table_version_storage_enabled) if requested
     async fn ensure_manifest_table_up_to_date(
         root: &str,
         storage_options: &Option<HashMap<String, String>>,
         session: Option<Arc<Session>>,
+        table_version_storage_enabled: bool,
     ) -> Result<DatasetConsistencyWrapper> {
         let manifest_path = format!("{}/{}", root, MANIFEST_TABLE_NAME);
         log::debug!("Attempting to load manifest from {}", manifest_path);
@@ -1070,20 +1593,42 @@ impl ManifestNamespace {
                     .update_field_metadata()
                     .update("object_id", [(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "0")])
                     .map_err(|e| {
-                        Error::io_source(box_error(std::io::Error::other(format!(
-                            "Failed to find object_id field for migration: {}",
-                            e
-                        ))))
+                        lance_core::Error::from(NamespaceError::Internal {
+                            message: format!(
+                                "Failed to find object_id field for migration: {:?}",
+                                e
+                            ),
+                        })
                     })?
                     .await
                     .map_err(|e| {
-                        Error::io_source(box_error(std::io::Error::other(format!(
-                            "Failed to migrate primary key metadata: {}",
-                            e
-                        ))))
+                        lance_core::Error::from(NamespaceError::Internal {
+                            message: format!("Failed to migrate primary key metadata: {:?}", e),
+                        })
                     })?;
             }
 
+            // Persist table_version_storage_enabled flag in __manifest so that once
+            // enabled, it becomes a permanent property of this namespace.
+            if table_version_storage_enabled {
+                let needs_flag = dataset
+                    .metadata()
+                    .get("table_version_storage_enabled")
+                    .map(|v| v != "true")
+                    .unwrap_or(true);
+
+                if needs_flag
+                    && let Err(e) = dataset
+                        .update_metadata([("table_version_storage_enabled", "true")])
+                        .await
+                {
+                    log::warn!(
+                        "Failed to persist table_version_storage_enabled flag in __manifest: {:?}",
+                        e
+                    );
+                }
+            }
+
             Ok(DatasetConsistencyWrapper::new(dataset))
         } else {
             log::info!("Creating new manifest table at {}", manifest_path);
@@ -1155,19 +1700,60 @@ impl ManifestNamespace {
                         .load()
                         .await
                         .map_err(|e| {
-                            Error::io_source(box_error(std::io::Error::other(format!(
-                                "Failed to load manifest dataset after creation conflict: {}",
-                                e
-                            ))))
+                            lance_core::Error::from(NamespaceError::Internal {
+                                message: format!(
+                                    "Failed to load manifest dataset after creation conflict: {}",
+                                    e
+                                ),
+                            })
                         })?;
                     Ok(DatasetConsistencyWrapper::new(dataset))
                 }
-                Err(e) => Err(Error::io_source(box_error(std::io::Error::other(format!(
-                    "Failed to create manifest dataset: {}",
-                    e
-                ))))),
+                Err(e) => Err(lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to create manifest dataset: {:?}", e),
+                })),
+            }
+        }
+    }
+
+    /// Sorts names alphabetically and applies pagination using page_token (start_after) and limit.
+    ///
+    /// Returns the next page token (last item in this page) if more results exist beyond the limit,
+    /// or `None` if this is the last page.
+    fn apply_pagination(
+        names: &mut Vec<String>,
+        page_token: Option<String>,
+        limit: Option<i32>,
+    ) -> Option<String> {
+        names.sort();
+
+        if let Some(start_after) = page_token {
+            if let Some(index) = names
+                .iter()
+                .position(|name| name.as_str() > start_after.as_str())
+            {
+                names.drain(0..index);
+            } else {
+                names.clear();
+            }
+        }
+
+        if let Some(limit) = limit
+            && limit >= 0
+        {
+            let limit = limit as usize;
+            if names.len() > limit {
+                let next_page_token = if limit > 0 {
+                    Some(names[limit - 1].clone())
+                } else {
+                    None
+                };
+                names.truncate(limit);
+                return next_page_token;
             }
         }
+
+        None
     }
 }
 
@@ -1178,10 +1764,11 @@ impl LanceNamespace for ManifestNamespace {
     }
 
     async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
-        let namespace_id = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Namespace ID is required".into()))?;
+        let namespace_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Namespace ID is required".to_string(),
+            })
+        })?;
 
         // Build filter to find tables in this namespace
         let filter = if namespace_id.is_empty() {
@@ -1200,16 +1787,14 @@ impl LanceNamespace for ManifestNamespace {
 
         let mut scanner = self.manifest_scanner().await?;
         scanner.filter(&filter).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to filter: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
         })?;
         scanner.project(&["object_id"]).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to project: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to project: {:?}", e),
+            })
         })?;
 
         let batches = Self::execute_scanner(scanner).await?;
@@ -1228,19 +1813,25 @@ impl LanceNamespace for ManifestNamespace {
             }
         }
 
-        Ok(ListTablesResponse::new(tables))
+        let next_page_token =
+            Self::apply_pagination(&mut tables, request.page_token, request.limit);
+        let mut response = ListTablesResponse::new(tables);
+        response.page_token = next_page_token;
+        Ok(response)
     }
 
     async fn describe_table(&self, request: DescribeTableRequest) -> Result<DescribeTableResponse> {
-        let table_id = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Table ID is required".into()))?;
+        let table_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Table ID is required".to_string(),
+            })
+        })?;
 
         if table_id.is_empty() {
-            return Err(Error::invalid_input_source(
-                "Table ID cannot be empty".into(),
-            ));
+            return Err(NamespaceError::InvalidInput {
+                message: "Table ID cannot be empty".to_string(),
+            }
+            .into());
         }
 
         let object_id = Self::str_object_id(table_id);
@@ -1318,33 +1909,36 @@ impl LanceNamespace for ManifestNamespace {
                     }
                 }
             }
-            None => Err(Error::namespace_source(
-                format!("Table '{}' not found", object_id).into(),
-            )),
+            None => Err(NamespaceError::TableNotFound {
+                message: Self::format_table_id(table_id),
+            }
+            .into()),
         }
     }
 
     async fn table_exists(&self, request: TableExistsRequest) -> Result<()> {
-        let table_id = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Table ID is required".into()))?;
+        let table_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Table ID is required".to_string(),
+            })
+        })?;
 
         if table_id.is_empty() {
-            return Err(Error::invalid_input_source(
-                "Table ID cannot be empty".into(),
-            ));
+            return Err(NamespaceError::InvalidInput {
+                message: "Table ID cannot be empty".to_string(),
+            }
+            .into());
         }
 
-        let (namespace, table_name) = Self::split_object_id(table_id);
-        let object_id = Self::build_object_id(&namespace, &table_name);
+        let object_id = Self::str_object_id(table_id);
         let exists = self.manifest_contains_object(&object_id).await?;
         if exists {
             Ok(())
         } else {
-            Err(Error::namespace_source(
-                format!("Table '{}' not found", table_name).into(),
-            ))
+            Err(NamespaceError::TableNotFound {
+                message: Self::format_table_id(table_id),
+            }
+            .into())
         }
     }
 
@@ -1353,15 +1947,17 @@ impl LanceNamespace for ManifestNamespace {
         request: CreateTableRequest,
         data: Bytes,
     ) -> Result<CreateTableResponse> {
-        let table_id = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Table ID is required".into()))?;
+        let table_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Table ID is required".to_string(),
+            })
+        })?;
 
         if table_id.is_empty() {
-            return Err(Error::invalid_input_source(
-                "Table ID cannot be empty".into(),
-            ));
+            return Err(NamespaceError::InvalidInput {
+                message: "Table ID cannot be empty".to_string(),
+            }
+            .into());
         }
 
         let (namespace, table_name) = Self::split_object_id(table_id);
@@ -1369,7 +1965,10 @@ impl LanceNamespace for ManifestNamespace {
 
         // Check if table already exists in manifest
         if self.manifest_contains_object(&object_id).await? {
-            return Err(Error::io(format!("Table '{}' already exists", table_name)));
+            return Err(NamespaceError::Internal {
+                message: format!("Table '{}' already exists", table_name),
+            }
+            .into());
         }
 
         // Create the physical table location with hash-based naming
@@ -1386,23 +1985,33 @@ impl LanceNamespace for ManifestNamespace {
 
         // Validate that request_data is provided
         if data.is_empty() {
-            return Err(Error::namespace_source(
-                "Request data (Arrow IPC stream) is required for create_table".into(),
-            ));
+            return Err(NamespaceError::InvalidInput {
+                message: "Request data (Arrow IPC stream) is required for create_table".to_string(),
+            }
+            .into());
         }
 
         // Write the data using Lance Dataset
         let cursor = Cursor::new(data.to_vec());
-        let stream_reader = StreamReader::try_new(cursor, None)
-            .map_err(|e| Error::io(format!("Failed to read IPC stream: {}", e)))?;
+        let stream_reader = StreamReader::try_new(cursor, None).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to read IPC stream: {:?}", e),
+            })
+        })?;
 
-        let batches: Vec<RecordBatch> =
-            stream_reader
-                .collect::<std::result::Result<Vec<_>, _>>()
-                .map_err(|e| Error::io(format!("Failed to collect batches: {}", e)))?;
+        let batches: Vec<RecordBatch> = stream_reader
+            .collect::<std::result::Result<Vec<_>, _>>()
+            .map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to collect batches: {:?}", e),
+            })
+        })?;
 
         if batches.is_empty() {
-            return Err(Error::io("No data provided for table creation"));
+            return Err(NamespaceError::Internal {
+                message: "No data provided for table creation".to_string(),
+            }
+            .into());
         }
 
         let schema = batches[0].schema();
@@ -1428,10 +2037,9 @@ impl LanceNamespace for ManifestNamespace {
         let _dataset = Dataset::write(Box::new(reader), &table_uri, Some(write_params))
             .await
             .map_err(|e| {
-                Error::io_source(box_error(std::io::Error::other(format!(
-                    "Failed to write dataset: {}",
-                    e
-                ))))
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to write dataset: {:?}", e),
+                })
             })?;
 
         // Register in manifest (store dir_name, not full URI)
@@ -1447,15 +2055,17 @@ impl LanceNamespace for ManifestNamespace {
     }
 
     async fn drop_table(&self, request: DropTableRequest) -> Result<DropTableResponse> {
-        let table_id = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Table ID is required".into()))?;
+        let table_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Table ID is required".to_string(),
+            })
+        })?;
 
         if table_id.is_empty() {
-            return Err(Error::invalid_input_source(
-                "Table ID cannot be empty".into(),
-            ));
+            return Err(NamespaceError::InvalidInput {
+                message: "Table ID cannot be empty".to_string(),
+            }
+            .into());
         }
 
         let (namespace, table_name) = Self::split_object_id(table_id);
@@ -1479,9 +2089,9 @@ impl LanceNamespace for ManifestNamespace {
                     .boxed()
                     .await
                     .map_err(|e| {
-                        Error::namespace_source(
-                            format!("Failed to delete table directory: {}", e).into(),
-                        )
+                        lance_core::Error::from(NamespaceError::Internal {
+                            message: format!("Failed to delete table directory: {:?}", e),
+                        })
                     })?;
 
                 Ok(DropTableResponse {
@@ -1490,9 +2100,10 @@ impl LanceNamespace for ManifestNamespace {
                     ..Default::default()
                 })
             }
-            None => Err(Error::namespace_source(
-                format!("Table '{}' not found", table_name).into(),
-            )),
+            None => Err(NamespaceError::TableNotFound {
+                message: table_name.to_string(),
+            }
+            .into()),
         }
     }
 
@@ -1500,10 +2111,11 @@ impl LanceNamespace for ManifestNamespace {
         &self,
         request: ListNamespacesRequest,
     ) -> Result<ListNamespacesResponse> {
-        let parent_namespace = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Namespace ID is required".into()))?;
+        let parent_namespace = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Namespace ID is required".to_string(),
+            })
+        })?;
 
         // Build filter to find direct child namespaces
         let filter = if parent_namespace.is_empty() {
@@ -1522,16 +2134,14 @@ impl LanceNamespace for ManifestNamespace {
 
         let mut scanner = self.manifest_scanner().await?;
         scanner.filter(&filter).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to filter: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
         })?;
         scanner.project(&["object_id"]).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to project: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to project: {:?}", e),
+            })
         })?;
 
         let batches = Self::execute_scanner(scanner).await?;
@@ -1550,17 +2160,22 @@ impl LanceNamespace for ManifestNamespace {
             }
         }
 
-        Ok(ListNamespacesResponse::new(namespaces))
+        let next_page_token =
+            Self::apply_pagination(&mut namespaces, request.page_token, request.limit);
+        let mut response = ListNamespacesResponse::new(namespaces);
+        response.page_token = next_page_token;
+        Ok(response)
     }
 
     async fn describe_namespace(
         &self,
         request: DescribeNamespaceRequest,
     ) -> Result<DescribeNamespaceResponse> {
-        let namespace_id = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Namespace ID is required".into()))?;
+        let namespace_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Namespace ID is required".to_string(),
+            })
+        })?;
 
         // Root namespace always exists
         if namespace_id.is_empty() {
@@ -1581,9 +2196,10 @@ impl LanceNamespace for ManifestNamespace {
                 properties: info.metadata,
                 ..Default::default()
             }),
-            None => Err(Error::namespace_source(
-                format!("Namespace '{}' not found", object_id).into(),
-            )),
+            None => Err(NamespaceError::NamespaceNotFound {
+                message: object_id.to_string(),
+            }
+            .into()),
         }
     }
 
@@ -1591,16 +2207,18 @@ impl LanceNamespace for ManifestNamespace {
         &self,
         request: CreateNamespaceRequest,
     ) -> Result<CreateNamespaceResponse> {
-        let namespace_id = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Namespace ID is required".into()))?;
+        let namespace_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Namespace ID is required".to_string(),
+            })
+        })?;
 
         // Root namespace always exists and cannot be created
         if namespace_id.is_empty() {
-            return Err(Error::namespace_source(
-                "Root namespace already exists and cannot be created".into(),
-            ));
+            return Err(NamespaceError::NamespaceAlreadyExists {
+                message: "root namespace".to_string(),
+            }
+            .into());
         }
 
         // Validate parent namespaces exist (but not the namespace being created)
@@ -1611,9 +2229,10 @@ impl LanceNamespace for ManifestNamespace {
 
         let object_id = namespace_id.join(DELIMITER);
         if self.manifest_contains_object(&object_id).await? {
-            return Err(Error::namespace_source(
-                format!("Namespace '{}' already exists", object_id).into(),
-            ));
+            return Err(NamespaceError::NamespaceAlreadyExists {
+                message: object_id.to_string(),
+            }
+            .into());
         }
 
         // Serialize properties if provided
@@ -1626,10 +2245,12 @@ impl LanceNamespace for ManifestNamespace {
         });
 
         self.insert_into_manifest_with_metadata(
-            object_id,
-            ObjectType::Namespace,
-            None,
-            metadata,
+            vec![ManifestEntry {
+                object_id,
+                object_type: ObjectType::Namespace,
+                location: None,
+                metadata,
+            }],
             None,
         )
         .await?;
@@ -1641,59 +2262,57 @@ impl LanceNamespace for ManifestNamespace {
     }
 
     async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result<DropNamespaceResponse> {
-        let namespace_id = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Namespace ID is required".into()))?;
+        let namespace_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Namespace ID is required".to_string(),
+            })
+        })?;
 
         // Root namespace always exists and cannot be dropped
         if namespace_id.is_empty() {
-            return Err(Error::namespace_source(
-                "Root namespace cannot be dropped".into(),
-            ));
+            return Err(NamespaceError::InvalidInput {
+                message: "Root namespace cannot be dropped".to_string(),
+            }
+            .into());
         }
 
         let object_id = namespace_id.join(DELIMITER);
 
         // Check if namespace exists
         if !self.manifest_contains_object(&object_id).boxed().await? {
-            return Err(Error::namespace_source(
-                format!("Namespace '{}' not found", object_id).into(),
-            ));
+            return Err(NamespaceError::NamespaceNotFound {
+                message: object_id.to_string(),
+            }
+            .into());
         }
 
         // Check for child namespaces
-        let prefix = format!("{}{}", object_id, DELIMITER);
+        let escaped_id = object_id.replace('\'', "''");
+        let prefix = format!("{}{}", escaped_id, DELIMITER);
         let filter = format!("starts_with(object_id, '{}')", prefix);
         let mut scanner = self.manifest_scanner().boxed().await?;
         scanner.filter(&filter).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to filter: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
         })?;
         scanner.project::<&str>(&[]).map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to project: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to project: {:?}", e),
+            })
         })?;
         scanner.with_row_id();
         let count = scanner.count_rows().boxed().await.map_err(|e| {
-            Error::io_source(box_error(std::io::Error::other(format!(
-                "Failed to count rows: {}",
-                e
-            ))))
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to count rows: {:?}", e),
+            })
         })?;
 
         if count > 0 {
-            return Err(Error::namespace_source(
-                format!(
-                    "Namespace '{}' is not empty (contains {} child objects)",
-                    object_id, count
-                )
-                .into(),
-            ));
+            return Err(NamespaceError::NamespaceNotEmpty {
+                message: format!("'{}' (contains {} child objects)", object_id, count),
+            }
+            .into());
         }
 
         self.delete_from_manifest(&object_id).boxed().await?;
@@ -1702,10 +2321,11 @@ impl LanceNamespace for ManifestNamespace {
     }
 
     async fn namespace_exists(&self, request: NamespaceExistsRequest) -> Result<()> {
-        let namespace_id = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Namespace ID is required".into()))?;
+        let namespace_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Namespace ID is required".to_string(),
+            })
+        })?;
 
         // Root namespace always exists
         if namespace_id.is_empty() {
@@ -1716,22 +2336,25 @@ impl LanceNamespace for ManifestNamespace {
         if self.manifest_contains_object(&object_id).await? {
             Ok(())
         } else {
-            Err(Error::namespace_source(
-                format!("Namespace '{}' not found", object_id).into(),
-            ))
+            Err(NamespaceError::NamespaceNotFound {
+                message: object_id.to_string(),
+            }
+            .into())
         }
     }
 
     async fn declare_table(&self, request: DeclareTableRequest) -> Result<DeclareTableResponse> {
-        let table_id = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Table ID is required".into()))?;
+        let table_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Table ID is required".to_string(),
+            })
+        })?;
 
         if table_id.is_empty() {
-            return Err(Error::invalid_input_source(
-                "Table ID cannot be empty".into(),
-            ));
+            return Err(NamespaceError::InvalidInput {
+                message: "Table ID cannot be empty".to_string(),
+            }
+            .into());
         }
 
         let (namespace, table_name) = Self::split_object_id(table_id);
@@ -1740,9 +2363,10 @@ impl LanceNamespace for ManifestNamespace {
         // Check if table already exists in manifest
         let existing = self.query_manifest_for_table(&object_id).await?;
         if existing.is_some() {
-            return Err(Error::namespace_source(
-                format!("Table '{}' already exists", table_name).into(),
-            ));
+            return Err(NamespaceError::TableAlreadyExists {
+                message: table_name.to_string(),
+            }
+            .into());
         }
 
         // Create table location path with hash-based naming
@@ -1762,13 +2386,13 @@ impl LanceNamespace for ManifestNamespace {
         if let Some(req_location) = &request.location {
             let req_location = req_location.trim_end_matches('/');
             if req_location != table_uri {
-                return Err(Error::namespace_source(
-                    format!(
+                return Err(NamespaceError::InvalidInput {
+                    message: format!(
                         "Cannot declare table {} at location {}, must be at location {}",
                         table_name, req_location, table_uri
-                    )
-                    .into(),
-                ));
+                    ),
+                }
+                .into());
             }
         }
 
@@ -1779,24 +2403,22 @@ impl LanceNamespace for ManifestNamespace {
             .create(&reserved_file_path)
             .await
             .map_err(|e| {
-                Error::namespace_source(
-                    format!(
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
                         "Failed to create .lance-reserved file for table {}: {}",
                         table_name, e
-                    )
-                    .into(),
-                )
+                    ),
+                })
             })?
             .shutdown()
             .await
             .map_err(|e| {
-                Error::namespace_source(
-                    format!(
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
                         "Failed to finalize .lance-reserved file for table {}: {}",
                         table_name, e
-                    )
-                    .into(),
-                )
+                    ),
+                })
             })?;
 
         // Add entry to manifest marking this as a declared table (store dir_name, not full path)
@@ -1825,15 +2447,17 @@ impl LanceNamespace for ManifestNamespace {
     }
 
     async fn register_table(&self, request: RegisterTableRequest) -> Result<RegisterTableResponse> {
-        let table_id = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Table ID is required".into()))?;
+        let table_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Table ID is required".to_string(),
+            })
+        })?;
 
         if table_id.is_empty() {
-            return Err(Error::invalid_input_source(
-                "Table ID cannot be empty".into(),
-            ));
+            return Err(NamespaceError::InvalidInput {
+                message: "Table ID cannot be empty".to_string(),
+            }
+            .into());
         }
 
         let location = request.location.clone();
@@ -1841,25 +2465,34 @@ impl LanceNamespace for ManifestNamespace {
         // Validate that location is a relative path within the root directory
         // We don't allow absolute URIs or paths that escape the root
         if location.contains("://") {
-            return Err(Error::invalid_input_source(format!(
-                "Absolute URIs are not allowed for register_table. Location must be a relative path within the root directory: {}",
-                location
-            ).into()));
+            return Err(NamespaceError::InvalidInput {
+                message: format!(
+                    "Absolute URIs are not allowed for register_table. Location must be a relative path within the root directory: {}",
+                    location
+                ),
+            }
+            .into());
         }
 
         if location.starts_with('/') {
-            return Err(Error::invalid_input_source(format!(
-                "Absolute paths are not allowed for register_table. Location must be a relative path within the root directory: {}",
-                location
-            ).into()));
+            return Err(NamespaceError::InvalidInput {
+                message: format!(
+                    "Absolute paths are not allowed for register_table. Location must be a relative path within the root directory: {}",
+                    location
+                ),
+            }
+            .into());
         }
 
         // Check for path traversal attempts
         if location.contains("..") {
-            return Err(Error::invalid_input_source(format!(
-                "Path traversal is not allowed. Location must be a relative path within the root directory: {}",
-                location
-            ).into()));
+            return Err(NamespaceError::InvalidInput {
+                message: format!(
+                    "Path traversal is not allowed. Location must be a relative path within the root directory: {}",
+                    location
+                ),
+            }
+            .into());
         }
 
         let (namespace, table_name) = Self::split_object_id(table_id);
@@ -1872,9 +2505,10 @@ impl LanceNamespace for ManifestNamespace {
 
         // Check if table already exists
         if self.manifest_contains_object(&object_id).await? {
-            return Err(Error::namespace_source(
-                format!("Table '{}' already exists", object_id).into(),
-            ));
+            return Err(NamespaceError::TableAlreadyExists {
+                message: object_id.to_string(),
+            }
+            .into());
         }
 
         // Register the table with its location in the manifest
@@ -1891,15 +2525,17 @@ impl LanceNamespace for ManifestNamespace {
         &self,
         request: DeregisterTableRequest,
     ) -> Result<DeregisterTableResponse> {
-        let table_id = request
-            .id
-            .as_ref()
-            .ok_or_else(|| Error::invalid_input_source("Table ID is required".into()))?;
+        let table_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Table ID is required".to_string(),
+            })
+        })?;
 
         if table_id.is_empty() {
-            return Err(Error::invalid_input_source(
-                "Table ID cannot be empty".into(),
-            ));
+            return Err(NamespaceError::InvalidInput {
+                message: "Table ID cannot be empty".to_string(),
+            }
+            .into());
         }
 
         let (namespace, table_name) = Self::split_object_id(table_id);
@@ -1915,9 +2551,10 @@ impl LanceNamespace for ManifestNamespace {
                 Self::construct_full_uri(&self.root, &info.location)?
             }
             None => {
-                return Err(Error::namespace_source(
-                    format!("Table '{}' not found", object_id).into(),
-                ));
+                return Err(NamespaceError::TableNotFound {
+                    message: object_id.to_string(),
+                }
+                .into());
             }
         };
 
@@ -2124,6 +2761,37 @@ mod tests {
         assert_eq!(response.tables.len(), 0);
     }
 
+    #[tokio::test]
+    async fn test_list_tables_pagination_limit_zero() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let dir_namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .build()
+            .await
+            .unwrap();
+
+        let buffer = create_test_ipc_data();
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["alpha".to_string()]);
+        dir_namespace
+            .create_table(create_request, Bytes::from(buffer))
+            .await
+            .unwrap();
+
+        let response = dir_namespace
+            .list_tables(ListTablesRequest {
+                id: Some(vec![]),
+                limit: Some(0),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert!(response.tables.is_empty());
+        assert!(response.page_token.is_none());
+    }
+
     #[rstest]
     #[case::with_optimization(true)]
     #[case::without_optimization(false)]
@@ -2913,5 +3581,158 @@ mod tests {
             trailing_slash_result, "s3://bucket/path/subdir/table.lance",
             "URL with existing trailing slash should still work"
         );
+
+        // Test that URLs with empty query string don't include trailing "?"
+        // This is important because URL::to_string() can add "?" for empty queries
+        let empty_query_result =
+            ManifestNamespace::construct_full_uri("s3://bucket/path?", "table.lance").unwrap();
+        assert_eq!(
+            empty_query_result, "s3://bucket/path/table.lance",
+            "URL with empty query string should not include trailing '?'"
+        );
+
+        // Test that URLs with actual query parameters have them stripped
+        // (query parameters are not meaningful for storage paths)
+        let query_param_result =
+            ManifestNamespace::construct_full_uri("s3://bucket/path?param=value", "table.lance")
+                .unwrap();
+        assert_eq!(
+            query_param_result, "s3://bucket/path/table.lance",
+            "URL with query parameters should have them stripped"
+        );
+    }
+
+    /// Test that concurrent create_table calls for the same table name don't
+    /// create duplicate entries in the manifest. Uses two independent
+    /// ManifestNamespace instances pointing at the same directory to simulate
+    /// two separate OS processes racing on table creation. The conflict_retries
+    /// setting on the MergeInsert ensures the second operation properly detects
+    /// the duplicate via WhenMatched::Fail after retrying against the latest data.
+    #[tokio::test]
+    async fn test_concurrent_create_table_no_duplicates() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        // Two independent namespace instances = two separate "processes"
+        // sharing the same underlying filesystem directory.
+        let ns1 = DirectoryNamespaceBuilder::new(temp_path)
+            .inline_optimization_enabled(false)
+            .build()
+            .await
+            .unwrap();
+        let ns2 = DirectoryNamespaceBuilder::new(temp_path)
+            .inline_optimization_enabled(false)
+            .build()
+            .await
+            .unwrap();
+
+        let buffer = create_test_ipc_data();
+
+        let mut req1 = CreateTableRequest::new();
+        req1.id = Some(vec!["race_table".to_string()]);
+        let mut req2 = CreateTableRequest::new();
+        req2.id = Some(vec!["race_table".to_string()]);
+
+        // Launch both create_table calls concurrently
+        let (result1, result2) = tokio::join!(
+            ns1.create_table(req1, Bytes::from(buffer.clone())),
+            ns2.create_table(req2, Bytes::from(buffer.clone())),
+        );
+
+        // Exactly one should succeed and one should fail
+        let success_count = [&result1, &result2].iter().filter(|r| r.is_ok()).count();
+        let failure_count = [&result1, &result2].iter().filter(|r| r.is_err()).count();
+        assert_eq!(
+            success_count, 1,
+            "Exactly one create should succeed, got: result1={:?}, result2={:?}",
+            result1, result2
+        );
+        assert_eq!(
+            failure_count, 1,
+            "Exactly one create should fail, got: result1={:?}, result2={:?}",
+            result1, result2
+        );
+
+        // Verify only one table entry exists in the manifest
+        let ns_check = DirectoryNamespaceBuilder::new(temp_path)
+            .inline_optimization_enabled(false)
+            .build()
+            .await
+            .unwrap();
+        let mut list_request = ListTablesRequest::new();
+        list_request.id = Some(vec![]);
+        let response = ns_check.list_tables(list_request).await.unwrap();
+        assert_eq!(
+            response.tables.len(),
+            1,
+            "Should have exactly 1 table, found: {:?}",
+            response.tables
+        );
+        assert_eq!(response.tables[0], "race_table");
+
+        // Also verify describe_table works (no "found 2" error)
+        let mut describe_request = DescribeTableRequest::new();
+        describe_request.id = Some(vec!["race_table".to_string()]);
+        let describe_result = ns_check.describe_table(describe_request).await;
+        assert!(
+            describe_result.is_ok(),
+            "describe_table should not fail with duplicate entries: {:?}",
+            describe_result
+        );
+    }
+
+    // --- apply_pagination unit tests ---
+
+    fn names(v: &[&str]) -> Vec<String> {
+        v.iter().map(|s| s.to_string()).collect()
+    }
+
+    #[test]
+    fn test_apply_pagination_no_token_no_limit() {
+        let mut n = names(&["b", "a", "c"]);
+        let next = ManifestNamespace::apply_pagination(&mut n, None, None);
+        assert_eq!(n, names(&["a", "b", "c"]));
+        assert_eq!(next, None);
+    }
+
+    #[test]
+    fn test_apply_pagination_limit_truncates_and_returns_token() {
+        let mut n = names(&["c", "a", "b"]);
+        let next = ManifestNamespace::apply_pagination(&mut n, None, Some(2));
+        assert_eq!(n, names(&["a", "b"]));
+        assert_eq!(next, Some("b".to_string()));
+    }
+
+    #[test]
+    fn test_apply_pagination_limit_zero_returns_empty_no_token() {
+        let mut n = names(&["a", "b", "c"]);
+        let next = ManifestNamespace::apply_pagination(&mut n, None, Some(0));
+        assert!(n.is_empty());
+        assert_eq!(next, None);
+    }
+
+    #[test]
+    fn test_apply_pagination_page_token_in_list() {
+        // "b" is in the list; should start from "c" (strict >)
+        let mut n = names(&["a", "b", "c", "d"]);
+        let next = ManifestNamespace::apply_pagination(&mut n, Some("b".to_string()), None);
+        assert_eq!(n, names(&["c", "d"]));
+        assert_eq!(next, None);
+    }
+
+    #[test]
+    fn test_apply_pagination_page_token_past_all_items() {
+        let mut n = names(&["a", "b", "c"]);
+        let next = ManifestNamespace::apply_pagination(&mut n, Some("z".to_string()), None);
+        assert!(n.is_empty());
+        assert_eq!(next, None);
+    }
+
+    #[test]
+    fn test_apply_pagination_token_and_limit_combined() {
+        let mut n = names(&["a", "b", "c", "d", "e"]);
+        let next = ManifestNamespace::apply_pagination(&mut n, Some("b".to_string()), Some(2));
+        assert_eq!(n, names(&["c", "d"]));
+        assert_eq!(next, Some("d".to_string()));
     }
 }
diff --git a/rust/lance-namespace-impls/src/lib.rs b/rust/lance-namespace-impls/src/lib.rs
index a67aff1784d..7570e470b27 100644
--- a/rust/lance-namespace-impls/src/lib.rs
+++ b/rust/lance-namespace-impls/src/lib.rs
@@ -82,7 +82,9 @@ pub mod rest_adapter;
 // Re-export connect builder
 pub use connect::ConnectBuilder;
 pub use context::{DynamicContextProvider, OperationInfo};
-pub use dir::{DirectoryNamespace, DirectoryNamespaceBuilder, manifest::ManifestNamespace};
+pub use dir::{
+    DirectoryNamespace, DirectoryNamespaceBuilder, OpsMetrics, manifest::ManifestNamespace,
+};
 
 // Re-export credential vending
 pub use credentials::{
diff --git a/rust/lance-namespace-impls/src/rest.rs b/rust/lance-namespace-impls/src/rest.rs
index 21451d89c39..80766632a25 100644
--- a/rust/lance-namespace-impls/src/rest.rs
+++ b/rust/lance-namespace-impls/src/rest.rs
@@ -7,6 +7,8 @@ use std::collections::HashMap;
 use std::str::FromStr;
 use std::sync::Arc;
 
+use crate::OpsMetrics;
+
 use async_trait::async_trait;
 use bytes::Bytes;
 use reqwest::header::{HeaderName, HeaderValue};
@@ -18,8 +20,9 @@ use lance_namespace::models::{
     AlterTableAddColumnsRequest, AlterTableAddColumnsResponse, AlterTableAlterColumnsRequest,
     AlterTableAlterColumnsResponse, AlterTableDropColumnsRequest, AlterTableDropColumnsResponse,
     AlterTransactionRequest, AlterTransactionResponse, AnalyzeTableQueryPlanRequest,
-    CountTableRowsRequest, CreateNamespaceRequest, CreateNamespaceResponse,
-    CreateTableIndexRequest, CreateTableIndexResponse, CreateTableRequest, CreateTableResponse,
+    BatchDeleteTableVersionsRequest, BatchDeleteTableVersionsResponse, CountTableRowsRequest,
+    CreateNamespaceRequest, CreateNamespaceResponse, CreateTableIndexRequest,
+    CreateTableIndexResponse, CreateTableRequest, CreateTableResponse,
     CreateTableScalarIndexResponse, CreateTableTagRequest, CreateTableTagResponse,
     CreateTableVersionRequest, CreateTableVersionResponse, DeclareTableRequest,
     DeclareTableResponse, DeleteFromTableRequest, DeleteFromTableResponse, DeleteTableTagRequest,
@@ -29,7 +32,7 @@ use lance_namespace::models::{
     DescribeTableVersionRequest, DescribeTableVersionResponse, DescribeTransactionRequest,
     DescribeTransactionResponse, DropNamespaceRequest, DropNamespaceResponse,
     DropTableIndexRequest, DropTableIndexResponse, DropTableRequest, DropTableResponse,
-    ExplainTableQueryPlanRequest, GetTableStatsRequest, GetTableStatsResponse,
+    ErrorResponse, ExplainTableQueryPlanRequest, GetTableStatsRequest, GetTableStatsResponse,
     GetTableTagVersionRequest, GetTableTagVersionResponse, InsertIntoTableRequest,
     InsertIntoTableResponse, ListNamespacesRequest, ListNamespacesResponse,
     ListTableIndicesRequest, ListTableIndicesResponse, ListTableTagsRequest, ListTableTagsResponse,
@@ -42,9 +45,10 @@ use lance_namespace::models::{
 };
 use serde::{Serialize, de::DeserializeOwned};
 
-use lance_core::{Error, Result, box_error};
+use lance_core::{Error, Result};
 
 use lance_namespace::LanceNamespace;
+use lance_namespace::error::NamespaceError;
 
 /// HTTP client wrapper that supports per-request header injection.
 ///
@@ -164,6 +168,8 @@ pub struct RestNamespaceBuilder {
     ssl_ca_cert: Option<String>,
     assert_hostname: bool,
     context_provider: Option<Arc<dyn DynamicContextProvider>>,
+    /// When true, tracks operation metrics. Default: false.
+    ops_metrics_enabled: bool,
 }
 
 impl std::fmt::Debug for RestNamespaceBuilder {
@@ -180,6 +186,7 @@ impl std::fmt::Debug for RestNamespaceBuilder {
                 "context_provider",
                 &self.context_provider.as_ref().map(|_| "Some(...)"),
             )
+            .field("ops_metrics_enabled", &self.ops_metrics_enabled)
             .finish()
     }
 }
@@ -203,6 +210,7 @@ impl RestNamespaceBuilder {
             ssl_ca_cert: None,
             assert_hostname: true,
             context_provider: None,
+            ops_metrics_enabled: false,
         }
     }
 
@@ -249,7 +257,9 @@ impl RestNamespaceBuilder {
     pub fn from_properties(properties: HashMap<String, String>) -> Result<Self> {
         // Extract URI (required)
         let uri = properties.get("uri").cloned().ok_or_else(|| {
-            Error::namespace_source("Missing required property 'uri' for REST namespace".into())
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Missing required property 'uri' for REST namespace".to_string(),
+            })
         })?;
 
         // Extract delimiter (optional)
@@ -278,6 +288,12 @@ impl RestNamespaceBuilder {
             .and_then(|v| v.parse::<bool>().ok())
             .unwrap_or(true);
 
+        // Extract ops_metrics_enabled (default: false)
+        let ops_metrics_enabled = properties
+            .get("ops_metrics_enabled")
+            .and_then(|v| v.parse::<bool>().ok())
+            .unwrap_or(false);
+
         Ok(Self {
             uri,
             delimiter,
@@ -287,6 +303,7 @@ impl RestNamespaceBuilder {
             ssl_ca_cert,
             assert_hostname,
             context_provider: None,
+            ops_metrics_enabled,
         })
     }
 
@@ -399,6 +416,18 @@ impl RestNamespaceBuilder {
         self
     }
 
+    /// Enable or disable operation metrics tracking.
+    ///
+    /// When enabled, the namespace will track how many times each API operation
+    /// is called. Use `retrieve_ops_metrics()` on the built namespace to get
+    /// the current counts.
+    ///
+    /// Default is false.
+    pub fn ops_metrics_enabled(mut self, enabled: bool) -> Self {
+        self.ops_metrics_enabled = enabled;
+        self
+    }
+
     /// Build the RestNamespace.
     ///
     /// # Returns
@@ -414,7 +443,10 @@ fn object_id_str(id: &Option<Vec<String>>, delimiter: &str) -> Result<String> {
     match id {
         Some(id_parts) if !id_parts.is_empty() => Ok(id_parts.join(delimiter)),
         Some(_) => Ok(delimiter.to_string()),
-        None => Err(Error::namespace_source("Object ID is required".into())),
+        None => Err(NamespaceError::InvalidInput {
+            message: "Object ID is required".to_string(),
+        }
+        .into()),
     }
 }
 
@@ -436,6 +468,8 @@ pub struct RestNamespace {
     delimiter: String,
     /// REST client that handles per-request header injection efficiently.
     rest_client: RestClient,
+    /// Operation metrics tracker, created when ops_metrics_enabled is true.
+    ops_metrics: Option<Arc<OpsMetrics>>,
 }
 
 impl std::fmt::Debug for RestNamespace {
@@ -487,9 +521,51 @@ impl RestNamespace {
             context_provider: builder.context_provider,
         };
 
+        let ops_metrics = if builder.ops_metrics_enabled {
+            Some(Arc::new(OpsMetrics::default()))
+        } else {
+            None
+        };
+
         Self {
             delimiter: builder.delimiter,
             rest_client,
+            ops_metrics,
+        }
+    }
+
+    /// Map a reqwest::Error to the appropriate NamespaceError variant.
+    ///
+    /// Timeout and connection errors are mapped to `ServiceUnavailable`,
+    /// while other errors are mapped to `Internal`.
+    fn request_error(e: reqwest::Error) -> lance_core::Error {
+        let message = format!("Failed to execute request: {:?}", e);
+        if e.is_timeout() || e.is_connect() {
+            NamespaceError::ServiceUnavailable { message }.into()
+        } else {
+            NamespaceError::Internal { message }.into()
+        }
+    }
+
+    /// Parse an error response body and return the appropriate NamespaceError.
+    ///
+    /// Deserializes the response as an `ErrorResponse` model (the spec-defined
+    /// flat JSON format with a required numeric `code` field). The error code is
+    /// the sole source of truth for error classification. When deserialization
+    /// fails, returns Internal with the raw response as context.
+    fn parse_error_response(status: reqwest::StatusCode, content: &str) -> lance_core::Error {
+        match serde_json::from_str::<ErrorResponse>(content) {
+            Ok(err_resp) => {
+                let message = err_resp.error.as_deref().unwrap_or(content);
+                NamespaceError::from_code(err_resp.code as u32, message).into()
+            }
+            Err(e) => NamespaceError::Internal {
+                message: format!(
+                    "Failed to parse error response: status={}, body={}, error={:?}",
+                    status, content, e
+                ),
+            }
+            .into(),
         }
     }
 
@@ -508,22 +584,24 @@ impl RestNamespace {
             .rest_client
             .execute(req_builder, operation, object_id)
             .await
-            .map_err(|e| Error::io_source(box_error(e)))?;
+            .map_err(Self::request_error)?;
 
         let status = resp.status();
-        let content = resp
-            .text()
-            .await
-            .map_err(|e| Error::io_source(box_error(e)))?;
+        let content = resp.text().await.map_err(|e| {
+            Error::from(NamespaceError::Internal {
+                message: format!("Failed to read response body: {:?}", e),
+            })
+        })?;
 
         if status.is_success() {
             serde_json::from_str(&content).map_err(|e| {
-                Error::namespace_source(format!("Failed to parse response: {}", e).into())
+                NamespaceError::Internal {
+                    message: format!("Failed to parse response: {:?}", e),
+                }
+                .into()
             })
         } else {
-            Err(Error::namespace_source(
-                format!("Response error: status={}, content={}", status, content).into(),
-            ))
+            Err(Self::parse_error_response(status, &content))
         }
     }
 
@@ -543,22 +621,24 @@ impl RestNamespace {
             .rest_client
             .execute(req_builder, operation, object_id)
             .await
-            .map_err(|e| Error::io_source(box_error(e)))?;
+            .map_err(Self::request_error)?;
 
         let status = resp.status();
-        let content = resp
-            .text()
-            .await
-            .map_err(|e| Error::io_source(box_error(e)))?;
+        let content = resp.text().await.map_err(|e| {
+            Error::from(NamespaceError::Internal {
+                message: format!("Failed to read response body: {:?}", e),
+            })
+        })?;
 
         if status.is_success() {
             serde_json::from_str(&content).map_err(|e| {
-                Error::namespace_source(format!("Failed to parse response: {}", e).into())
+                NamespaceError::Internal {
+                    message: format!("Failed to parse response: {:?}", e),
+                }
+                .into()
             })
         } else {
-            Err(Error::namespace_source(
-                format!("Response error: status={}, content={}", status, content).into(),
-            ))
+            Err(Self::parse_error_response(status, &content))
         }
     }
 
@@ -578,19 +658,18 @@ impl RestNamespace {
             .rest_client
             .execute(req_builder, operation, object_id)
             .await
-            .map_err(|e| Error::io_source(box_error(e)))?;
+            .map_err(Self::request_error)?;
 
         let status = resp.status();
         if status.is_success() {
             Ok(())
         } else {
-            let content = resp
-                .text()
-                .await
-                .map_err(|e| Error::io_source(box_error(e)))?;
-            Err(Error::namespace_source(
-                format!("Response error: status={}, content={}", status, content).into(),
-            ))
+            let content = resp.text().await.map_err(|e| {
+                Error::from(NamespaceError::Internal {
+                    message: format!("Failed to read response body: {:?}", e),
+                })
+            })?;
+            Err(Self::parse_error_response(status, &content))
         }
     }
 
@@ -610,63 +689,59 @@ impl RestNamespace {
             .rest_client
             .execute(req_builder, operation, object_id)
             .await
-            .map_err(|e| Error::io_source(box_error(e)))?;
+            .map_err(Self::request_error)?;
 
         let status = resp.status();
-        let content = resp
-            .text()
-            .await
-            .map_err(|e| Error::io_source(box_error(e)))?;
+        let content = resp.text().await.map_err(|e| {
+            Error::from(NamespaceError::Internal {
+                message: format!("Failed to read response body: {:?}", e),
+            })
+        })?;
 
         if status.is_success() {
             serde_json::from_str(&content).map_err(|e| {
-                Error::namespace_source(format!("Failed to parse response: {}", e).into())
+                NamespaceError::Internal {
+                    message: format!("Failed to parse response: {:?}", e),
+                }
+                .into()
             })
         } else {
-            Err(Error::namespace_source(
-                format!("Response error: status={}, content={}", status, content).into(),
-            ))
+            Err(Self::parse_error_response(status, &content))
         }
     }
 
-    /// Execute a POST request with JSON body and get binary response.
-    #[allow(dead_code)]
-    async fn post_json_binary<T: Serialize>(
-        &self,
-        path: &str,
-        query: &[(&str, &str)],
-        body: &T,
-        operation: &str,
-        object_id: &str,
-    ) -> Result<Bytes> {
-        let url = format!("{}{}", self.rest_client.base_path(), path);
-        let req_builder = self.rest_client.client().post(&url).query(query).json(body);
+    /// Get the base endpoint URL for this namespace
+    pub fn endpoint(&self) -> &str {
+        self.rest_client.base_path()
+    }
 
-        let resp = self
-            .rest_client
-            .execute(req_builder, operation, object_id)
-            .await
-            .map_err(|e| Error::io_source(box_error(e)))?;
+    /// Retrieve a snapshot of operation metrics.
+    ///
+    /// Returns a HashMap where keys are operation names (e.g., "list_tables", "describe_table")
+    /// and values are the number of times each operation was called.
+    ///
+    /// Returns an empty HashMap if `ops_metrics_enabled` was false when building the namespace.
+    pub fn retrieve_ops_metrics(&self) -> HashMap<String, u64> {
+        self.ops_metrics
+            .as_ref()
+            .map(|m| m.retrieve())
+            .unwrap_or_default()
+    }
 
-        let status = resp.status();
-        if status.is_success() {
-            resp.bytes()
-                .await
-                .map_err(|e| Error::io_source(box_error(e)))
-        } else {
-            let content = resp
-                .text()
-                .await
-                .map_err(|e| Error::io_source(box_error(e)))?;
-            Err(Error::namespace_source(
-                format!("Response error: status={}, content={}", status, content).into(),
-            ))
+    /// Reset all operation metrics counters to zero.
+    ///
+    /// Does nothing if `ops_metrics_enabled` was false when building the namespace.
+    pub fn reset_ops_metrics(&self) {
+        if let Some(ref metrics) = self.ops_metrics {
+            metrics.reset();
         }
     }
 
-    /// Get the base endpoint URL for this namespace
-    pub fn endpoint(&self) -> &str {
-        self.rest_client.base_path()
+    /// Increment the counter for an operation.
+    fn record_op(&self, operation: &str) {
+        if let Some(ref metrics) = self.ops_metrics {
+            metrics.increment(operation);
+        }
     }
 }
 
@@ -676,6 +751,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: ListNamespacesRequest,
     ) -> Result<ListNamespacesResponse> {
+        self.record_op("list_namespaces");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/namespace/{}/list", encoded_id);
@@ -697,6 +773,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: DescribeNamespaceRequest,
     ) -> Result<DescribeNamespaceResponse> {
+        self.record_op("describe_namespace");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/namespace/{}/describe", encoded_id);
@@ -709,6 +786,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: CreateNamespaceRequest,
     ) -> Result<CreateNamespaceResponse> {
+        self.record_op("create_namespace");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/namespace/{}/create", encoded_id);
@@ -718,6 +796,7 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result<DropNamespaceResponse> {
+        self.record_op("drop_namespace");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/namespace/{}/drop", encoded_id);
@@ -727,6 +806,7 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn namespace_exists(&self, request: NamespaceExistsRequest) -> Result<()> {
+        self.record_op("namespace_exists");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/namespace/{}/exists", encoded_id);
@@ -736,6 +816,7 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
+        self.record_op("list_tables");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/namespace/{}/table/list", encoded_id);
@@ -754,6 +835,7 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn describe_table(&self, request: DescribeTableRequest) -> Result<DescribeTableResponse> {
+        self.record_op("describe_table");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/describe", encoded_id);
@@ -773,6 +855,7 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn register_table(&self, request: RegisterTableRequest) -> Result<RegisterTableResponse> {
+        self.record_op("register_table");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/register", encoded_id);
@@ -782,6 +865,7 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn table_exists(&self, request: TableExistsRequest) -> Result<()> {
+        self.record_op("table_exists");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/exists", encoded_id);
@@ -791,6 +875,7 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn drop_table(&self, request: DropTableRequest) -> Result<DropTableResponse> {
+        self.record_op("drop_table");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/drop", encoded_id);
@@ -803,6 +888,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: DeregisterTableRequest,
     ) -> Result<DeregisterTableResponse> {
+        self.record_op("deregister_table");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/deregister", encoded_id);
@@ -812,6 +898,7 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn count_table_rows(&self, request: CountTableRowsRequest) -> Result<i64> {
+        self.record_op("count_table_rows");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/count_rows", encoded_id);
@@ -824,6 +911,7 @@ impl LanceNamespace for RestNamespace {
         request: CreateTableRequest,
         request_data: Bytes,
     ) -> Result<CreateTableResponse> {
+        self.record_op("create_table");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/create", encoded_id);
@@ -838,6 +926,7 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn declare_table(&self, request: DeclareTableRequest) -> Result<DeclareTableResponse> {
+        self.record_op("declare_table");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/declare", encoded_id);
@@ -851,6 +940,7 @@ impl LanceNamespace for RestNamespace {
         request: InsertIntoTableRequest,
         request_data: Bytes,
     ) -> Result<InsertIntoTableResponse> {
+        self.record_op("insert_into_table");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/insert", encoded_id);
@@ -875,11 +965,14 @@ impl LanceNamespace for RestNamespace {
         request: MergeInsertIntoTableRequest,
         request_data: Bytes,
     ) -> Result<MergeInsertIntoTableResponse> {
+        self.record_op("merge_insert_into_table");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
 
         let on = request.on.as_deref().ok_or_else(|| {
-            Error::namespace_source("'on' field is required for merge insert".into())
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "'on' field is required for merge insert".to_string(),
+            })
         })?;
 
         let path = format!("/v1/table/{}/merge_insert", encoded_id);
@@ -935,6 +1028,7 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn update_table(&self, request: UpdateTableRequest) -> Result<UpdateTableResponse> {
+        self.record_op("update_table");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/update", encoded_id);
@@ -947,6 +1041,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: DeleteFromTableRequest,
     ) -> Result<DeleteFromTableResponse> {
+        self.record_op("delete_from_table");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/delete", encoded_id);
@@ -956,10 +1051,12 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn query_table(&self, request: QueryTableRequest) -> Result<Bytes> {
+        self.record_op("query_table");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/query", encoded_id);
         let query = [("delimiter", self.delimiter.as_str())];
+        let operation = "query_table";
 
         let url = format!("{}{}", self.rest_client.base_path(), path);
         let req_builder = self
@@ -971,23 +1068,24 @@ impl LanceNamespace for RestNamespace {
 
         let resp = self
             .rest_client
-            .execute(req_builder, "query_table", &id)
+            .execute(req_builder, operation, &id)
             .await
-            .map_err(|e| Error::io_source(box_error(e)))?;
+            .map_err(Self::request_error)?;
 
         let status = resp.status();
         if status.is_success() {
-            resp.bytes()
-                .await
-                .map_err(|e| Error::io_source(box_error(e)))
+            resp.bytes().await.map_err(|e| {
+                Error::from(NamespaceError::Internal {
+                    message: format!("Failed to read response bytes: {:?}", e),
+                })
+            })
         } else {
-            let content = resp
-                .text()
-                .await
-                .map_err(|e| Error::io_source(box_error(e)))?;
-            Err(Error::namespace_source(
-                format!("Response error: status={}, content={}", status, content).into(),
-            ))
+            let content = resp.text().await.map_err(|e| {
+                Error::from(NamespaceError::Internal {
+                    message: format!("Failed to read response body: {:?}", e),
+                })
+            })?;
+            Err(Self::parse_error_response(status, &content))
         }
     }
 
@@ -995,6 +1093,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: CreateTableIndexRequest,
     ) -> Result<CreateTableIndexResponse> {
+        self.record_op("create_table_index");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/create_index", encoded_id);
@@ -1007,6 +1106,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: ListTableIndicesRequest,
     ) -> Result<ListTableIndicesResponse> {
+        self.record_op("list_table_indices");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/index/list", encoded_id);
@@ -1019,6 +1119,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: DescribeTableIndexStatsRequest,
     ) -> Result<DescribeTableIndexStatsResponse> {
+        self.record_op("describe_table_index_stats");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let index_name = request.index_name.as_deref().unwrap_or("");
@@ -1036,6 +1137,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: DescribeTransactionRequest,
     ) -> Result<DescribeTransactionResponse> {
+        self.record_op("describe_transaction");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/transaction/{}/describe", encoded_id);
@@ -1048,6 +1150,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: AlterTransactionRequest,
     ) -> Result<AlterTransactionResponse> {
+        self.record_op("alter_transaction");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/transaction/{}/alter", encoded_id);
@@ -1060,6 +1163,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: CreateTableIndexRequest,
     ) -> Result<CreateTableScalarIndexResponse> {
+        self.record_op("create_table_scalar_index");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/create_scalar_index", encoded_id);
@@ -1072,6 +1176,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: DropTableIndexRequest,
     ) -> Result<DropTableIndexResponse> {
+        self.record_op("drop_table_index");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let index_name = request.index_name.as_deref().unwrap_or("");
@@ -1086,6 +1191,7 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn list_all_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
+        self.record_op("list_all_tables");
         let path = "/v1/table";
         let mut query = vec![("delimiter", self.delimiter.as_str())];
         let page_token_str;
@@ -1102,6 +1208,7 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn restore_table(&self, request: RestoreTableRequest) -> Result<RestoreTableResponse> {
+        self.record_op("restore_table");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/restore", encoded_id);
@@ -1111,6 +1218,7 @@ impl LanceNamespace for RestNamespace {
     }
 
     async fn rename_table(&self, request: RenameTableRequest) -> Result<RenameTableResponse> {
+        self.record_op("rename_table");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/rename", encoded_id);
@@ -1123,6 +1231,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: ListTableVersionsRequest,
     ) -> Result<ListTableVersionsResponse> {
+        self.record_op("list_table_versions");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/version/list", encoded_id);
@@ -1150,6 +1259,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: CreateTableVersionRequest,
     ) -> Result<CreateTableVersionResponse> {
+        self.record_op("create_table_version");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/version/create", encoded_id);
@@ -1162,6 +1272,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: DescribeTableVersionRequest,
     ) -> Result<DescribeTableVersionResponse> {
+        self.record_op("describe_table_version");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/version/describe", encoded_id);
@@ -1170,10 +1281,24 @@ impl LanceNamespace for RestNamespace {
             .await
     }
 
+    async fn batch_delete_table_versions(
+        &self,
+        request: BatchDeleteTableVersionsRequest,
+    ) -> Result<BatchDeleteTableVersionsResponse> {
+        self.record_op("batch_delete_table_versions");
+        let id = object_id_str(&request.id, &self.delimiter)?;
+        let encoded_id = urlencode(&id);
+        let path = format!("/v1/table/{}/version/delete", encoded_id);
+        let query = [("delimiter", self.delimiter.as_str())];
+        self.post_json(&path, &query, &request, "batch_delete_table_versions", &id)
+            .await
+    }
+
     async fn update_table_schema_metadata(
         &self,
         request: UpdateTableSchemaMetadataRequest,
     ) -> Result<UpdateTableSchemaMetadataResponse> {
+        self.record_op("update_table_schema_metadata");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/schema_metadata/update", encoded_id);
@@ -1198,6 +1323,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: GetTableStatsRequest,
     ) -> Result<GetTableStatsResponse> {
+        self.record_op("get_table_stats");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/stats", encoded_id);
@@ -1210,6 +1336,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: ExplainTableQueryPlanRequest,
     ) -> Result<String> {
+        self.record_op("explain_table_query_plan");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/explain_plan", encoded_id);
@@ -1222,6 +1349,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: AnalyzeTableQueryPlanRequest,
     ) -> Result<String> {
+        self.record_op("analyze_table_query_plan");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/analyze_plan", encoded_id);
@@ -1234,6 +1362,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: AlterTableAddColumnsRequest,
     ) -> Result<AlterTableAddColumnsResponse> {
+        self.record_op("alter_table_add_columns");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/add_columns", encoded_id);
@@ -1246,6 +1375,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: AlterTableAlterColumnsRequest,
     ) -> Result<AlterTableAlterColumnsResponse> {
+        self.record_op("alter_table_alter_columns");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/alter_columns", encoded_id);
@@ -1258,6 +1388,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: AlterTableDropColumnsRequest,
     ) -> Result<AlterTableDropColumnsResponse> {
+        self.record_op("alter_table_drop_columns");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/drop_columns", encoded_id);
@@ -1270,6 +1401,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: ListTableTagsRequest,
     ) -> Result<ListTableTagsResponse> {
+        self.record_op("list_table_tags");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/tags/list", encoded_id);
@@ -1291,6 +1423,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: GetTableTagVersionRequest,
     ) -> Result<GetTableTagVersionResponse> {
+        self.record_op("get_table_tag_version");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/tags/version", encoded_id);
@@ -1303,6 +1436,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: CreateTableTagRequest,
     ) -> Result<CreateTableTagResponse> {
+        self.record_op("create_table_tag");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/tags/create", encoded_id);
@@ -1315,6 +1449,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: DeleteTableTagRequest,
     ) -> Result<DeleteTableTagResponse> {
+        self.record_op("delete_table_tag");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/tags/delete", encoded_id);
@@ -1327,6 +1462,7 @@ impl LanceNamespace for RestNamespace {
         &self,
         request: UpdateTableTagRequest,
     ) -> Result<UpdateTableTagResponse> {
+        self.record_op("update_table_tag");
         let id = object_id_str(&request.id, &self.delimiter)?;
         let encoded_id = urlencode(&id);
         let path = format!("/v1/table/{}/tags/update", encoded_id);
diff --git a/rust/lance-namespace-impls/src/rest_adapter.rs b/rust/lance-namespace-impls/src/rest_adapter.rs
index e2d30299151..83901e62eb2 100644
--- a/rust/lance-namespace-impls/src/rest_adapter.rs
+++ b/rust/lance-namespace-impls/src/rest_adapter.rs
@@ -12,12 +12,12 @@ use std::sync::Arc;
 use axum::{
     Json, Router, ServiceExt,
     body::Bytes,
-    extract::{Path, Query, Request, State},
+    extract::{FromRequest, Path, Query, Request, State},
     http::{HeaderMap, StatusCode},
     response::{IntoResponse, Response},
     routing::{get, post},
 };
-use serde::Deserialize;
+use serde::{Deserialize, de::DeserializeOwned};
 use tokio::sync::watch;
 use tower::Layer;
 use tower_http::normalize_path::NormalizePathLayer;
@@ -25,6 +25,7 @@ use tower_http::trace::TraceLayer;
 
 use lance_core::{Error, Result};
 use lance_namespace::LanceNamespace;
+use lance_namespace::error::NamespaceError;
 use lance_namespace::models::*;
 
 /// Configuration for the REST server
@@ -81,6 +82,10 @@ impl RestAdapter {
                 "/v1/table/:id/version/describe",
                 post(describe_table_version),
             )
+            .route(
+                "/v1/table/:id/version/delete",
+                post(batch_delete_table_versions),
+            )
             .route("/v1/table/:id/stats", get(get_table_stats))
             // Table data operations
             .route("/v1/table/:id/create", post(create_table))
@@ -97,7 +102,7 @@ impl RestAdapter {
                 "/v1/table/:id/create_scalar_index",
                 post(create_table_scalar_index),
             )
-            .route("/v1/table/:id/index/list", get(list_table_indices))
+            .route("/v1/table/:id/index/list", post(list_table_indices))
             .route(
                 "/v1/table/:id/index/:index_name/stats",
                 get(describe_table_index_stats),
@@ -149,7 +154,9 @@ impl RestAdapter {
 
         let listener = tokio::net::TcpListener::bind(&addr).await.map_err(|e| {
             log::error!("RestAdapter::start() failed to bind to {}: {}", addr, e);
-            Error::io_source(Box::new(e))
+            Error::from(NamespaceError::Internal {
+                message: format!("Failed to bind to {}: {:?}", addr, e),
+            })
         })?;
 
         // Get the actual port (important when port 0 was specified)
@@ -225,9 +232,51 @@ impl RestAdapterHandle {
 }
 
 // ============================================================================
-// Query Parameters
+// Query Parameters and Extractors
 // ============================================================================
 
+/// Optional JSON body extractor that allows empty request bodies.
+/// Similar to sophon's MaybeJson - returns None if body is empty.
+struct MaybeJson<T>(Option<T>);
+
+impl<S, T> FromRequest<S> for MaybeJson<T>
+where
+    S: Send + Sync,
+    T: DeserializeOwned + Send + 'static,
+{
+    type Rejection = Response;
+
+    fn from_request<'life0, 'async_trait>(
+        req: Request,
+        state: &'life0 S,
+    ) -> std::pin::Pin<
+        Box<
+            dyn std::future::Future<Output = std::result::Result<Self, Self::Rejection>>
+                + Send
+                + 'async_trait,
+        >,
+    >
+    where
+        'life0: 'async_trait,
+        Self: 'async_trait,
+    {
+        Box::pin(async move {
+            let bytes = Bytes::from_request(req, state)
+                .await
+                .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()).into_response())?;
+
+            if bytes.is_empty() {
+                return Ok(Self(None));
+            }
+
+            match serde_json::from_slice(&bytes) {
+                Ok(value) => Ok(Self(Some(value))),
+                Err(e) => Err((StatusCode::BAD_REQUEST, e.to_string()).into_response()),
+            }
+        })
+    }
+}
+
 #[derive(Debug, Deserialize)]
 struct DelimiterQuery {
     delimiter: Option<String>,
@@ -245,66 +294,61 @@ struct PaginationQuery {
 // Error Conversion
 // ============================================================================
 
-/// Convert Lance errors to HTTP responses
+/// Map a NamespaceError error code to an HTTP status code.
+fn error_code_to_status(code: u32) -> StatusCode {
+    match lance_namespace::error::ErrorCode::from_u32(code) {
+        Some(lance_namespace::error::ErrorCode::NamespaceNotFound)
+        | Some(lance_namespace::error::ErrorCode::TableNotFound)
+        | Some(lance_namespace::error::ErrorCode::TableIndexNotFound)
+        | Some(lance_namespace::error::ErrorCode::TableTagNotFound)
+        | Some(lance_namespace::error::ErrorCode::TransactionNotFound)
+        | Some(lance_namespace::error::ErrorCode::TableVersionNotFound)
+        | Some(lance_namespace::error::ErrorCode::TableColumnNotFound) => StatusCode::NOT_FOUND,
+        Some(lance_namespace::error::ErrorCode::NamespaceAlreadyExists)
+        | Some(lance_namespace::error::ErrorCode::TableAlreadyExists)
+        | Some(lance_namespace::error::ErrorCode::TableIndexAlreadyExists)
+        | Some(lance_namespace::error::ErrorCode::TableTagAlreadyExists)
+        | Some(lance_namespace::error::ErrorCode::ConcurrentModification) => StatusCode::CONFLICT,
+        Some(lance_namespace::error::ErrorCode::NamespaceNotEmpty)
+        | Some(lance_namespace::error::ErrorCode::InvalidTableState) => StatusCode::CONFLICT,
+        Some(lance_namespace::error::ErrorCode::InvalidInput)
+        | Some(lance_namespace::error::ErrorCode::TableSchemaValidationError) => {
+            StatusCode::BAD_REQUEST
+        }
+        Some(lance_namespace::error::ErrorCode::Unsupported) => StatusCode::NOT_ACCEPTABLE,
+        Some(lance_namespace::error::ErrorCode::PermissionDenied) => StatusCode::FORBIDDEN,
+        Some(lance_namespace::error::ErrorCode::Unauthenticated) => StatusCode::UNAUTHORIZED,
+        Some(lance_namespace::error::ErrorCode::ServiceUnavailable) => {
+            StatusCode::SERVICE_UNAVAILABLE
+        }
+        Some(lance_namespace::error::ErrorCode::Throttling) => StatusCode::TOO_MANY_REQUESTS,
+        Some(lance_namespace::error::ErrorCode::Internal) | None => {
+            StatusCode::INTERNAL_SERVER_ERROR
+        }
+    }
+}
+
+/// Convert Lance errors to HTTP responses using the spec's `ErrorResponse` model.
 fn error_to_response(err: Error) -> Response {
     match err {
         Error::Namespace { source, .. } => {
-            let error_msg = source.to_string();
-            if error_msg.contains("not found") || error_msg.contains("does not exist") {
-                (
-                    StatusCode::NOT_FOUND,
-                    Json(serde_json::json!({
-                        "error": {
-                            "message": error_msg,
-                            "type": "NamespaceNotFoundException"
-                        }
-                    })),
-                )
-                    .into_response()
-            } else if error_msg.contains("already exists") {
-                (
-                    StatusCode::BAD_REQUEST,
-                    Json(serde_json::json!({
-                        "error": {
-                            "message": error_msg,
-                            "type": "TableAlreadyExistsException"
-                        }
-                    })),
-                )
-                    .into_response()
+            if let Some(ns_err) = source.downcast_ref::<NamespaceError>() {
+                let code = ns_err.code().as_u32();
+                let status = error_code_to_status(code);
+                let mut resp = ErrorResponse::new(code as i32);
+                resp.error = Some(ns_err.message().to_string());
+                (status, Json(resp)).into_response()
             } else {
-                (
-                    StatusCode::BAD_REQUEST,
-                    Json(serde_json::json!({
-                        "error": {
-                            "message": error_msg,
-                            "type": "NamespaceException"
-                        }
-                    })),
-                )
-                    .into_response()
+                let mut resp = ErrorResponse::new(18);
+                resp.error = Some(source.to_string());
+                (StatusCode::INTERNAL_SERVER_ERROR, Json(resp)).into_response()
             }
         }
-        Error::IO { source, .. } => (
-            StatusCode::INTERNAL_SERVER_ERROR,
-            Json(serde_json::json!({
-                "error": {
-                    "message": source.to_string(),
-                    "type": "InternalServerError"
-                }
-            })),
-        )
-            .into_response(),
-        _ => (
-            StatusCode::INTERNAL_SERVER_ERROR,
-            Json(serde_json::json!({
-                "error": {
-                    "message": err.to_string(),
-                    "type": "InternalServerError"
-                }
-            })),
-        )
-            .into_response(),
+        _ => {
+            let mut resp = ErrorResponse::new(18);
+            resp.error = Some(err.to_string());
+            (StatusCode::INTERNAL_SERVER_ERROR, Json(resp)).into_response()
+        }
     }
 }
 
@@ -783,6 +827,26 @@ async fn describe_table_version(
     }
 }
 
+async fn batch_delete_table_versions(
+    State(backend): State<Arc<dyn LanceNamespace>>,
+    headers: HeaderMap,
+    Path(id): Path<String>,
+    Query(params): Query<DelimiterQuery>,
+    Json(body): Json<BatchDeleteTableVersionsRequest>,
+) -> Response {
+    let request = BatchDeleteTableVersionsRequest {
+        id: Some(parse_id(&id, params.delimiter.as_deref())),
+        identity: extract_identity(&headers),
+        ranges: body.ranges,
+        ..Default::default()
+    };
+
+    match backend.batch_delete_table_versions(request).await {
+        Ok(response) => (StatusCode::OK, Json(response)).into_response(),
+        Err(e) => error_to_response(e),
+    }
+}
+
 async fn get_table_stats(
     State(backend): State<Arc<dyn LanceNamespace>>,
     headers: HeaderMap,
@@ -861,15 +925,11 @@ async fn list_table_indices(
     headers: HeaderMap,
     Path(id): Path<String>,
     Query(params): Query<DelimiterQuery>,
+    MaybeJson(body): MaybeJson<ListTableIndicesRequest>,
 ) -> Response {
-    let request = ListTableIndicesRequest {
-        id: Some(parse_id(&id, params.delimiter.as_deref())),
-        version: None,
-        page_token: None,
-        limit: None,
-        identity: extract_identity(&headers),
-        ..Default::default()
-    };
+    let mut request = body.unwrap_or_default();
+    request.id = Some(parse_id(&id, params.delimiter.as_deref()));
+    request.identity = extract_identity(&headers);
 
     match backend.list_table_indices(request).await {
         Ok(response) => (StatusCode::OK, Json(response)).into_response(),
@@ -1347,6 +1407,55 @@ mod tests {
             Bytes::from(buffer)
         }
 
+        /// Helper to create Arrow IPC data with vector column for testing vector index
+        fn create_test_vector_data(num_rows: usize, dim: i32) -> Bytes {
+            use arrow::array::{FixedSizeListArray, Float32Array, Int32Array};
+            use arrow::datatypes::{DataType, Field, Schema};
+            use arrow::ipc::writer::StreamWriter;
+            use arrow::record_batch::RecordBatch;
+
+            let schema = Arc::new(Schema::new(vec![
+                Field::new("id", DataType::Int32, false),
+                Field::new(
+                    "vector",
+                    DataType::FixedSizeList(
+                        Arc::new(Field::new("item", DataType::Float32, true)),
+                        dim,
+                    ),
+                    true,
+                ),
+            ]));
+
+            let ids: Vec<i32> = (0..num_rows as i32).collect();
+            let vector_values: Vec<f32> = (0..(num_rows * dim as usize))
+                .map(|i| (i as f32) * 0.01)
+                .collect();
+
+            let vector_field = Arc::new(Field::new("item", DataType::Float32, true));
+            let vectors = FixedSizeListArray::try_new(
+                vector_field,
+                dim,
+                Arc::new(Float32Array::from(vector_values)),
+                None,
+            )
+            .unwrap();
+
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(Int32Array::from(ids)), Arc::new(vectors)],
+            )
+            .unwrap();
+
+            let mut buffer = Vec::new();
+            {
+                let mut writer = StreamWriter::try_new(&mut buffer, &schema).unwrap();
+                writer.write(&batch).unwrap();
+                writer.finish().unwrap();
+            }
+
+            Bytes::from(buffer)
+        }
+
         #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
         async fn test_trailing_slash_handling() {
             let fixture = RestServerFixture::new().await;
@@ -2183,8 +2292,8 @@ mod tests {
             );
             let err_msg = result.unwrap_err().to_string();
             assert!(
-                err_msg.contains("is not empty"),
-                "Error should be 'is not empty', got: {}",
+                err_msg.contains("not empty"),
+                "Error should contain 'not empty', got: {}",
                 err_msg
             );
         }
@@ -2284,8 +2393,8 @@ mod tests {
             assert!(result.is_err(), "Cannot create root namespace");
             let err_msg = result.unwrap_err().to_string();
             assert!(
-                err_msg.contains("Root namespace already exists and cannot be created"),
-                "Error should be 'Root namespace already exists and cannot be created', got: {}",
+                err_msg.contains("already exists") && err_msg.contains("root namespace"),
+                "Error should contain 'already exists' and 'root namespace', got: {}",
                 err_msg
             );
 
@@ -3021,5 +3130,144 @@ mod tests {
                 "Latest version should be 1"
             );
         }
+
+        #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+        async fn test_create_and_list_table_index() {
+            let fixture = RestServerFixture::new().await;
+            let table_data = create_test_arrow_data();
+
+            // Create namespace
+            let create_ns_req = CreateNamespaceRequest {
+                id: Some(vec!["index_test_ns".to_string()]),
+                ..Default::default()
+            };
+            fixture
+                .namespace
+                .create_namespace(create_ns_req)
+                .await
+                .unwrap();
+
+            // Create table
+            let create_table_req = CreateTableRequest {
+                id: Some(vec![
+                    "index_test_ns".to_string(),
+                    "index_test_table".to_string(),
+                ]),
+                mode: Some("Create".to_string()),
+                ..Default::default()
+            };
+            fixture
+                .namespace
+                .create_table(create_table_req, table_data)
+                .await
+                .unwrap();
+
+            // Create scalar index on 'id' column
+            let create_index_req = CreateTableIndexRequest {
+                id: Some(vec![
+                    "index_test_ns".to_string(),
+                    "index_test_table".to_string(),
+                ]),
+                column: "id".to_string(),
+                index_type: "BTREE".to_string(),
+                name: Some("id_idx".to_string()),
+                ..Default::default()
+            };
+            let result = fixture.namespace.create_table_index(create_index_req).await;
+            assert!(result.is_ok(), "Failed to create index: {:?}", result.err());
+
+            // List indices
+            let list_indices_req = ListTableIndicesRequest {
+                id: Some(vec![
+                    "index_test_ns".to_string(),
+                    "index_test_table".to_string(),
+                ]),
+                ..Default::default()
+            };
+            let result = fixture.namespace.list_table_indices(list_indices_req).await;
+            assert!(result.is_ok(), "Failed to list indices: {:?}", result.err());
+            let indices = result.unwrap();
+            assert_eq!(indices.indexes.len(), 1, "Should have exactly one index");
+            assert_eq!(
+                indices.indexes[0].index_name, "id_idx",
+                "Index name should match"
+            );
+            assert_eq!(
+                indices.indexes[0].columns,
+                vec!["id"],
+                "Index column should be 'id'"
+            );
+        }
+
+        #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+        async fn test_create_vector_index() {
+            let fixture = RestServerFixture::new().await;
+            // Create 256 rows with 8-dimensional vectors for vector index
+            let table_data = create_test_vector_data(256, 8);
+
+            // Create namespace
+            let create_ns_req = CreateNamespaceRequest {
+                id: Some(vec!["vector_index_ns".to_string()]),
+                ..Default::default()
+            };
+            fixture
+                .namespace
+                .create_namespace(create_ns_req)
+                .await
+                .unwrap();
+
+            // Create table with vector data
+            let create_table_req = CreateTableRequest {
+                id: Some(vec![
+                    "vector_index_ns".to_string(),
+                    "vector_table".to_string(),
+                ]),
+                mode: Some("Create".to_string()),
+                ..Default::default()
+            };
+            fixture
+                .namespace
+                .create_table(create_table_req, table_data)
+                .await
+                .unwrap();
+
+            // Create vector index on 'vector' column using IVF_FLAT
+            let mut create_index_req =
+                CreateTableIndexRequest::new("vector".to_string(), "IVF_FLAT".to_string());
+            create_index_req.id = Some(vec![
+                "vector_index_ns".to_string(),
+                "vector_table".to_string(),
+            ]);
+            create_index_req.name = Some("vector_idx".to_string());
+            create_index_req.distance_type = Some("l2".to_string());
+            let result = fixture.namespace.create_table_index(create_index_req).await;
+            assert!(
+                result.is_ok(),
+                "Failed to create vector index: {:?}",
+                result.err()
+            );
+
+            // List indices to verify
+            let list_indices_req = ListTableIndicesRequest {
+                id: Some(vec![
+                    "vector_index_ns".to_string(),
+                    "vector_table".to_string(),
+                ]),
+                ..Default::default()
+            };
+            let result = fixture.namespace.list_table_indices(list_indices_req).await;
+            assert!(result.is_ok(), "Failed to list indices: {:?}", result.err());
+            let indices = result.unwrap();
+            assert_eq!(indices.indexes.len(), 1, "Should have exactly one index");
+            assert_eq!(
+                indices.indexes[0].index_name, "vector_idx",
+                "Index name should match"
+            );
+            assert_eq!(
+                indices.indexes[0].columns,
+                vec!["vector"],
+                "Index column should be 'vector'"
+            );
+        }
     }
 }
diff --git a/rust/lance-namespace/Cargo.toml b/rust/lance-namespace/Cargo.toml
index f0aa59c91b8..1bb1358d486 100644
--- a/rust/lance-namespace/Cargo.toml
+++ b/rust/lance-namespace/Cargo.toml
@@ -16,6 +16,7 @@ async-trait.workspace = true
 bytes.workspace = true
 arrow.workspace = true
 lance-core.workspace = true
+serde.workspace = true
 snafu.workspace = true
 lance-namespace-reqwest-client.workspace = true
 
diff --git a/rust/lance-namespace/src/error.rs b/rust/lance-namespace/src/error.rs
index 78b850b5e61..8a73d4db8e8 100644
--- a/rust/lance-namespace/src/error.rs
+++ b/rust/lance-namespace/src/error.rs
@@ -77,7 +77,7 @@ pub enum ErrorCode {
     /// Table schema validation failed
     TableSchemaValidationError = 20,
     /// Request was throttled due to rate limiting or too many concurrent operations
-    Throttled = 21,
+    Throttling = 21,
 }
 
 impl ErrorCode {
@@ -112,7 +112,7 @@ impl ErrorCode {
             18 => Some(Self::Internal),
             19 => Some(Self::InvalidTableState),
             20 => Some(Self::TableSchemaValidationError),
-            21 => Some(Self::Throttled),
+            21 => Some(Self::Throttling),
             _ => None,
         }
     }
@@ -142,7 +142,7 @@ impl std::fmt::Display for ErrorCode {
             Self::Internal => "Internal",
             Self::InvalidTableState => "InvalidTableState",
             Self::TableSchemaValidationError => "TableSchemaValidationError",
-            Self::Throttled => "Throttled",
+            Self::Throttling => "Throttling",
         };
         write!(f, "{}", name)
     }
@@ -258,11 +258,43 @@ pub enum NamespaceError {
     TableSchemaValidationError { message: String },
 
     /// Request was throttled due to rate limiting or too many concurrent operations.
-    #[snafu(display("Throttled: {message}"))]
-    Throttled { message: String },
+    #[snafu(display("Throttling: {message}"))]
+    Throttling { message: String },
 }
 
 impl NamespaceError {
+    /// Returns the inner message without the Display prefix.
+    ///
+    /// Useful when serializing across boundaries (e.g. REST) where
+    /// the receiver will reconstruct the variant from the error code
+    /// and re-apply its own Display formatting.
+    pub fn message(&self) -> &str {
+        match self {
+            Self::Unsupported { message }
+            | Self::NamespaceNotFound { message }
+            | Self::NamespaceAlreadyExists { message }
+            | Self::NamespaceNotEmpty { message }
+            | Self::TableNotFound { message }
+            | Self::TableAlreadyExists { message }
+            | Self::TableIndexNotFound { message }
+            | Self::TableIndexAlreadyExists { message }
+            | Self::TableTagNotFound { message }
+            | Self::TableTagAlreadyExists { message }
+            | Self::TransactionNotFound { message }
+            | Self::TableVersionNotFound { message }
+            | Self::TableColumnNotFound { message }
+            | Self::InvalidInput { message }
+            | Self::ConcurrentModification { message }
+            | Self::PermissionDenied { message }
+            | Self::Unauthenticated { message }
+            | Self::ServiceUnavailable { message }
+            | Self::Internal { message }
+            | Self::InvalidTableState { message }
+            | Self::TableSchemaValidationError { message }
+            | Self::Throttling { message } => message,
+        }
+    }
+
     /// Returns the error code for this error.
     ///
     /// Use this for programmatic error handling across language boundaries.
@@ -289,7 +321,7 @@ impl NamespaceError {
             Self::Internal { .. } => ErrorCode::Internal,
             Self::InvalidTableState { .. } => ErrorCode::InvalidTableState,
             Self::TableSchemaValidationError { .. } => ErrorCode::TableSchemaValidationError,
-            Self::Throttled { .. } => ErrorCode::Throttled,
+            Self::Throttling { .. } => ErrorCode::Throttling,
         }
     }
 
@@ -322,7 +354,7 @@ impl NamespaceError {
             Some(ErrorCode::TableSchemaValidationError) => {
                 Self::TableSchemaValidationError { message }
             }
-            Some(ErrorCode::Throttled) => Self::Throttled { message },
+            Some(ErrorCode::Throttling) => Self::Throttling { message },
             None => Self::Internal { message },
         }
     }
diff --git a/rust/lance-namespace/src/namespace.rs b/rust/lance-namespace/src/namespace.rs
index 25776e26e76..610dd03af9e 100644
--- a/rust/lance-namespace/src/namespace.rs
+++ b/rust/lance-namespace/src/namespace.rs
@@ -339,8 +339,7 @@ pub trait LanceNamespace: Send + Sync + std::fmt::Debug {
 
     /// Batch delete table versions.
     ///
-    /// Deletes multiple version records from a table. This operation supports
-    /// deleting ranges of versions for efficient bulk cleanup.
+    /// Deletes version records for a single table using `request.id` + `request.ranges`.
     ///
     /// # Arguments
     ///
diff --git a/rust/lance-namespace/src/schema.rs b/rust/lance-namespace/src/schema.rs
index 3f44847bcd4..69aa59a51e9 100644
--- a/rust/lance-namespace/src/schema.rs
+++ b/rust/lance-namespace/src/schema.rs
@@ -248,9 +248,12 @@ pub fn convert_json_arrow_field(json_field: &JsonArrowField) -> Result<Field> {
 
 /// Convert JsonArrowDataType to Arrow DataType
 pub fn convert_json_arrow_type(json_type: &JsonArrowDataType) -> Result<DataType> {
+    use std::sync::Arc;
+
     let type_name = json_type.r#type.to_lowercase();
 
     match type_name.as_str() {
+        // Primitive types
         "null" => Ok(DataType::Null),
         "bool" | "boolean" => Ok(DataType::Boolean),
         "int8" => Ok(DataType::Int8),
@@ -261,10 +264,108 @@ pub fn convert_json_arrow_type(json_type: &JsonArrowDataType) -> Result<DataType
         "uint32" => Ok(DataType::UInt32),
         "int64" => Ok(DataType::Int64),
         "uint64" => Ok(DataType::UInt64),
+        "float16" => Ok(DataType::Float16),
         "float32" => Ok(DataType::Float32),
         "float64" => Ok(DataType::Float64),
+
+        // Decimal types - encoding: precision * 1000 + scale
+        // Decoding must handle negative scale: precision = ((encoded + 128) / 1000)
+        "decimal32" => {
+            let encoded = json_type.length.unwrap_or(0);
+            let precision = ((encoded + 128) / 1000) as u8;
+            let scale = (encoded - precision as i64 * 1000) as i8;
+            Ok(DataType::Decimal32(precision, scale))
+        }
+        "decimal64" => {
+            let encoded = json_type.length.unwrap_or(0);
+            let precision = ((encoded + 128) / 1000) as u8;
+            let scale = (encoded - precision as i64 * 1000) as i8;
+            Ok(DataType::Decimal64(precision, scale))
+        }
+        "decimal128" => {
+            let encoded = json_type.length.unwrap_or(0);
+            let precision = ((encoded + 128) / 1000) as u8;
+            let scale = (encoded - precision as i64 * 1000) as i8;
+            Ok(DataType::Decimal128(precision, scale))
+        }
+        "decimal256" => {
+            let encoded = json_type.length.unwrap_or(0);
+            let precision = ((encoded + 128) / 1000) as u8;
+            let scale = (encoded - precision as i64 * 1000) as i8;
+            Ok(DataType::Decimal256(precision, scale))
+        }
+
+        // Date/Time types
+        "date32" => Ok(DataType::Date32),
+        "date64" => Ok(DataType::Date64),
+        "timestamp" => Ok(DataType::Timestamp(
+            arrow::datatypes::TimeUnit::Microsecond,
+            None,
+        )),
+        "duration" => Ok(DataType::Duration(arrow::datatypes::TimeUnit::Microsecond)),
+
+        // String and Binary types
         "utf8" => Ok(DataType::Utf8),
+        "large_utf8" => Ok(DataType::LargeUtf8),
         "binary" => Ok(DataType::Binary),
+        "large_binary" => Ok(DataType::LargeBinary),
+        "fixed_size_binary" => {
+            let size = json_type.length.unwrap_or(0) as i32;
+            Ok(DataType::FixedSizeBinary(size))
+        }
+
+        // Nested types
+        "list" => {
+            let inner = json_type
+                .fields
+                .as_ref()
+                .and_then(|f| f.first())
+                .ok_or_else(|| Error::namespace("list type missing inner field"))?;
+            Ok(DataType::List(Arc::new(convert_json_arrow_field(inner)?)))
+        }
+        "large_list" => {
+            let inner = json_type
+                .fields
+                .as_ref()
+                .and_then(|f| f.first())
+                .ok_or_else(|| Error::namespace("large_list type missing inner field"))?;
+            Ok(DataType::LargeList(Arc::new(convert_json_arrow_field(
+                inner,
+            )?)))
+        }
+        "fixed_size_list" => {
+            let inner = json_type
+                .fields
+                .as_ref()
+                .and_then(|f| f.first())
+                .ok_or_else(|| Error::namespace("fixed_size_list type missing inner field"))?;
+            let size = json_type.length.unwrap_or(0) as i32;
+            Ok(DataType::FixedSizeList(
+                Arc::new(convert_json_arrow_field(inner)?),
+                size,
+            ))
+        }
+        "struct" => {
+            let fields = json_type
+                .fields
+                .as_ref()
+                .ok_or_else(|| Error::namespace("struct type missing fields"))?;
+            let arrow_fields: Result<Vec<Field>> =
+                fields.iter().map(convert_json_arrow_field).collect();
+            Ok(DataType::Struct(arrow_fields?.into()))
+        }
+        "map" => {
+            let entries = json_type
+                .fields
+                .as_ref()
+                .and_then(|f| f.first())
+                .ok_or_else(|| Error::namespace("map type missing entries field"))?;
+            Ok(DataType::Map(
+                Arc::new(convert_json_arrow_field(entries)?),
+                false,
+            ))
+        }
+
         _ => Err(Error::namespace(format!(
             "Unsupported Arrow type: {}",
             type_name
@@ -524,4 +625,205 @@ mod tests {
         let float16 = arrow_type_to_json(&DataType::Float16).unwrap();
         assert_eq!(float16.r#type, "float16");
     }
+
+    /// Verify that convert_json_arrow_type (deserialization) is the inverse of
+    /// arrow_type_to_json (serialization) for all supported types.
+    #[test]
+    fn test_json_arrow_type_roundtrip() {
+        use arrow::datatypes::Field;
+
+        let cases: Vec<DataType> = vec![
+            // Scalars
+            DataType::Null,
+            DataType::Boolean,
+            DataType::Int8,
+            DataType::UInt8,
+            DataType::Int16,
+            DataType::UInt16,
+            DataType::Int32,
+            DataType::UInt32,
+            DataType::Int64,
+            DataType::UInt64,
+            DataType::Float16,
+            DataType::Float32,
+            DataType::Float64,
+            DataType::Utf8,
+            DataType::LargeUtf8,
+            DataType::Binary,
+            DataType::LargeBinary,
+            DataType::Date32,
+            DataType::Date64,
+            DataType::FixedSizeBinary(16),
+            // Decimal types with positive and negative scales
+            DataType::Decimal32(10, -2),
+            DataType::Decimal32(9, 3),
+            DataType::Decimal64(18, -5),
+            DataType::Decimal64(10, 4),
+            DataType::Decimal128(9, -2),
+            DataType::Decimal128(38, 10),
+            DataType::Decimal256(38, 10),
+            DataType::Decimal256(76, -10),
+            // Timestamp and Duration
+            DataType::Timestamp(arrow::datatypes::TimeUnit::Microsecond, None),
+            DataType::Duration(arrow::datatypes::TimeUnit::Microsecond),
+            // Nested
+            DataType::List(Arc::new(Field::new("item", DataType::Int32, true))),
+            DataType::LargeList(Arc::new(Field::new("item", DataType::Utf8, true))),
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, false)), 128),
+            DataType::Struct(
+                vec![
+                    Field::new("a", DataType::Int64, false),
+                    Field::new("b", DataType::Utf8, true),
+                ]
+                .into(),
+            ),
+            // Map
+            DataType::Map(
+                Arc::new(Field::new(
+                    "entries",
+                    DataType::Struct(
+                        vec![
+                            Field::new("keys", DataType::Utf8, false),
+                            Field::new("values", DataType::Int32, true),
+                        ]
+                        .into(),
+                    ),
+                    false,
+                )),
+                false,
+            ),
+        ];
+
+        for dt in &cases {
+            let json = arrow_type_to_json(dt)
+                .unwrap_or_else(|e| panic!("arrow_type_to_json failed for {:?}: {}", dt, e));
+            let back = convert_json_arrow_type(&json)
+                .unwrap_or_else(|e| panic!("convert_json_arrow_type failed for {:?}: {}", dt, e));
+            assert_eq!(&back, dt, "Roundtrip mismatch for {:?}: got {:?}", dt, back);
+        }
+    }
+
+    #[test]
+    fn test_decimal_negative_scale_roundtrip() {
+        // Explicitly test the cases requested by reviewer
+        let cases = vec![
+            DataType::Decimal32(10, -2),
+            DataType::Decimal128(9, -2),
+            DataType::Decimal256(38, 10),
+        ];
+        for dt in &cases {
+            let json = arrow_type_to_json(dt).unwrap();
+            let back = convert_json_arrow_type(&json).unwrap();
+            assert_eq!(&back, dt, "Decimal roundtrip failed for {:?}", dt);
+        }
+    }
+
+    #[test]
+    fn test_schema_with_metadata_roundtrip() {
+        let mut metadata = HashMap::new();
+        metadata.insert("key1".to_string(), "value1".to_string());
+        metadata.insert("key2".to_string(), "value2".to_string());
+
+        let arrow_schema = ArrowSchema::new_with_metadata(
+            vec![
+                Field::new("id", DataType::Int32, false),
+                Field::new("name", DataType::Utf8, true),
+            ],
+            metadata.clone(),
+        );
+
+        let json_schema = arrow_schema_to_json(&arrow_schema).unwrap();
+        assert_eq!(json_schema.metadata.as_ref().unwrap(), &metadata);
+
+        let roundtrip = convert_json_arrow_schema(&json_schema).unwrap();
+        assert_eq!(roundtrip.metadata(), &metadata);
+    }
+
+    #[test]
+    fn test_dictionary_type_unwraps_to_value_type() {
+        let dict_type = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
+        let json = arrow_type_to_json(&dict_type).unwrap();
+        assert_eq!(json.r#type, "utf8");
+    }
+
+    #[test]
+    fn test_map_keys_sorted_unsupported() {
+        let map_type = DataType::Map(
+            Arc::new(Field::new(
+                "entries",
+                DataType::Struct(
+                    vec![
+                        Field::new("keys", DataType::Utf8, false),
+                        Field::new("values", DataType::Int32, true),
+                    ]
+                    .into(),
+                ),
+                false,
+            )),
+            true, // keys_sorted = true
+        );
+        let result = arrow_type_to_json(&map_type);
+        assert!(result.is_err());
+        assert!(result.unwrap_err().to_string().contains("keys_sorted=true"));
+    }
+
+    #[test]
+    fn test_unsupported_types_error() {
+        // RunEndEncoded
+        let ree = DataType::RunEndEncoded(
+            Arc::new(Field::new("run_ends", DataType::Int32, false)),
+            Arc::new(Field::new("values", DataType::Utf8, true)),
+        );
+        assert!(arrow_type_to_json(&ree).is_err());
+
+        // ListView
+        let lv = DataType::ListView(Arc::new(Field::new("item", DataType::Int32, true)));
+        assert!(arrow_type_to_json(&lv).is_err());
+
+        // LargeListView
+        let llv = DataType::LargeListView(Arc::new(Field::new("item", DataType::Int32, true)));
+        assert!(arrow_type_to_json(&llv).is_err());
+
+        // Utf8View / BinaryView
+        assert!(arrow_type_to_json(&DataType::Utf8View).is_err());
+        assert!(arrow_type_to_json(&DataType::BinaryView).is_err());
+    }
+
+    #[test]
+    fn test_large_list_roundtrip() {
+        let inner_field = Field::new("item", DataType::Float64, true);
+        let large_list = DataType::LargeList(Arc::new(inner_field));
+
+        let json = arrow_type_to_json(&large_list).unwrap();
+        assert_eq!(json.r#type, "large_list");
+
+        let back = convert_json_arrow_type(&json).unwrap();
+        assert_eq!(back, large_list);
+    }
+
+    #[test]
+    fn test_field_with_metadata_roundtrip() {
+        let mut field_meta = HashMap::new();
+        field_meta.insert("custom_key".to_string(), "custom_val".to_string());
+
+        let field = Field::new("col", DataType::Int64, false).with_metadata(field_meta.clone());
+        let schema = ArrowSchema::new(vec![field]);
+
+        let json_schema = arrow_schema_to_json(&schema).unwrap();
+        let roundtrip = convert_json_arrow_schema(&json_schema).unwrap();
+        assert_eq!(roundtrip.field(0).metadata(), &field_meta);
+    }
+
+    #[test]
+    fn test_nested_list_with_field_metadata() {
+        let mut meta = HashMap::new();
+        meta.insert("encoding".to_string(), "delta".to_string());
+
+        let inner = Field::new("item", DataType::Int32, true).with_metadata(meta.clone());
+        let list_type = DataType::List(Arc::new(inner));
+
+        let json = arrow_type_to_json(&list_type).unwrap();
+        let fields = json.fields.as_ref().unwrap();
+        assert_eq!(fields[0].metadata.as_ref().unwrap(), &meta);
+    }
 }
diff --git a/rust/lance-table/Cargo.toml b/rust/lance-table/Cargo.toml
index 961e11d45c3..c9cfc828f70 100644
--- a/rust/lance-table/Cargo.toml
+++ b/rust/lance-table/Cargo.toml
@@ -23,7 +23,7 @@ arrow-ipc.workspace = true
 arrow-schema.workspace = true
 async-trait.workspace = true
 aws-credential-types = { workspace = true, optional = true }
-aws-sdk-dynamodb = { workspace = true, optional = true }
+aws-sdk-dynamodb = { workspace = true, optional = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
 byteorder.workspace = true
 bytes.workspace = true
 chrono.workspace = true
@@ -72,5 +72,9 @@ features = ["protoc"]
 name = "row_id_index"
 harness = false
 
+[[bench]]
+name = "manifest_intern"
+harness = false
+
 [lints]
 workspace = true
diff --git a/rust/lance-table/benches/manifest_intern.rs b/rust/lance-table/benches/manifest_intern.rs
new file mode 100644
index 00000000000..09ba9264b21
--- /dev/null
+++ b/rust/lance-table/benches/manifest_intern.rs
@@ -0,0 +1,261 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+// Benchmarks use eprintln! to report memory stats alongside criterion output.
+#![allow(clippy::print_stderr)]
+
+//! Benchmark for manifest fragment interning.
+//!
+//! Measures memory savings and deserialization throughput when interning
+//! `DataFile.fields`, `DataFile.column_indices`, and
+//! `RowDatasetVersionMeta::Inline` bytes across many fragments.
+
+use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
+use deepsize::DeepSizeOf;
+use prost::Message;
+
+use lance_table::format::pb;
+use lance_table::format::{DataFileFieldInterner, Fragment};
+
+fn num_fragments() -> u64 {
+    std::env::var("BENCH_NUM_FRAGMENTS")
+        .map(|s| s.parse().unwrap())
+        .unwrap_or(100_000)
+}
+
+/// Build a vector of protobuf DataFragment messages that simulate a
+/// homogeneous, post-compaction table: every fragment has the same field
+/// list, column indices, and version metadata bytes.
+fn make_uniform_pb_fragments(n: u64, num_fields: usize) -> Vec<pb::DataFragment> {
+    let fields: Vec<i32> = (0..num_fields as i32).collect();
+    let column_indices: Vec<i32> = (0..num_fields as i32).collect();
+
+    // Simulate version metadata: a small protobuf-encoded payload
+    // (identical across all fragments post-compaction)
+    let version_bytes: Vec<u8> = {
+        let seq = pb::RowDatasetVersionSequence {
+            runs: vec![pb::RowDatasetVersionRun {
+                span: Some(pb::U64Segment {
+                    segment: Some(pb::u64_segment::Segment::Range(pb::u64_segment::Range {
+                        start: 0,
+                        end: 1000,
+                    })),
+                }),
+                version: 42,
+            }],
+        };
+        seq.encode_to_vec()
+    };
+
+    (0..n)
+        .map(|i| pb::DataFragment {
+            id: i,
+            files: vec![pb::DataFile {
+                path: format!("data/{i}.lance"),
+                fields: fields.clone(),
+                column_indices: column_indices.clone(),
+                file_major_version: 2,
+                file_minor_version: 0,
+                file_size_bytes: 0,
+                base_id: None,
+            }],
+            deletion_file: None,
+            row_id_sequence: None,
+            physical_rows: 1000,
+            last_updated_at_version_sequence: Some(
+                pb::data_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions(
+                    version_bytes.clone(),
+                ),
+            ),
+            created_at_version_sequence: Some(
+                pb::data_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions(
+                    version_bytes.clone(),
+                ),
+            ),
+        })
+        .collect()
+}
+
+/// Deserialize protobuf fragments WITHOUT interning (baseline).
+fn deserialize_without_interning(protos: &[pb::DataFragment]) -> Vec<Fragment> {
+    protos
+        .iter()
+        .map(|p| Fragment::try_from(p.clone()).unwrap())
+        .collect()
+}
+
+/// Deserialize protobuf fragments WITH interning.
+fn deserialize_with_interning(protos: &[pb::DataFragment]) -> Vec<Fragment> {
+    let mut interner = DataFileFieldInterner::default();
+    protos
+        .iter()
+        .map(|p| interner.intern_fragment(p.clone()).unwrap())
+        .collect()
+}
+
+/// Build fragments where each group shares the same version metadata,
+/// simulating many small appends without compaction.
+fn make_diverse_pb_fragments(
+    n: u64,
+    num_fields: usize,
+    unique_versions: u64,
+) -> Vec<pb::DataFragment> {
+    let fields: Vec<i32> = (0..num_fields as i32).collect();
+    let column_indices: Vec<i32> = (0..num_fields as i32).collect();
+    let group_size = n / unique_versions;
+
+    let version_payloads: Vec<Vec<u8>> = (0..unique_versions)
+        .map(|v| {
+            let seq = pb::RowDatasetVersionSequence {
+                runs: vec![pb::RowDatasetVersionRun {
+                    span: Some(pb::U64Segment {
+                        segment: Some(pb::u64_segment::Segment::Range(pb::u64_segment::Range {
+                            start: 0,
+                            end: 1000,
+                        })),
+                    }),
+                    version: v,
+                }],
+            };
+            seq.encode_to_vec()
+        })
+        .collect();
+
+    (0..n)
+        .map(|i| {
+            let version_idx = (i / group_size).min(unique_versions - 1) as usize;
+            pb::DataFragment {
+                id: i,
+                files: vec![pb::DataFile {
+                    path: format!("data/{i}.lance"),
+                    fields: fields.clone(),
+                    column_indices: column_indices.clone(),
+                    file_major_version: 2,
+                    file_minor_version: 0,
+                    file_size_bytes: 0,
+                    base_id: None,
+                }],
+                deletion_file: None,
+                row_id_sequence: None,
+                physical_rows: 1000,
+                last_updated_at_version_sequence: Some(
+                    pb::data_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions(
+                        version_payloads[version_idx].clone(),
+                    ),
+                ),
+                created_at_version_sequence: Some(
+                    pb::data_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions(
+                        version_payloads[version_idx].clone(),
+                    ),
+                ),
+            }
+        })
+        .collect()
+}
+
+fn bench_deserialization(c: &mut Criterion) {
+    let mut group = c.benchmark_group("manifest_intern");
+    let n = num_fragments();
+
+    for num_fields in [10, 50] {
+        let protos = make_uniform_pb_fragments(n, num_fields);
+
+        group.bench_with_input(
+            BenchmarkId::new("deserialize_no_intern", num_fields),
+            &num_fields,
+            |b, _| {
+                b.iter(|| deserialize_without_interning(&protos));
+            },
+        );
+
+        group.bench_with_input(
+            BenchmarkId::new("deserialize_with_intern", num_fields),
+            &num_fields,
+            |b, _| {
+                b.iter(|| deserialize_with_interning(&protos));
+            },
+        );
+    }
+
+    // Benchmark with many unique version payloads
+    for unique_versions in [10, 100, 500] {
+        let protos = make_diverse_pb_fragments(n, 10, unique_versions);
+
+        group.bench_with_input(
+            BenchmarkId::new("deserialize_no_intern_diverse", unique_versions),
+            &unique_versions,
+            |b, _| {
+                b.iter(|| deserialize_without_interning(&protos));
+            },
+        );
+
+        group.bench_with_input(
+            BenchmarkId::new("deserialize_with_intern_diverse", unique_versions),
+            &unique_versions,
+            |b, _| {
+                b.iter(|| deserialize_with_interning(&protos));
+            },
+        );
+    }
+
+    group.finish();
+}
+
+fn bench_memory(c: &mut Criterion) {
+    let mut group = c.benchmark_group("manifest_memory");
+    let n = num_fragments();
+
+    for num_fields in [10, 50] {
+        let protos = make_uniform_pb_fragments(n, num_fields);
+
+        let no_intern = deserialize_without_interning(&protos);
+        let with_intern = deserialize_with_interning(&protos);
+
+        let size_no_intern = no_intern.deep_size_of();
+        let size_with_intern = with_intern.deep_size_of();
+
+        eprintln!(
+            "\n[{} fragments, {} fields] Memory without interning: {:.2} MB",
+            n,
+            num_fields,
+            size_no_intern as f64 / 1_048_576.0
+        );
+        eprintln!(
+            "[{} fragments, {} fields] Memory with interning:    {:.2} MB",
+            n,
+            num_fields,
+            size_with_intern as f64 / 1_048_576.0
+        );
+        eprintln!(
+            "[{} fragments, {} fields] Savings:                  {:.2} MB ({:.1}%)",
+            n,
+            num_fields,
+            (size_no_intern - size_with_intern) as f64 / 1_048_576.0,
+            (1.0 - size_with_intern as f64 / size_no_intern as f64) * 100.0
+        );
+
+        // Benchmark deep_size_of measurement itself (sanity check)
+        group.bench_with_input(
+            BenchmarkId::new("deep_size_of_interned", num_fields),
+            &num_fields,
+            |b, _| {
+                b.iter(|| with_intern.deep_size_of());
+            },
+        );
+
+        drop(no_intern);
+        drop(with_intern);
+    }
+
+    group.finish();
+}
+
+#[cfg(target_os = "linux")]
+criterion_group!(
+    name = benches;
+    config = Criterion::default().with_profiler(pprof::criterion::PProfProfiler::new(100, pprof::criterion::Output::Flamegraph(None)));
+    targets = bench_deserialization, bench_memory
+);
+#[cfg(not(target_os = "linux"))]
+criterion_group!(benches, bench_deserialization, bench_memory);
+criterion_main!(benches);
diff --git a/rust/lance-table/src/format.rs b/rust/lance-table/src/format.rs
index db065199532..842c76f1e58 100644
--- a/rust/lance-table/src/format.rs
+++ b/rust/lance-table/src/format.rs
@@ -13,7 +13,7 @@ pub use crate::rowids::version::{
     RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence,
 };
 pub use fragment::*;
-pub use index::IndexMetadata;
+pub use index::{IndexFile, IndexMetadata, index_metadata_codec, list_index_files_with_sizes};
 
 pub use manifest::{
     BasePath, DETACHED_VERSION_MASK, DataStorageFormat, Manifest, SelfDescribingFileReader,
diff --git a/rust/lance-table/src/format/fragment.rs b/rust/lance-table/src/format/fragment.rs
index 01c5b535f8e..e1643b0712e 100644
--- a/rust/lance-table/src/format/fragment.rs
+++ b/rust/lance-table/src/format/fragment.rs
@@ -1,7 +1,9 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use std::collections::HashMap;
 use std::num::NonZero;
+use std::sync::Arc;
 
 use deepsize::DeepSizeOf;
 use lance_core::Error;
@@ -9,7 +11,7 @@ use lance_file::format::{MAJOR_VERSION, MINOR_VERSION};
 use lance_file::version::LanceFileVersion;
 use lance_io::utils::CachedFileSize;
 use object_store::path::Path;
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
 
 use crate::format::pb;
 
@@ -22,12 +24,16 @@ use lance_core::error::Result;
 /// Lance Data File
 ///
 /// A data file is one piece of file storing data.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)]
 pub struct DataFile {
     /// Relative path of the data file to dataset root.
     pub path: String,
     /// The ids of fields in this file.
-    pub fields: Vec<i32>,
+    ///
+    /// When identical across many fragments (common case), multiple `DataFile`
+    /// instances share a single heap allocation via `Arc`, significantly
+    /// reducing manifest memory for large tables.
+    pub fields: Arc<[i32]>,
     /// The offsets of the fields listed in `fields`, empty in v1 files
     ///
     /// Note that -1 is a possibility and it indices that the field has
@@ -37,13 +43,10 @@ pub struct DataFile {
     /// `column_indices`; such columns are ignored by field-id–based projection.
     /// For example, some fields, such as blob fields, occupy multiple
     /// columns in the file but only have a single field id.
-    #[serde(default)]
-    pub column_indices: Vec<i32>,
+    pub column_indices: Arc<[i32]>,
     /// The major version of the file format used to write this file.
-    #[serde(default)]
     pub file_major_version: u32,
     /// The minor version of the file format used to write this file.
-    #[serde(default)]
     pub file_minor_version: u32,
 
     /// The size of the file in bytes, if known.
@@ -53,6 +56,52 @@ pub struct DataFile {
     pub base_id: Option<u32>,
 }
 
+// Custom Serialize: convert Arc<[i32]> to slice for transparent JSON output
+impl Serialize for DataFile {
+    fn serialize<S: Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
+        use serde::ser::SerializeStruct;
+        let mut s = serializer.serialize_struct("DataFile", 7)?;
+        s.serialize_field("path", &self.path)?;
+        s.serialize_field("fields", self.fields.as_ref())?;
+        s.serialize_field("column_indices", self.column_indices.as_ref())?;
+        s.serialize_field("file_major_version", &self.file_major_version)?;
+        s.serialize_field("file_minor_version", &self.file_minor_version)?;
+        s.serialize_field("file_size_bytes", &self.file_size_bytes)?;
+        s.serialize_field("base_id", &self.base_id)?;
+        s.end()
+    }
+}
+
+// Custom Deserialize: read Vec<i32> and convert to Arc<[i32]>
+impl<'de> Deserialize<'de> for DataFile {
+    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
+        #[derive(Deserialize)]
+        struct DataFileHelper {
+            path: String,
+            fields: Vec<i32>,
+            #[serde(default)]
+            column_indices: Vec<i32>,
+            #[serde(default)]
+            file_major_version: u32,
+            #[serde(default)]
+            file_minor_version: u32,
+            file_size_bytes: CachedFileSize,
+            base_id: Option<u32>,
+        }
+
+        let helper = DataFileHelper::deserialize(deserializer)?;
+        Ok(Self {
+            path: helper.path,
+            fields: Arc::from(helper.fields),
+            column_indices: Arc::from(helper.column_indices),
+            file_major_version: helper.file_major_version,
+            file_minor_version: helper.file_minor_version,
+            file_size_bytes: helper.file_size_bytes,
+            base_id: helper.base_id,
+        })
+    }
+}
+
 impl DataFile {
     pub fn new(
         path: impl Into<String>,
@@ -65,8 +114,8 @@ impl DataFile {
     ) -> Self {
         Self {
             path: path.into(),
-            fields,
-            column_indices,
+            fields: Arc::from(fields),
+            column_indices: Arc::from(column_indices),
             file_major_version,
             file_minor_version,
             file_size_bytes: file_size_bytes.into(),
@@ -82,8 +131,8 @@ impl DataFile {
     ) -> Self {
         Self {
             path: path.into(),
-            fields: vec![],
-            column_indices: vec![],
+            fields: Arc::from([]),
+            column_indices: Arc::from([]),
             file_major_version,
             file_minor_version,
             file_size_bytes: Default::default(),
@@ -158,8 +207,8 @@ impl From<&DataFile> for pb::DataFile {
     fn from(df: &DataFile) -> Self {
         Self {
             path: df.path.clone(),
-            fields: df.fields.clone(),
-            column_indices: df.column_indices.clone(),
+            fields: df.fields.to_vec(),
+            column_indices: df.column_indices.to_vec(),
             file_major_version: df.file_major_version,
             file_minor_version: df.file_minor_version,
             file_size_bytes: df.file_size_bytes.get().map_or(0, |v| v.get()),
@@ -174,8 +223,8 @@ impl TryFrom<pb::DataFile> for DataFile {
     fn try_from(proto: pb::DataFile) -> Result<Self> {
         Ok(Self {
             path: proto.path,
-            fields: proto.fields,
-            column_indices: proto.column_indices,
+            fields: Arc::from(proto.fields),
+            column_indices: Arc::from(proto.column_indices),
             file_major_version: proto.file_major_version,
             file_minor_version: proto.file_minor_version,
             file_size_bytes: CachedFileSize::new(proto.file_size_bytes),
@@ -184,6 +233,157 @@ impl TryFrom<pb::DataFile> for DataFile {
     }
 }
 
+/// Interns repeated data so that fragments with identical content share a
+/// single heap allocation via `Arc`.
+///
+/// At 20M fragments the deduplication typically saves multiple GB of heap
+/// because every fragment in a homogeneous table carries the same field list,
+/// and post-compaction fragments share identical version metadata bytes.
+///
+/// Uses a `Vec`-based linear scan when the cache is small (<=16 entries)
+/// and upgrades to `HashMap` for larger caches. In the common homogeneous
+/// case (1-3 unique values), linear scan avoids per-fragment hashing overhead.
+#[derive(Default)]
+pub struct DataFileFieldInterner {
+    fields: InternCache<i32>,
+    column_indices: InternCache<i32>,
+    inline_bytes: InternCache<u8>,
+}
+
+/// A cache that uses linear scan for small sizes and HashMap for large.
+/// The threshold is chosen so that scan + compare is cheaper than hash for
+/// typical payload sizes (20-200 bytes).
+enum InternCache<T: Eq + std::hash::Hash + Clone> {
+    Small(Vec<Arc<[T]>>),
+    Large(HashMap<Arc<[T]>, ()>),
+}
+
+const INTERN_CACHE_UPGRADE_THRESHOLD: usize = 16;
+
+impl<T: Eq + std::hash::Hash + Clone> Default for InternCache<T> {
+    fn default() -> Self {
+        Self::Small(Vec::new())
+    }
+}
+
+impl<T: Eq + std::hash::Hash + Clone> InternCache<T> {
+    fn intern(&mut self, v: Vec<T>) -> Arc<[T]> {
+        match self {
+            Self::Small(entries) => {
+                for existing in entries.iter() {
+                    if existing.as_ref() == v.as_slice() {
+                        return existing.clone();
+                    }
+                }
+                let arc: Arc<[T]> = Arc::from(v);
+                entries.push(arc.clone());
+                if entries.len() > INTERN_CACHE_UPGRADE_THRESHOLD {
+                    let mut map = HashMap::with_capacity(entries.len());
+                    for e in entries.drain(..) {
+                        map.insert(e, ());
+                    }
+                    *self = Self::Large(map);
+                }
+                arc
+            }
+            Self::Large(map) => {
+                if let Some((existing, _)) = map.get_key_value(v.as_slice()) {
+                    existing.clone()
+                } else {
+                    let arc: Arc<[T]> = Arc::from(v);
+                    map.insert(arc.clone(), ());
+                    arc
+                }
+            }
+        }
+    }
+}
+
+impl DataFileFieldInterner {
+    /// Intern a `RowDatasetVersionMeta`, deduplicating inline byte payloads.
+    /// Accepts the protobuf oneof value directly to avoid an intermediate
+    /// `Arc<[u8]>` allocation that would need to be `.to_vec()`'d for the key lookup.
+    fn intern_last_updated_version_meta(
+        cache: &mut InternCache<u8>,
+        pb: pb::data_fragment::LastUpdatedAtVersionSequence,
+    ) -> Result<RowDatasetVersionMeta> {
+        match pb {
+            pb::data_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions(data) => {
+                Ok(RowDatasetVersionMeta::Inline(cache.intern(data)))
+            }
+            pb::data_fragment::LastUpdatedAtVersionSequence::ExternalLastUpdatedAtVersions(
+                file,
+            ) => Ok(RowDatasetVersionMeta::External(ExternalFile {
+                path: file.path,
+                offset: file.offset,
+                size: file.size,
+            })),
+        }
+    }
+
+    /// Intern a `RowDatasetVersionMeta`, deduplicating inline byte payloads.
+    fn intern_created_version_meta(
+        cache: &mut InternCache<u8>,
+        pb: pb::data_fragment::CreatedAtVersionSequence,
+    ) -> Result<RowDatasetVersionMeta> {
+        match pb {
+            pb::data_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions(data) => {
+                Ok(RowDatasetVersionMeta::Inline(cache.intern(data)))
+            }
+            pb::data_fragment::CreatedAtVersionSequence::ExternalCreatedAtVersions(file) => {
+                Ok(RowDatasetVersionMeta::External(ExternalFile {
+                    path: file.path,
+                    offset: file.offset,
+                    size: file.size,
+                }))
+            }
+        }
+    }
+
+    /// Convert a protobuf `DataFile`, interning `fields` and `column_indices`.
+    pub fn intern_data_file(&mut self, proto: pb::DataFile) -> Result<DataFile> {
+        Ok(DataFile {
+            path: proto.path,
+            fields: self.fields.intern(proto.fields),
+            column_indices: self.column_indices.intern(proto.column_indices),
+            file_major_version: proto.file_major_version,
+            file_minor_version: proto.file_minor_version,
+            file_size_bytes: CachedFileSize::new(proto.file_size_bytes),
+            base_id: proto.base_id,
+        })
+    }
+
+    /// Convert a protobuf `DataFragment`, interning fields and version metadata.
+    pub fn intern_fragment(&mut self, p: pb::DataFragment) -> Result<Fragment> {
+        let physical_rows = if p.physical_rows > 0 {
+            Some(p.physical_rows as usize)
+        } else {
+            None
+        };
+        let last_updated_at_version_meta = p
+            .last_updated_at_version_sequence
+            .map(|pb| Self::intern_last_updated_version_meta(&mut self.inline_bytes, pb))
+            .transpose()?;
+        let created_at_version_meta = p
+            .created_at_version_sequence
+            .map(|pb| Self::intern_created_version_meta(&mut self.inline_bytes, pb))
+            .transpose()?;
+        Ok(Fragment {
+            id: p.id,
+            files: p
+                .files
+                .into_iter()
+                .map(|f| self.intern_data_file(f))
+                .collect::<Result<_>>()?,
+            deletion_file: p.deletion_file.map(DeletionFile::try_from).transpose()?,
+            row_id_meta: p.row_id_sequence.map(RowIdMeta::try_from).transpose()?,
+            physical_rows,
+            last_updated_at_version_meta,
+            created_at_version_meta,
+        })
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
 #[serde(rename_all = "lowercase")]
 pub enum DeletionFileType {
@@ -624,9 +824,9 @@ mod tests {
     fn data_file_validate_allows_extra_columns() {
         let data_file = DataFile {
             path: "foo.lance".to_string(),
-            fields: vec![1, 2],
+            fields: Arc::from([1, 2]),
             // One extra column without a field id mapping
-            column_indices: vec![0, 1, 2],
+            column_indices: Arc::from([0, 1, 2]),
             file_major_version: MAJOR_VERSION as u32,
             file_minor_version: MINOR_VERSION as u32,
             file_size_bytes: Default::default(),
diff --git a/rust/lance-table/src/format/index.rs b/rust/lance-table/src/format/index.rs
index 8db98bcdeba..945d8364123 100644
--- a/rust/lance-table/src/format/index.rs
+++ b/rust/lance-table/src/format/index.rs
@@ -3,16 +3,29 @@
 
 //! Metadata for index
 
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use chrono::{DateTime, Utc};
 use deepsize::DeepSizeOf;
+use futures::StreamExt;
+use lance_io::object_store::ObjectStore;
+use object_store::path::Path;
 use roaring::RoaringBitmap;
 use uuid::Uuid;
 
 use super::pb;
 use lance_core::{Error, Result};
 
+/// Metadata about a single file within an index segment.
+#[derive(Debug, Clone, PartialEq, DeepSizeOf)]
+pub struct IndexFile {
+    /// Path relative to the index directory (e.g., "index.idx", "auxiliary.idx")
+    pub path: String,
+    /// Size of the file in bytes
+    pub size_bytes: u64,
+}
+
 /// Index metadata
 #[derive(Debug, Clone, PartialEq)]
 pub struct IndexMetadata {
@@ -56,6 +69,13 @@ pub struct IndexMetadata {
     /// The base path index of the index files. Used when the index is imported or referred from another dataset.
     /// Lance uses it as key of the base_paths field in Manifest to determine the actual base path of the index files.
     pub base_id: Option<u32>,
+
+    /// List of files and their sizes for this index segment.
+    /// This enables skipping HEAD calls when opening indices and provides
+    /// visibility into index storage size via describe_indices().
+    /// This is None if the file sizes are unknown. This happens for indices created
+    /// before this field was added.
+    pub files: Option<Vec<IndexFile>>,
 }
 
 impl IndexMetadata {
@@ -66,6 +86,38 @@ impl IndexMetadata {
         let fragment_bitmap = self.fragment_bitmap.as_ref()?;
         Some(fragment_bitmap & existing_fragments)
     }
+
+    /// Returns a map of relative file paths to their sizes.
+    /// Returns an empty map if file information is not available.
+    pub fn file_size_map(&self) -> HashMap<String, u64> {
+        self.files
+            .as_ref()
+            .map(|files| {
+                files
+                    .iter()
+                    .map(|f| (f.path.clone(), f.size_bytes))
+                    .collect()
+            })
+            .unwrap_or_default()
+    }
+
+    /// Returns the total size of all files in this index segment in bytes.
+    /// Returns None if file information is not available.
+    pub fn total_size_bytes(&self) -> Option<u64> {
+        self.files
+            .as_ref()
+            .map(|files| files.iter().map(|f| f.size_bytes).sum())
+    }
+
+    /// Returns the set of fragments which are part of the fragment bitmap
+    /// but no longer in the dataset.
+    pub fn deleted_fragment_bitmap(
+        &self,
+        existing_fragments: &RoaringBitmap,
+    ) -> Option<RoaringBitmap> {
+        let fragment_bitmap = self.fragment_bitmap.as_ref()?;
+        Some(fragment_bitmap - existing_fragments)
+    }
 }
 
 impl DeepSizeOf for IndexMetadata {
@@ -79,6 +131,7 @@ impl DeepSizeOf for IndexMetadata {
                 .as_ref()
                 .map(|fragment_bitmap| fragment_bitmap.serialized_size())
                 .unwrap_or(0)
+            + self.files.deep_size_of_children(context)
     }
 }
 
@@ -94,6 +147,21 @@ impl TryFrom<pb::IndexMetadata> for IndexMetadata {
             )?)
         };
 
+        let files = if proto.files.is_empty() {
+            None
+        } else {
+            Some(
+                proto
+                    .files
+                    .into_iter()
+                    .map(|f| IndexFile {
+                        path: f.path,
+                        size_bytes: f.size_bytes,
+                    })
+                    .collect(),
+            )
+        };
+
         Ok(Self {
             uuid: proto.uuid.as_ref().map(Uuid::try_from).ok_or_else(|| {
                 Error::invalid_input("uuid field does not exist in Index metadata".to_string())
@@ -109,6 +177,7 @@ impl TryFrom<pb::IndexMetadata> for IndexMetadata {
                     .expect("Invalid timestamp in index metadata")
             }),
             base_id: proto.base_id,
+            files,
         })
     }
 }
@@ -125,6 +194,20 @@ impl From<&IndexMetadata> for pb::IndexMetadata {
             fragment_bitmap.clear();
         }
 
+        let files = idx
+            .files
+            .as_ref()
+            .map(|files| {
+                files
+                    .iter()
+                    .map(|f| pb::IndexFile {
+                        path: f.path.clone(),
+                        size_bytes: f.size_bytes,
+                    })
+                    .collect()
+            })
+            .unwrap_or_default();
+
         Self {
             uuid: Some((&idx.uuid).into()),
             name: idx.name.clone(),
@@ -138,6 +221,148 @@ impl From<&IndexMetadata> for pb::IndexMetadata {
             index_version: Some(idx.index_version),
             created_at: idx.created_at.map(|dt| dt.timestamp_millis() as u64),
             base_id: idx.base_id,
+            files,
+        }
+    }
+}
+
+/// Returns a [`CacheCodec`](lance_core::cache::CacheCodec) for `Vec<IndexMetadata>`.
+///
+/// Uses `pb::IndexSection` (which wraps `repeated IndexMetadata`) as the wire
+/// format, reusing the existing `TryFrom`/`From` conversions.
+///
+/// Uses [`CacheCodec::new`](lance_core::cache::CacheCodec::new) because the
+/// orphan rule prevents `impl CacheCodecImpl for Vec<IndexMetadata>`.
+type ArcAny = Arc<dyn std::any::Any + Send + Sync>;
+
+fn serialize_index_metadata(
+    any: &ArcAny,
+    writer: &mut dyn std::io::Write,
+) -> lance_core::Result<()> {
+    use prost::Message;
+    let vec = any
+        .downcast_ref::<Vec<IndexMetadata>>()
+        .expect("index_metadata_codec: wrong type (this is a bug in the cache layer)");
+    let section = pb::IndexSection {
+        indices: vec.iter().map(pb::IndexMetadata::from).collect(),
+    };
+    writer.write_all(&section.encode_to_vec())?;
+    Ok(())
+}
+
+fn deserialize_index_metadata(data: &bytes::Bytes) -> lance_core::Result<ArcAny> {
+    use prost::Message;
+    let section = pb::IndexSection::decode(data.as_ref())?;
+    let indices: Vec<IndexMetadata> = section
+        .indices
+        .into_iter()
+        .map(IndexMetadata::try_from)
+        .collect::<lance_core::Result<_>>()?;
+    Ok(Arc::new(indices))
+}
+
+pub fn index_metadata_codec() -> lance_core::cache::CacheCodec {
+    lance_core::cache::CacheCodec::new(serialize_index_metadata, deserialize_index_metadata)
+}
+
+/// List all files in an index directory with their sizes.
+///
+/// Returns a list of `IndexFile` structs containing relative paths and sizes.
+/// This is used to capture file metadata after index creation/modification.
+pub async fn list_index_files_with_sizes(
+    object_store: &ObjectStore,
+    index_dir: &Path,
+) -> Result<Vec<IndexFile>> {
+    let mut files = Vec::new();
+    let mut stream = object_store.read_dir_all(index_dir, None);
+    while let Some(meta) = stream.next().await {
+        let meta = meta?;
+        // Get relative path by stripping the index_dir prefix
+        let relative_path = meta
+            .location
+            .as_ref()
+            .strip_prefix(index_dir.as_ref())
+            .map(|s| s.trim_start_matches('/').to_string())
+            .unwrap_or_else(|| meta.location.filename().unwrap_or("").to_string());
+        files.push(IndexFile {
+            path: relative_path,
+            size_bytes: meta.size,
+        });
+    }
+    Ok(files)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::HashMap;
+
+    /// Demonstrates the pattern a disk-backed cache backend would use:
+    /// serialize entries to bytes, store in a key-value map, then
+    /// deserialize on retrieval.
+    #[test]
+    fn test_index_metadata_codec_roundtrip() {
+        let codec = index_metadata_codec();
+
+        let original = vec![
+            IndexMetadata {
+                uuid: Uuid::new_v4(),
+                name: "my_index".to_string(),
+                fields: vec![0, 1],
+                dataset_version: 42,
+                fragment_bitmap: Some(RoaringBitmap::from_iter([1, 2, 3])),
+                index_details: None,
+                index_version: 1,
+                created_at: None,
+                base_id: None,
+                files: Some(vec![IndexFile {
+                    path: "index.idx".to_string(),
+                    size_bytes: 1024,
+                }]),
+            },
+            IndexMetadata {
+                uuid: Uuid::new_v4(),
+                name: "second_index".to_string(),
+                fields: vec![2],
+                dataset_version: 43,
+                fragment_bitmap: None,
+                index_details: None,
+                index_version: 2,
+                created_at: None,
+                base_id: Some(7),
+                files: None,
+            },
+        ];
+
+        // Simulate a disk-backed store: HashMap<String, Vec<u8>>
+        let mut store: HashMap<String, Vec<u8>> = HashMap::new();
+
+        // Serialize into the store
+        let key = "dataset/v42/Vec<IndexMetadata>".to_string();
+        let mut buf = Vec::new();
+        let entry: Arc<dyn std::any::Any + Send + Sync> = Arc::new(original.clone());
+        codec.serialize(&entry, &mut buf).unwrap();
+        store.insert(key.clone(), buf);
+
+        // Deserialize from the store
+        let bytes = store.get(&key).unwrap();
+        let recovered = codec
+            .deserialize(&bytes::Bytes::copy_from_slice(bytes))
+            .unwrap();
+        let recovered = recovered
+            .downcast::<Vec<IndexMetadata>>()
+            .expect("downcast should succeed");
+
+        assert_eq!(original.len(), recovered.len());
+        for (orig, rec) in original.iter().zip(recovered.iter()) {
+            assert_eq!(orig.uuid, rec.uuid);
+            assert_eq!(orig.name, rec.name);
+            assert_eq!(orig.fields, rec.fields);
+            assert_eq!(orig.dataset_version, rec.dataset_version);
+            assert_eq!(orig.fragment_bitmap, rec.fragment_bitmap);
+            assert_eq!(orig.index_version, rec.index_version);
+            assert_eq!(orig.base_id, rec.base_id);
+            assert_eq!(orig.files, rec.files);
         }
     }
 }
diff --git a/rust/lance-table/src/format/manifest.rs b/rust/lance-table/src/format/manifest.rs
index 71de80c547f..d2b5f2d31c6 100644
--- a/rust/lance-table/src/format/manifest.rs
+++ b/rust/lance-table/src/format/manifest.rs
@@ -17,6 +17,7 @@ use std::sync::Arc;
 
 use super::Fragment;
 use crate::feature_flags::{FLAG_STABLE_ROW_IDS, has_deprecated_v2_feature_flag};
+use crate::format::fragment::DataFileFieldInterner;
 use crate::format::pb;
 use lance_core::cache::LanceCache;
 use lance_core::datatypes::Schema;
@@ -426,7 +427,7 @@ impl Manifest {
         let fragment_max_id = self
             .fragments
             .iter()
-            .flat_map(|f| f.files.iter().flat_map(|file| file.fields.as_slice()))
+            .flat_map(|f| f.files.iter().flat_map(|file| file.fields.iter()))
             .max()
             .copied();
         let fragment_max_id = fragment_max_id.unwrap_or(-1);
@@ -863,10 +864,11 @@ impl TryFrom<pb::Manifest> for Manifest {
             }),
             _ => None,
         };
+        let mut interner = DataFileFieldInterner::default();
         let fragments = Arc::new(
             p.fragments
                 .into_iter()
-                .map(Fragment::try_from)
+                .map(|f| interner.intern_fragment(f))
                 .collect::<Result<Vec<_>>>()?,
         );
         let fragment_offsets = compute_fragment_offsets(fragments.as_slice());
diff --git a/rust/lance-table/src/io/commit.rs b/rust/lance-table/src/io/commit.rs
index acc7eaaec14..079b4bb4da2 100644
--- a/rust/lance-table/src/io/commit.rs
+++ b/rust/lance-table/src/io/commit.rs
@@ -115,6 +115,19 @@ impl ManifestNamingScheme {
         }
     }
 
+    /// Parse a detached version from a filename like `d123456.manifest`.
+    ///
+    /// Returns the full version number with the detached mask bit set.
+    pub fn parse_detached_version(filename: &str) -> Option<u64> {
+        if !filename.starts_with(DETACHED_VERSION_PREFIX) {
+            return None;
+        }
+        let without_prefix = &filename[DETACHED_VERSION_PREFIX.len()..];
+        without_prefix
+            .split_once('.')
+            .and_then(|(version_str, _)| version_str.parse::<u64>().ok())
+    }
+
     pub fn detect_scheme(filename: &str) -> Option<Self> {
         if filename.starts_with(DETACHED_VERSION_PREFIX) {
             // Currently, detached versions must imply V2
@@ -261,9 +274,14 @@ async fn current_manifest_path(
     let manifest_files = object_store.list(Some(base.child(VERSIONS_DIR)));
 
     let mut valid_manifests = manifest_files.try_filter_map(|res| {
-        if let Some(scheme) = ManifestNamingScheme::detect_scheme(res.location.filename().unwrap())
-        {
-            future::ready(Ok(Some((scheme, res))))
+        let filename = res.location.filename().unwrap();
+        if let Some(scheme) = ManifestNamingScheme::detect_scheme(filename) {
+            // Only include if we can parse a version (skip detached versions)
+            if scheme.parse_version(filename).is_some() {
+                future::ready(Ok(Some((scheme, res))))
+            } else {
+                future::ready(Ok(None))
+            }
         } else {
             future::ready(Ok(None))
         }
@@ -428,6 +446,38 @@ fn list_manifests<'a>(
         .boxed()
 }
 
+/// Convert object metadata to ManifestLocation for detached manifests.
+fn detached_manifest_location_from_meta(
+    meta: object_store::ObjectMeta,
+) -> Option<ManifestLocation> {
+    let filename = meta.location.filename()?;
+    let version = ManifestNamingScheme::parse_detached_version(filename)?;
+    Some(ManifestLocation {
+        version,
+        path: meta.location,
+        size: Some(meta.size),
+        naming_scheme: ManifestNamingScheme::V2,
+        e_tag: meta.e_tag,
+    })
+}
+
+/// List all detached manifest files in the versions directory.
+pub fn list_detached_manifests<'a>(
+    base_path: &Path,
+    object_store: &'a dyn OSObjectStore,
+) -> impl Stream<Item = Result<ManifestLocation>> + 'a {
+    object_store
+        .read_dir_all(&base_path.child(VERSIONS_DIR), None)
+        .filter_map(|obj_meta| {
+            futures::future::ready(
+                obj_meta
+                    .map(detached_manifest_location_from_meta)
+                    .transpose(),
+            )
+        })
+        .boxed()
+}
+
 fn make_staging_manifest_path(base: &Path) -> Result<Path> {
     let id = uuid::Uuid::new_v4().to_string();
     Path::parse(format!("{base}-{id}")).map_err(|e| Error::io_source(Box::new(e)))
@@ -464,6 +514,17 @@ pub trait CommitHandler: Debug + Send + Sync {
         default_resolve_version(base_path, version, object_store).await
     }
 
+    /// List detached manifest locations.
+    ///
+    /// Returns a stream of detached manifest locations in arbitrary order.
+    fn list_detached_manifest_locations<'a>(
+        &self,
+        base_path: &Path,
+        object_store: &'a ObjectStore,
+    ) -> BoxStream<'a, Result<ManifestLocation>> {
+        list_detached_manifests(base_path, &object_store.inner).boxed()
+    }
+
     /// If `sorted_descending` is `true`, the stream will yield manifests in descending
     /// order of version. When the object store has a lexicographically
     /// ordered list and the naming scheme is V2, this will use an optimized
@@ -681,7 +742,9 @@ pub async fn commit_handler_from_url(
 
     match url.scheme() {
         "file" | "file-object-store" => Ok(local_handler),
-        "s3" | "gs" | "az" | "memory" | "oss" | "cos" => Ok(Arc::new(ConditionalPutCommitHandler)),
+        "s3" | "gs" | "az" | "abfss" | "memory" | "oss" | "cos" => {
+            Ok(Arc::new(ConditionalPutCommitHandler))
+        }
         #[cfg(not(feature = "dynamodb"))]
         "s3+ddb" => Err(Error::invalid_input_source(
             "`s3+ddb://` scheme requires `dynamodb` feature to be enabled".into(),
@@ -1216,4 +1279,77 @@ mod tests {
         assert_eq!(location.naming_scheme, naming_scheme);
         assert_eq!(location.path, naming_scheme.manifest_path(&base, 11));
     }
+
+    #[test]
+    fn test_parse_detached_version() {
+        // Valid detached version filenames
+        assert_eq!(
+            ManifestNamingScheme::parse_detached_version("d12345.manifest"),
+            Some(12345)
+        );
+        assert_eq!(
+            ManifestNamingScheme::parse_detached_version("d9223372036854775808.manifest"),
+            Some(9223372036854775808)
+        );
+
+        // Invalid: not starting with 'd' prefix
+        assert_eq!(
+            ManifestNamingScheme::parse_detached_version("12345.manifest"),
+            None
+        );
+
+        // Invalid: regular V2 manifest
+        assert_eq!(
+            ManifestNamingScheme::parse_detached_version("18446744073709551615.manifest"),
+            None
+        );
+
+        // Invalid: no extension
+        assert_eq!(ManifestNamingScheme::parse_detached_version("d12345"), None);
+    }
+
+    #[tokio::test]
+    async fn test_list_detached_manifests() {
+        use crate::format::DETACHED_VERSION_MASK;
+        use futures::TryStreamExt;
+
+        let object_store = ObjectStore::memory();
+        let base = Path::from("base");
+        let versions_dir = base.child(VERSIONS_DIR);
+
+        // Create some regular manifests
+        for version in [1, 2, 3] {
+            let path = ManifestNamingScheme::V2.manifest_path(&base, version);
+            object_store.put(&path, b"".as_slice()).await.unwrap();
+        }
+
+        // Create some detached manifests
+        let detached_versions: Vec<u64> = vec![
+            100 | DETACHED_VERSION_MASK,
+            200 | DETACHED_VERSION_MASK,
+            300 | DETACHED_VERSION_MASK,
+        ];
+        for version in &detached_versions {
+            let path = versions_dir.child(format!("d{}.manifest", version));
+            object_store.put(&path, b"".as_slice()).await.unwrap();
+        }
+
+        // List detached manifests
+        let detached_locations: Vec<ManifestLocation> =
+            list_detached_manifests(&base, &object_store.inner)
+                .try_collect()
+                .await
+                .unwrap();
+
+        assert_eq!(detached_locations.len(), 3);
+        for loc in &detached_locations {
+            assert_eq!(loc.naming_scheme, ManifestNamingScheme::V2);
+        }
+
+        let mut found_versions: Vec<u64> = detached_locations.iter().map(|l| l.version).collect();
+        found_versions.sort();
+        let mut expected_versions = detached_versions.clone();
+        expected_versions.sort();
+        assert_eq!(found_versions, expected_versions);
+    }
 }
diff --git a/rust/lance-table/src/rowids/index.rs b/rust/lance-table/src/rowids/index.rs
index 0bf4eb76027..7ad04bd961d 100644
--- a/rust/lance-table/src/rowids/index.rs
+++ b/rust/lance-table/src/rowids/index.rs
@@ -97,6 +97,7 @@ fn decompose_sequence(
 ) -> Vec<(RangeInclusive<u64>, (U64Segment, U64Segment))> {
     let mut start_address: u64 = RowAddress::first_row(frag_index.fragment_id).into();
     let mut current_offset = 0u32;
+    let no_deletions = frag_index.deletion_vector.is_empty();
 
     frag_index
         .row_id_sequence
@@ -105,38 +106,80 @@ fn decompose_sequence(
         .filter_map(|segment| {
             let segment_len = segment.len();
 
-            let active_pairs: Vec<(u64, u64)> = segment
-                .iter()
-                .enumerate()
-                .filter_map(|(i, row_id)| {
-                    let row_offset = current_offset + i as u32;
-                    if !frag_index.deletion_vector.contains(row_offset) {
-                        let address = start_address + i as u64;
-                        Some((row_id, address))
-                    } else {
-                        None
-                    }
-                })
-                .collect();
+            let result = if no_deletions {
+                decompose_segment_no_deletions(segment, start_address)
+            } else {
+                decompose_segment_with_deletions(
+                    segment,
+                    start_address,
+                    current_offset,
+                    &frag_index.deletion_vector,
+                )
+            };
 
             current_offset += segment_len as u32;
             start_address += segment_len as u64;
 
-            if active_pairs.is_empty() {
-                return None;
-            }
-
-            let row_ids: Vec<u64> = active_pairs.iter().map(|(rid, _)| *rid).collect();
-            let addresses: Vec<u64> = active_pairs.iter().map(|(_, addr)| *addr).collect();
-
-            let row_id_segment = U64Segment::from_iter(row_ids.iter().copied());
-            let address_segment = U64Segment::from_iter(addresses.iter().copied());
+            result
+        })
+        .collect()
+}
 
-            let coverage = row_id_segment.range()?;
+/// Build an IndexChunk from a list of (row_id, address) pairs.
+fn build_chunk_from_pairs(pairs: Vec<(u64, u64)>) -> Option<IndexChunk> {
+    if pairs.is_empty() {
+        return None;
+    }
+    let (row_ids, addresses): (Vec<u64>, Vec<u64>) = pairs.into_iter().unzip();
+    let row_id_segment = U64Segment::from_iter(row_ids);
+    let address_segment = U64Segment::from_iter(addresses);
+    let coverage = row_id_segment.range()?;
+    Some((coverage, (row_id_segment, address_segment)))
+}
 
+/// Fast path: no deletions. O(1) for Range segments.
+fn decompose_segment_no_deletions(segment: &U64Segment, start_address: u64) -> Option<IndexChunk> {
+    match segment {
+        U64Segment::Range(range) if !range.is_empty() => {
+            let len = range.end - range.start;
+            let row_id_segment = U64Segment::Range(range.clone());
+            let address_segment = U64Segment::Range(start_address..start_address + len);
+            let coverage = range.start..=range.end - 1;
             Some((coverage, (row_id_segment, address_segment)))
+        }
+        _ if segment.is_empty() => None,
+        _ => {
+            // Non-Range segments: must iterate to build address mapping.
+            let pairs: Vec<(u64, u64)> = segment
+                .iter()
+                .enumerate()
+                .map(|(i, row_id)| (row_id, start_address + i as u64))
+                .collect();
+            build_chunk_from_pairs(pairs)
+        }
+    }
+}
+
+/// Slow path: has deletions, must check each row.
+fn decompose_segment_with_deletions(
+    segment: &U64Segment,
+    start_address: u64,
+    current_offset: u32,
+    deletion_vector: &DeletionVector,
+) -> Option<IndexChunk> {
+    let pairs: Vec<(u64, u64)> = segment
+        .iter()
+        .enumerate()
+        .filter_map(|(i, row_id)| {
+            let row_offset = current_offset + i as u32;
+            if !deletion_vector.contains(row_offset) {
+                Some((row_id, start_address + i as u64))
+            } else {
+                None
+            }
         })
-        .collect()
+        .collect();
+    build_chunk_from_pairs(pairs)
 }
 
 type IndexChunk = (RangeInclusive<u64>, (U64Segment, U64Segment));
@@ -547,6 +590,132 @@ mod tests {
         })
     }
 
+    #[test]
+    fn test_large_range_segments_no_deletions() {
+        // Simulates a real-world scenario: many fragments with large Range segments
+        // and no deletions. Before optimization, this would iterate over all rows
+        // (O(total_rows)). After optimization, it's O(num_fragments).
+        let rows_per_fragment = 250_000u64;
+        let num_fragments = 100u32;
+        let mut offset = 0u64;
+
+        let fragment_indices: Vec<FragmentRowIdIndex> = (0..num_fragments)
+            .map(|frag_id| {
+                let start = offset;
+                offset += rows_per_fragment;
+                FragmentRowIdIndex {
+                    fragment_id: frag_id,
+                    row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::Range(
+                        start..start + rows_per_fragment,
+                    )])),
+                    deletion_vector: Arc::new(DeletionVector::default()),
+                }
+            })
+            .collect();
+
+        let start = std::time::Instant::now();
+        let index = RowIdIndex::new(&fragment_indices).unwrap();
+        let elapsed = start.elapsed();
+
+        // Verify correctness at boundaries
+        assert_eq!(index.get(0), Some(RowAddress::new_from_parts(0, 0)));
+        assert_eq!(
+            index.get(rows_per_fragment - 1),
+            Some(RowAddress::new_from_parts(0, rows_per_fragment as u32 - 1))
+        );
+        assert_eq!(
+            index.get(rows_per_fragment),
+            Some(RowAddress::new_from_parts(1, 0))
+        );
+        let last_row = num_fragments as u64 * rows_per_fragment - 1;
+        assert_eq!(
+            index.get(last_row),
+            Some(RowAddress::new_from_parts(
+                num_fragments - 1,
+                rows_per_fragment as u32 - 1
+            ))
+        );
+        assert_eq!(index.get(last_row + 1), None);
+
+        // With the optimization, building an index for 25M rows across 100 fragments
+        // should complete in well under 1 second (typically < 1ms).
+        assert!(
+            elapsed.as_secs() < 1,
+            "Index build took {:?} for {} fragments x {} rows = {} total rows. \
+             This suggests the O(rows) -> O(fragments) optimization is not working.",
+            elapsed,
+            num_fragments,
+            rows_per_fragment,
+            num_fragments as u64 * rows_per_fragment,
+        );
+    }
+
+    #[test]
+    fn test_large_range_segments_with_deletions() {
+        let rows_per_fragment = 1_000u64;
+        let num_fragments = 10u32;
+        let mut offset = 0u64;
+
+        let fragment_indices: Vec<FragmentRowIdIndex> = (0..num_fragments)
+            .map(|frag_id| {
+                let start = offset;
+                offset += rows_per_fragment;
+
+                // Delete every 3rd row (offsets 0, 3, 6, ...) within each fragment.
+                let mut deleted = roaring::RoaringBitmap::new();
+                for i in (0..rows_per_fragment as u32).step_by(3) {
+                    deleted.insert(i);
+                }
+
+                FragmentRowIdIndex {
+                    fragment_id: frag_id,
+                    row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::Range(
+                        start..start + rows_per_fragment,
+                    )])),
+                    deletion_vector: Arc::new(DeletionVector::Bitmap(deleted)),
+                }
+            })
+            .collect();
+
+        let index = RowIdIndex::new(&fragment_indices).unwrap();
+
+        // Deleted rows (offset 0, 3, 6, ...) should not be found.
+        // Row ID 0 has offset 0 in fragment 0 -> deleted.
+        assert_eq!(index.get(0), None);
+        // Row ID 3 has offset 3 in fragment 0 -> deleted.
+        assert_eq!(index.get(3), None);
+
+        // Non-deleted rows should resolve correctly.
+        // Row ID 1 has offset 1 in fragment 0 -> address (frag=0, row=1).
+        assert_eq!(index.get(1), Some(RowAddress::new_from_parts(0, 1)));
+        // Row ID 2 has offset 2 in fragment 0 -> address (frag=0, row=2).
+        assert_eq!(index.get(2), Some(RowAddress::new_from_parts(0, 2)));
+        // Row ID 4 has offset 4 in fragment 0 -> address (frag=0, row=4).
+        assert_eq!(index.get(4), Some(RowAddress::new_from_parts(0, 4)));
+
+        // Check second fragment: row IDs start at 1000.
+        // Row ID 1000 has offset 0 in fragment 1 -> deleted.
+        assert_eq!(index.get(rows_per_fragment), None);
+        // Row ID 1001 has offset 1 in fragment 1 -> address (frag=1, row=1).
+        assert_eq!(
+            index.get(rows_per_fragment + 1),
+            Some(RowAddress::new_from_parts(1, 1))
+        );
+
+        // Last fragment, last non-deleted row.
+        // Row ID 9999 has offset 999 in fragment 9 -> 999 % 3 == 0 -> deleted.
+        let last_row = num_fragments as u64 * rows_per_fragment - 1;
+        assert_eq!(index.get(last_row), None);
+        // Row ID 9998 has offset 998 -> 998 % 3 == 2 -> not deleted.
+        assert_eq!(
+            index.get(last_row - 1),
+            Some(RowAddress::new_from_parts(num_fragments - 1, 998))
+        );
+
+        // Out of range.
+        assert_eq!(index.get(last_row + 1), None);
+    }
+
     proptest::proptest! {
         #[test]
         fn test_new_index_robustness(row_ids in arbitrary_row_ids(0..5, 0..32)) {
diff --git a/rust/lance-table/src/rowids/version.rs b/rust/lance-table/src/rowids/version.rs
index f1c528c91ef..80f3d06db60 100644
--- a/rust/lance-table/src/rowids/version.rs
+++ b/rust/lance-table/src/rowids/version.rs
@@ -7,10 +7,14 @@
 //! update version for each row in a Lance dataset, enabling efficient
 //! cross-version diff operations.
 
+use std::sync::Arc;
+
 use deepsize::DeepSizeOf;
 use lance_core::Error;
 use lance_core::Result;
 use prost::Message;
+use serde::de::Deserializer;
+use serde::ser::Serializer;
 use serde::{Deserialize, Serialize};
 
 use crate::format::{ExternalFile, Fragment, pb};
@@ -210,19 +214,59 @@ impl<'a> Iterator for VersionsIter<'a> {
 
 /// Metadata about the location of dataset version sequence data
 /// Following the same pattern as RowIdMeta
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+///
+/// When stored inline, identical byte sequences are shared across fragments
+/// via `Arc<[u8]>` to reduce manifest memory for large tables.
+#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)]
 pub enum RowDatasetVersionMeta {
     /// Small sequences stored inline in the fragment metadata
-    Inline(Vec<u8>),
+    Inline(Arc<[u8]>),
     /// Large sequences stored in external files
     External(ExternalFile),
 }
 
+// Custom Serialize: convert Arc<[u8]> to slice for transparent JSON output
+impl Serialize for RowDatasetVersionMeta {
+    fn serialize<S: Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
+        #[derive(Serialize)]
+        #[serde(untagged)]
+        enum Helper<'a> {
+            Inline { inline: &'a [u8] },
+            External { external: &'a ExternalFile },
+        }
+
+        match self {
+            Self::Inline(data) => Helper::Inline {
+                inline: data.as_ref(),
+            }
+            .serialize(serializer),
+            Self::External(file) => Helper::External { external: file }.serialize(serializer),
+        }
+    }
+}
+
+// Custom Deserialize: read Vec<u8> and convert to Arc<[u8]>
+impl<'de> Deserialize<'de> for RowDatasetVersionMeta {
+    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
+        #[derive(Deserialize)]
+        #[serde(untagged)]
+        enum Helper {
+            Inline { inline: Vec<u8> },
+            External { external: ExternalFile },
+        }
+
+        match Helper::deserialize(deserializer)? {
+            Helper::Inline { inline } => Ok(Self::Inline(Arc::from(inline))),
+            Helper::External { external } => Ok(Self::External(external)),
+        }
+    }
+}
+
 impl RowDatasetVersionMeta {
     /// Create inline metadata from a version sequence
     pub fn from_sequence(sequence: &RowDatasetVersionSequence) -> lance_core::Result<Self> {
         let bytes = write_dataset_versions(sequence);
-        Ok(Self::Inline(bytes))
+        Ok(Self::Inline(Arc::from(bytes)))
     }
 
     /// Create external metadata reference
@@ -248,7 +292,7 @@ pub fn last_updated_at_version_meta_to_pb(
     meta.as_ref().map(|m| match m {
         RowDatasetVersionMeta::Inline(data) => {
             pb::data_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions(
-                data.clone(),
+                data.to_vec(),
             )
         }
         RowDatasetVersionMeta::External(file) => {
@@ -269,7 +313,7 @@ pub fn created_at_version_meta_to_pb(
 ) -> Option<pb::data_fragment::CreatedAtVersionSequence> {
     meta.as_ref().map(|m| match m {
         RowDatasetVersionMeta::Inline(data) => {
-            pb::data_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions(data.clone())
+            pb::data_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions(data.to_vec())
         }
         RowDatasetVersionMeta::External(file) => {
             pb::data_fragment::CreatedAtVersionSequence::ExternalCreatedAtVersions(
@@ -561,7 +605,7 @@ impl TryFrom<pb::data_fragment::LastUpdatedAtVersionSequence> for RowDatasetVers
     fn try_from(value: pb::data_fragment::LastUpdatedAtVersionSequence) -> Result<Self> {
         match value {
             pb::data_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions(data) => {
-                Ok(Self::Inline(data))
+                Ok(Self::Inline(Arc::from(data)))
             }
             pb::data_fragment::LastUpdatedAtVersionSequence::ExternalLastUpdatedAtVersions(
                 file,
@@ -580,7 +624,7 @@ impl TryFrom<pb::data_fragment::CreatedAtVersionSequence> for RowDatasetVersionM
     fn try_from(value: pb::data_fragment::CreatedAtVersionSequence) -> Result<Self> {
         match value {
             pb::data_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions(data) => {
-                Ok(Self::Inline(data))
+                Ok(Self::Inline(Arc::from(data)))
             }
             pb::data_fragment::CreatedAtVersionSequence::ExternalCreatedAtVersions(file) => {
                 Ok(Self::External(ExternalFile {
diff --git a/rust/lance-table/src/utils/stream.rs b/rust/lance-table/src/utils/stream.rs
index 31f5fc87ced..f6fbbd45a61 100644
--- a/rust/lance-table/src/utils/stream.rs
+++ b/rust/lance-table/src/utils/stream.rs
@@ -161,6 +161,56 @@ fn apply_deletions_as_nulls(batch: RecordBatch, mask: &BooleanArray) -> Result<R
     )?)
 }
 
+/// Extract version values for a batch selection by binary-searching over
+/// precomputed RLE run offsets. Single-run fragments (the common case)
+/// take the O(1) fast path.
+fn version_values_for_selection(
+    sequence: &crate::rowids::version::RowDatasetVersionSequence,
+    params: &ReadBatchParams,
+    batch_offset: u32,
+    num_rows: u32,
+) -> Result<Vec<u64>> {
+    let selection = params
+        .slice(batch_offset as usize, num_rows as usize)
+        .unwrap()
+        .to_ranges()
+        .unwrap();
+
+    if sequence.runs.len() == 1 {
+        return Ok(vec![sequence.runs[0].version(); num_rows as usize]);
+    }
+
+    let mut versions = Vec::with_capacity(num_rows as usize);
+    let run_offsets: Vec<usize> = sequence
+        .runs
+        .iter()
+        .scan(0usize, |acc, run| {
+            let start = *acc;
+            *acc += run.len();
+            Some(start)
+        })
+        .collect();
+    let total_len: usize = sequence.runs.iter().map(|r| r.len()).sum();
+
+    for r in &selection {
+        for pos in r.start..r.end {
+            let pos = pos as usize;
+            if pos >= total_len {
+                return Err(lance_core::Error::internal(format!(
+                    "version column position {} out of range (total_len={})",
+                    pos, total_len
+                )));
+            }
+            let run_idx = match run_offsets.binary_search(&pos) {
+                Ok(idx) => idx,
+                Err(idx) => idx - 1,
+            };
+            versions.push(sequence.runs[run_idx].version());
+        }
+    }
+    Ok(versions)
+}
+
 /// Configuration needed to apply row ids and deletions to a batch
 #[derive(Debug)]
 pub struct RowIdAndDeletesConfig {
@@ -296,24 +346,12 @@ pub fn apply_row_id_and_deletes(
 
         if config.with_row_last_updated_at_version {
             let version_arr = if let Some(sequence) = &config.last_updated_at_sequence {
-                // Get the range of rows for this batch
-                let selection = config
-                    .params
-                    .slice(batch_offset as usize, num_rows as usize)
-                    .unwrap()
-                    .to_ranges()
-                    .unwrap();
-                // Extract version values for the selected ranges
-                let versions: Vec<u64> = selection
-                    .iter()
-                    .flat_map(|r| {
-                        sequence
-                            .versions()
-                            .skip(r.start as usize)
-                            .take((r.end - r.start) as usize)
-                    })
-                    .collect();
-                Arc::new(UInt64Array::from(versions))
+                Arc::new(UInt64Array::from(version_values_for_selection(
+                    sequence,
+                    &config.params,
+                    batch_offset,
+                    num_rows,
+                )?))
             } else {
                 // Default to version 1 if sequence not provided
                 Arc::new(UInt64Array::from(vec![1u64; num_rows as usize]))
@@ -324,24 +362,12 @@ pub fn apply_row_id_and_deletes(
 
         if config.with_row_created_at_version {
             let version_arr = if let Some(sequence) = &config.created_at_sequence {
-                // Get the range of rows for this batch
-                let selection = config
-                    .params
-                    .slice(batch_offset as usize, num_rows as usize)
-                    .unwrap()
-                    .to_ranges()
-                    .unwrap();
-                // Extract version values for the selected ranges
-                let versions: Vec<u64> = selection
-                    .iter()
-                    .flat_map(|r| {
-                        sequence
-                            .versions()
-                            .skip(r.start as usize)
-                            .take((r.end - r.start) as usize)
-                    })
-                    .collect();
-                Arc::new(UInt64Array::from(versions))
+                Arc::new(UInt64Array::from(version_values_for_selection(
+                    sequence,
+                    &config.params,
+                    batch_offset,
+                    num_rows,
+                )?))
             } else {
                 // Default to version 1 if sequence not provided
                 Arc::new(UInt64Array::from(vec![1u64; num_rows as usize]))
@@ -647,4 +673,134 @@ mod tests {
             }
         }
     }
+
+    #[tokio::test]
+    async fn test_version_column_with_deletions() {
+        use crate::rowids::segment::U64Segment;
+        use crate::rowids::version::{RowDatasetVersionRun, RowDatasetVersionSequence};
+
+        let seq = Arc::new(RowDatasetVersionSequence {
+            runs: vec![RowDatasetVersionRun {
+                span: U64Segment::Range(0..100),
+                version: 42,
+            }],
+        });
+
+        let data = batch_task_stream(
+            lance_datagen::gen_batch()
+                .col("x", lance_datagen::array::rand::<Int32Type>())
+                .into_reader_stream(RowCount::from(10), BatchCount::from(10))
+                .0,
+        );
+
+        let config = RowIdAndDeletesConfig {
+            params: ReadBatchParams::RangeFull,
+            with_row_id: true,
+            with_row_addr: false,
+            with_row_last_updated_at_version: false,
+            with_row_created_at_version: true,
+            deletion_vector: Some(Arc::new(DeletionVector::Bitmap(RoaringBitmap::from_iter(
+                0..35,
+            )))),
+            row_id_sequence: None,
+            last_updated_at_sequence: None,
+            created_at_sequence: Some(seq),
+            make_deletions_null: false,
+            total_num_rows: 100,
+        };
+        let stream = super::wrap_with_row_id_and_delete(data, 0, config);
+        let batches: Vec<_> = stream
+            .buffered(1)
+            .try_filter(|b| std::future::ready(b.num_rows() > 0))
+            .try_collect()
+            .await
+            .unwrap();
+
+        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(total_rows, 65);
+
+        for batch in &batches {
+            let versions = batch
+                .column_by_name("_row_created_at_version")
+                .unwrap()
+                .as_primitive::<UInt64Type>()
+                .values();
+            assert!(versions.iter().all(|&v| v == 42));
+        }
+    }
+
+    #[tokio::test]
+    async fn test_version_column_multi_run() {
+        use crate::rowids::segment::U64Segment;
+        use crate::rowids::version::{RowDatasetVersionRun, RowDatasetVersionSequence};
+
+        // 3 runs: 0..40 v1, 40..70 v2, 70..100 v3
+        let seq = Arc::new(RowDatasetVersionSequence {
+            runs: vec![
+                RowDatasetVersionRun {
+                    span: U64Segment::Range(0..40),
+                    version: 1,
+                },
+                RowDatasetVersionRun {
+                    span: U64Segment::Range(40..70),
+                    version: 2,
+                },
+                RowDatasetVersionRun {
+                    span: U64Segment::Range(70..100),
+                    version: 3,
+                },
+            ],
+        });
+
+        // Delete 0..20 and 60..80 (spans run boundary).
+        // Survivors: 20..40 (v1), 40..60 (v2), 80..100 (v3) = 60 rows
+        let mut deletions = RoaringBitmap::from_iter(0..20);
+        deletions.extend(60..80);
+
+        let data = batch_task_stream(
+            lance_datagen::gen_batch()
+                .col("x", lance_datagen::array::rand::<Int32Type>())
+                .into_reader_stream(RowCount::from(10), BatchCount::from(10))
+                .0,
+        );
+
+        let config = RowIdAndDeletesConfig {
+            params: ReadBatchParams::RangeFull,
+            with_row_id: true,
+            with_row_addr: false,
+            with_row_last_updated_at_version: false,
+            with_row_created_at_version: true,
+            deletion_vector: Some(Arc::new(DeletionVector::Bitmap(deletions))),
+            row_id_sequence: None,
+            last_updated_at_sequence: None,
+            created_at_sequence: Some(seq),
+            make_deletions_null: false,
+            total_num_rows: 100,
+        };
+        let stream = super::wrap_with_row_id_and_delete(data, 0, config);
+        let batches: Vec<_> = stream
+            .buffered(1)
+            .try_filter(|b| std::future::ready(b.num_rows() > 0))
+            .try_collect()
+            .await
+            .unwrap();
+
+        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(total_rows, 60);
+
+        let all_versions: Vec<u64> = batches
+            .iter()
+            .flat_map(|b| {
+                b.column_by_name("_row_created_at_version")
+                    .unwrap()
+                    .as_primitive::<UInt64Type>()
+                    .values()
+                    .to_vec()
+            })
+            .collect();
+
+        assert!(all_versions[..20].iter().all(|&v| v == 1));
+        assert!(all_versions[20..40].iter().all(|&v| v == 2));
+        assert!(all_versions[40..60].iter().all(|&v| v == 3));
+    }
 }
diff --git a/rust/lance-testing/src/lib.rs b/rust/lance-testing/src/lib.rs
index 135530b317b..b7e6d968ac3 100644
--- a/rust/lance-testing/src/lib.rs
+++ b/rust/lance-testing/src/lib.rs
@@ -2,3 +2,4 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 pub mod datagen;
+pub mod progress;
diff --git a/rust/lance-testing/src/progress.rs b/rust/lance-testing/src/progress.rs
new file mode 100644
index 00000000000..9b7f91281e8
--- /dev/null
+++ b/rust/lance-testing/src/progress.rs
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+/// Define a test-only progress recorder that captures stage events in memory.
+#[macro_export]
+macro_rules! define_stage_event_progress {
+    ($name:ident, $progress_trait:path, $result:ty) => {
+        #[derive(Debug, Default)]
+        struct $name {
+            events: ::std::sync::Mutex<Vec<(String, String, u64)>>,
+        }
+
+        impl $name {
+            fn recorded_events(&self) -> Vec<(String, String, u64)> {
+                self.events
+                    .lock()
+                    .expect("recording progress mutex poisoned")
+                    .clone()
+            }
+        }
+
+        #[::async_trait::async_trait]
+        impl $progress_trait for $name {
+            async fn stage_start(&self, stage: &str, total: Option<u64>, _unit: &str) -> $result {
+                self.events
+                    .lock()
+                    .expect("recording progress mutex poisoned")
+                    .push(("start".to_string(), stage.to_string(), total.unwrap_or(0)));
+                Ok(())
+            }
+
+            async fn stage_progress(&self, stage: &str, completed: u64) -> $result {
+                self.events
+                    .lock()
+                    .expect("recording progress mutex poisoned")
+                    .push(("progress".to_string(), stage.to_string(), completed));
+                Ok(())
+            }
+
+            async fn stage_complete(&self, stage: &str) -> $result {
+                self.events
+                    .lock()
+                    .expect("recording progress mutex poisoned")
+                    .push(("complete".to_string(), stage.to_string(), 0));
+                Ok(())
+            }
+        }
+    };
+}
diff --git a/rust/lance-tokenizer/Cargo.toml b/rust/lance-tokenizer/Cargo.toml
new file mode 100644
index 00000000000..0f9909522ce
--- /dev/null
+++ b/rust/lance-tokenizer/Cargo.toml
@@ -0,0 +1,28 @@
+[package]
+name = "lance-tokenizer"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+readme = "README.md"
+description = "Tokenizer abstractions and implementations for Lance"
+keywords.workspace = true
+categories.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+jieba-rs = { workspace = true, optional = true }
+lindera = { workspace = true, optional = true }
+rust-stemmers = "1.2.0"
+serde = { workspace = true, features = ["derive"] }
+unicode-normalization = "0.1.25"
+
+[features]
+jieba-rs = ["dep:jieba-rs"]
+lindera = ["dep:lindera"]
+tokenizer-jieba = ["jieba-rs"]
+tokenizer-lindera = ["lindera"]
+
+[lints]
+workspace = true
diff --git a/rust/lance-tokenizer/README.md b/rust/lance-tokenizer/README.md
new file mode 100644
index 00000000000..99541efd133
--- /dev/null
+++ b/rust/lance-tokenizer/README.md
@@ -0,0 +1,3 @@
+# lance-tokenizer
+
+Internal tokenizer abstractions and implementations used by Lance.
diff --git a/rust/lance-tokenizer/src/alphanum_only.rs b/rust/lance-tokenizer/src/alphanum_only.rs
new file mode 100644
index 00000000000..6c83db1b412
--- /dev/null
+++ b/rust/lance-tokenizer/src/alphanum_only.rs
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+// SPDX-License-Identifier: MIT
+// Adapted from Tantivy v0.24.2 alphanum-only filter.
+// Copyright (c) 2017-present Tantivy contributors.
+
+use crate::{Token, TokenFilter, TokenStream, Tokenizer};
+
+#[derive(Clone)]
+pub struct AlphaNumOnlyFilter;
+
+pub struct AlphaNumOnlyFilterStream<T> {
+    tail: T,
+}
+
+impl<T> AlphaNumOnlyFilterStream<T> {
+    fn predicate(&self, token: &Token) -> bool {
+        token.text.chars().all(|ch| ch.is_ascii_alphanumeric())
+    }
+}
+
+impl TokenFilter for AlphaNumOnlyFilter {
+    type Tokenizer<T: Tokenizer> = AlphaNumOnlyFilterWrapper<T>;
+
+    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T> {
+        AlphaNumOnlyFilterWrapper(tokenizer)
+    }
+}
+
+#[derive(Clone)]
+pub struct AlphaNumOnlyFilterWrapper<T>(T);
+
+impl<T: Tokenizer> Tokenizer for AlphaNumOnlyFilterWrapper<T> {
+    type TokenStream<'a> = AlphaNumOnlyFilterStream<T::TokenStream<'a>>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        AlphaNumOnlyFilterStream {
+            tail: self.0.token_stream(text),
+        }
+    }
+}
+
+impl<T: TokenStream> TokenStream for AlphaNumOnlyFilterStream<T> {
+    fn advance(&mut self) -> bool {
+        while self.tail.advance() {
+            if self.predicate(self.tail.token()) {
+                return true;
+            }
+        }
+        false
+    }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+}
diff --git a/rust/lance-tokenizer/src/analyzer.rs b/rust/lance-tokenizer/src/analyzer.rs
new file mode 100644
index 00000000000..6d119d47871
--- /dev/null
+++ b/rust/lance-tokenizer/src/analyzer.rs
@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+// SPDX-License-Identifier: MIT
+// Adapted from Tantivy v0.24.2 tokenizer analyzer.
+// Copyright (c) 2017-present Tantivy contributors.
+
+use crate::{BoxTokenStream, TokenFilter, Tokenizer};
+
+#[derive(Clone)]
+pub struct TextAnalyzer {
+    tokenizer: Box<dyn BoxableTokenizer>,
+}
+
+impl<T: Tokenizer + Clone> From<T> for TextAnalyzer {
+    fn from(tokenizer: T) -> Self {
+        Self::builder(tokenizer).build()
+    }
+}
+
+impl Default for TextAnalyzer {
+    fn default() -> Self {
+        Self::from(crate::RawTokenizer::default())
+    }
+}
+
+impl TextAnalyzer {
+    pub fn builder<T: Tokenizer>(tokenizer: T) -> TextAnalyzerBuilder<T> {
+        TextAnalyzerBuilder { tokenizer }
+    }
+
+    pub fn token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a> {
+        self.tokenizer.token_stream(text)
+    }
+}
+
+pub trait BoxableTokenizer: 'static + Send + Sync {
+    fn box_token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a>;
+
+    fn box_clone(&self) -> Box<dyn BoxableTokenizer>;
+}
+
+impl<T: Tokenizer> BoxableTokenizer for T {
+    fn box_token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a> {
+        BoxTokenStream::new(self.token_stream(text))
+    }
+
+    fn box_clone(&self) -> Box<dyn BoxableTokenizer> {
+        Box::new(self.clone())
+    }
+}
+
+impl Clone for Box<dyn BoxableTokenizer> {
+    fn clone(&self) -> Self {
+        (**self).box_clone()
+    }
+}
+
+impl Tokenizer for Box<dyn BoxableTokenizer> {
+    type TokenStream<'a> = BoxTokenStream<'a>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        (**self).box_token_stream(text)
+    }
+}
+
+pub struct TextAnalyzerBuilder<T = Box<dyn BoxableTokenizer>> {
+    tokenizer: T,
+}
+
+impl<T: Tokenizer> TextAnalyzerBuilder<T> {
+    pub fn filter<F: TokenFilter>(self, token_filter: F) -> TextAnalyzerBuilder<F::Tokenizer<T>> {
+        TextAnalyzerBuilder {
+            tokenizer: token_filter.transform(self.tokenizer),
+        }
+    }
+
+    pub fn dynamic(self) -> TextAnalyzerBuilder {
+        TextAnalyzerBuilder {
+            tokenizer: Box::new(self.tokenizer),
+        }
+    }
+
+    pub fn filter_dynamic<F: TokenFilter>(self, token_filter: F) -> TextAnalyzerBuilder {
+        self.filter(token_filter).dynamic()
+    }
+
+    pub fn build(self) -> TextAnalyzer {
+        TextAnalyzer {
+            tokenizer: Box::new(self.tokenizer),
+        }
+    }
+}
diff --git a/rust/lance-tokenizer/src/ascii_folding_filter.rs b/rust/lance-tokenizer/src/ascii_folding_filter.rs
new file mode 100644
index 00000000000..376c0e1ebdb
--- /dev/null
+++ b/rust/lance-tokenizer/src/ascii_folding_filter.rs
@@ -0,0 +1,152 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::mem;
+
+use unicode_normalization::{UnicodeNormalization, char::is_combining_mark};
+
+use crate::{Token, TokenFilter, TokenStream, Tokenizer};
+
+#[derive(Clone)]
+pub struct AsciiFoldingFilter;
+
+impl TokenFilter for AsciiFoldingFilter {
+    type Tokenizer<T: Tokenizer> = AsciiFoldingFilterWrapper<T>;
+
+    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T> {
+        AsciiFoldingFilterWrapper {
+            tokenizer,
+            buffer: String::new(),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct AsciiFoldingFilterWrapper<T> {
+    tokenizer: T,
+    buffer: String,
+}
+
+impl<T: Tokenizer> Tokenizer for AsciiFoldingFilterWrapper<T> {
+    type TokenStream<'a> = AsciiFoldingFilterTokenStream<'a, T::TokenStream<'a>>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        self.buffer.clear();
+        AsciiFoldingFilterTokenStream {
+            buffer: &mut self.buffer,
+            tail: self.tokenizer.token_stream(text),
+        }
+    }
+}
+
+pub struct AsciiFoldingFilterTokenStream<'a, T> {
+    buffer: &'a mut String,
+    tail: T,
+}
+
+impl<T: TokenStream> TokenStream for AsciiFoldingFilterTokenStream<'_, T> {
+    fn advance(&mut self) -> bool {
+        if !self.tail.advance() {
+            return false;
+        }
+        if !self.token_mut().text.is_ascii() {
+            to_ascii(&self.tail.token().text, self.buffer);
+            mem::swap(&mut self.tail.token_mut().text, self.buffer);
+        }
+        true
+    }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+}
+
+fn to_ascii(text: &str, output: &mut String) {
+    output.clear();
+    for ch in text.chars() {
+        if ch.is_ascii() {
+            output.push(ch);
+            continue;
+        }
+
+        if let Some(mapped) = fold_char(ch) {
+            output.push_str(mapped);
+            continue;
+        }
+
+        let original_len = output.len();
+        for decomposed in ch.nfkd() {
+            if decomposed.is_ascii() {
+                output.push(decomposed);
+            } else if is_combining_mark(decomposed) {
+                continue;
+            } else if let Some(mapped) = fold_char(decomposed) {
+                output.push_str(mapped);
+            }
+        }
+
+        if output.len() == original_len {
+            output.push(ch);
+        }
+    }
+}
+
+fn fold_char(ch: char) -> Option<&'static str> {
+    match ch {
+        'ß' => Some("ss"),
+        'ẞ' => Some("SS"),
+        'Æ' => Some("AE"),
+        'æ' => Some("ae"),
+        'Œ' => Some("OE"),
+        'œ' => Some("oe"),
+        'Ø' => Some("O"),
+        'ø' => Some("o"),
+        'Ł' => Some("L"),
+        'ł' => Some("l"),
+        'Đ' | 'Ð' => Some("D"),
+        'đ' | 'ð' => Some("d"),
+        'Þ' => Some("TH"),
+        'þ' => Some("th"),
+        'Ħ' => Some("H"),
+        'ħ' => Some("h"),
+        'Ŧ' => Some("T"),
+        'ŧ' => Some("t"),
+        'Ŋ' => Some("N"),
+        'ŋ' => Some("n"),
+        'ı' => Some("i"),
+        'ĸ' => Some("k"),
+        'ſ' => Some("s"),
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{AsciiFoldingFilter, RawTokenizer, TextAnalyzer, Token};
+
+    fn collect_tokens(text: &str) -> Vec<Token> {
+        let mut analyzer = TextAnalyzer::builder(RawTokenizer::default())
+            .filter(AsciiFoldingFilter)
+            .build();
+        let mut stream = analyzer.token_stream(text);
+        let mut tokens = Vec::new();
+        stream.process(&mut |token| tokens.push(token.clone()));
+        tokens
+    }
+
+    #[test]
+    fn test_ascii_folding_accents() {
+        let tokens = collect_tokens("café");
+        assert_eq!(tokens[0].text, "cafe");
+    }
+
+    #[test]
+    fn test_ascii_folding_sharp_s() {
+        let tokens = collect_tokens("straße");
+        assert_eq!(tokens[0].text, "strasse");
+    }
+}
diff --git a/rust/lance-tokenizer/src/jieba.rs b/rust/lance-tokenizer/src/jieba.rs
new file mode 100644
index 00000000000..5ba51bf6a06
--- /dev/null
+++ b/rust/lance-tokenizer/src/jieba.rs
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use crate::{TextAnalyzer, TextAnalyzerBuilder, Token, TokenStream, Tokenizer};
+
+#[derive(Clone)]
+pub struct JiebaTokenizer {
+    jieba: jieba_rs::Jieba,
+}
+
+impl JiebaTokenizer {
+    pub fn new(jieba: jieba_rs::Jieba) -> Self {
+        Self { jieba }
+    }
+
+    pub fn analyzer(self) -> TextAnalyzer {
+        TextAnalyzer::builder(self).build()
+    }
+
+    pub fn analyzer_builder(self) -> TextAnalyzerBuilder {
+        TextAnalyzer::builder(self).dynamic()
+    }
+}
+
+pub struct JiebaTokenStream {
+    tokens: Vec<Token>,
+    index: usize,
+}
+
+impl TokenStream for JiebaTokenStream {
+    fn advance(&mut self) -> bool {
+        if self.index < self.tokens.len() {
+            self.index += 1;
+            true
+        } else {
+            false
+        }
+    }
+
+    fn token(&self) -> &Token {
+        &self.tokens[self.index - 1]
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        &mut self.tokens[self.index - 1]
+    }
+}
+
+impl Tokenizer for JiebaTokenizer {
+    type TokenStream<'a> = JiebaTokenStream;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        let mut indices = text.char_indices().collect::<Vec<_>>();
+        indices.push((text.len(), '\0'));
+        let orig_tokens = self
+            .jieba
+            .tokenize(text, jieba_rs::TokenizeMode::Search, true);
+        let tokens = orig_tokens
+            .into_iter()
+            .map(|token| Token {
+                offset_from: indices[token.start].0,
+                offset_to: indices[token.end].0,
+                position: token.start,
+                text: text[indices[token.start].0..indices[token.end].0].to_owned(),
+                position_length: token.end - token.start,
+            })
+            .collect();
+        JiebaTokenStream { tokens, index: 0 }
+    }
+}
diff --git a/rust/lance-tokenizer/src/lib.rs b/rust/lance-tokenizer/src/lib.rs
new file mode 100644
index 00000000000..54cc45dbbf4
--- /dev/null
+++ b/rust/lance-tokenizer/src/lib.rs
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+mod alphanum_only;
+mod analyzer;
+mod ascii_folding_filter;
+#[cfg(feature = "tokenizer-jieba")]
+mod jieba;
+mod lower_caser;
+mod ngram_tokenizer;
+mod raw_tokenizer;
+mod remove_long;
+mod simple_tokenizer;
+mod stemmer;
+mod stop_word_filter;
+mod tokenizer_api;
+mod whitespace_tokenizer;
+
+#[cfg(feature = "tokenizer-lindera")]
+mod lindera;
+
+pub use alphanum_only::AlphaNumOnlyFilter;
+pub use analyzer::{TextAnalyzer, TextAnalyzerBuilder};
+pub use ascii_folding_filter::AsciiFoldingFilter;
+#[cfg(feature = "tokenizer-jieba")]
+pub use jieba::JiebaTokenizer;
+#[cfg(feature = "tokenizer-lindera")]
+pub use lindera::LinderaTokenizer;
+pub use lower_caser::LowerCaser;
+pub use ngram_tokenizer::NgramTokenizer;
+pub use raw_tokenizer::RawTokenizer;
+pub use remove_long::RemoveLongFilter;
+pub use simple_tokenizer::{SimpleTokenStream, SimpleTokenizer};
+pub use stemmer::{Language, Stemmer};
+pub use stop_word_filter::StopWordFilter;
+pub use tokenizer_api::{BoxTokenStream, Token, TokenFilter, TokenStream, Tokenizer};
+pub use whitespace_tokenizer::WhitespaceTokenizer;
diff --git a/rust/lance-tokenizer/src/lindera.rs b/rust/lance-tokenizer/src/lindera.rs
new file mode 100644
index 00000000000..14d210217e4
--- /dev/null
+++ b/rust/lance-tokenizer/src/lindera.rs
@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+// SPDX-License-Identifier: MIT
+// Adapted from lindera-tantivy v0.44.1.
+// Copyright (c) lindera-tantivy contributors.
+
+use std::path::Path;
+
+use lindera::token::Token as LinderaToken;
+use lindera::tokenizer::{Tokenizer as LinderaCoreTokenizer, TokenizerBuilder};
+
+use crate::{TextAnalyzer, TextAnalyzerBuilder, Token, TokenStream, Tokenizer};
+
+#[derive(Clone)]
+pub struct LinderaTokenizer {
+    tokenizer: LinderaCoreTokenizer,
+    token: Token,
+}
+
+impl LinderaTokenizer {
+    pub fn new() -> std::io::Result<Self> {
+        let builder = TokenizerBuilder::new().map_err(invalid_data)?;
+        let tokenizer = builder.build().map_err(invalid_data)?;
+        Ok(Self {
+            tokenizer,
+            token: Token::default(),
+        })
+    }
+
+    pub fn from_file(file_path: &Path) -> std::io::Result<Self> {
+        let builder = TokenizerBuilder::from_file(file_path).map_err(invalid_data)?;
+        let tokenizer = builder.build().map_err(invalid_data)?;
+        Ok(Self {
+            tokenizer,
+            token: Token::default(),
+        })
+    }
+
+    pub fn from_segmenter(segmenter: lindera::segmenter::Segmenter) -> Self {
+        Self {
+            tokenizer: LinderaCoreTokenizer::new(segmenter),
+            token: Token::default(),
+        }
+    }
+
+    pub fn analyzer(self) -> TextAnalyzer {
+        TextAnalyzer::builder(self).build()
+    }
+
+    pub fn analyzer_builder(self) -> TextAnalyzerBuilder {
+        TextAnalyzer::builder(self).dynamic()
+    }
+}
+
+pub struct LinderaTokenStream<'a> {
+    tokens: Vec<LinderaToken<'a>>,
+    token: &'a mut Token,
+}
+
+impl<'a> TokenStream for LinderaTokenStream<'a> {
+    fn advance(&mut self) -> bool {
+        if self.tokens.is_empty() {
+            return false;
+        }
+        let token = self.tokens.remove(0);
+        self.token.text = token.text.to_string();
+        self.token.offset_from = token.byte_start;
+        self.token.offset_to = token.byte_end;
+        self.token.position = token.position;
+        self.token.position_length = token.position_length;
+        true
+    }
+
+    fn token(&self) -> &Token {
+        self.token
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.token
+    }
+}
+
+impl Tokenizer for LinderaTokenizer {
+    type TokenStream<'a> = LinderaTokenStream<'a>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        self.token.reset();
+        LinderaTokenStream {
+            tokens: self.tokenizer.tokenize(text).unwrap_or_default(),
+            token: &mut self.token,
+        }
+    }
+}
+
+fn invalid_data(err: impl std::fmt::Debug) -> std::io::Error {
+    std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{err:?}"))
+}
diff --git a/rust/lance-tokenizer/src/lower_caser.rs b/rust/lance-tokenizer/src/lower_caser.rs
new file mode 100644
index 00000000000..a041ac04e1f
--- /dev/null
+++ b/rust/lance-tokenizer/src/lower_caser.rs
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+// SPDX-License-Identifier: MIT
+// Adapted from Tantivy v0.24.2 lower caser.
+// Copyright (c) 2017-present Tantivy contributors.
+
+use std::mem;
+
+use crate::{Token, TokenFilter, TokenStream, Tokenizer};
+
+#[derive(Clone)]
+pub struct LowerCaser;
+
+impl TokenFilter for LowerCaser {
+    type Tokenizer<T: Tokenizer> = LowerCaserFilter<T>;
+
+    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T> {
+        LowerCaserFilter {
+            tokenizer,
+            buffer: String::new(),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct LowerCaserFilter<T> {
+    tokenizer: T,
+    buffer: String,
+}
+
+impl<T: Tokenizer> Tokenizer for LowerCaserFilter<T> {
+    type TokenStream<'a> = LowerCaserTokenStream<'a, T::TokenStream<'a>>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        self.buffer.clear();
+        LowerCaserTokenStream {
+            buffer: &mut self.buffer,
+            tail: self.tokenizer.token_stream(text),
+        }
+    }
+}
+
+pub struct LowerCaserTokenStream<'a, T> {
+    buffer: &'a mut String,
+    tail: T,
+}
+
+fn to_lowercase_unicode(text: &str, output: &mut String) {
+    output.clear();
+    output.reserve(50);
+    for ch in text.chars() {
+        output.extend(ch.to_lowercase());
+    }
+}
+
+impl<T: TokenStream> TokenStream for LowerCaserTokenStream<'_, T> {
+    fn advance(&mut self) -> bool {
+        if !self.tail.advance() {
+            return false;
+        }
+        if self.token_mut().text.is_ascii() {
+            self.token_mut().text.make_ascii_lowercase();
+        } else {
+            to_lowercase_unicode(&self.tail.token().text, self.buffer);
+            mem::swap(&mut self.tail.token_mut().text, self.buffer);
+        }
+        true
+    }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+}
diff --git a/rust/lance-tokenizer/src/ngram_tokenizer.rs b/rust/lance-tokenizer/src/ngram_tokenizer.rs
new file mode 100644
index 00000000000..1cfdd052977
--- /dev/null
+++ b/rust/lance-tokenizer/src/ngram_tokenizer.rs
@@ -0,0 +1,221 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+// SPDX-License-Identifier: MIT
+// Adapted from Tantivy v0.24.2 ngram tokenizer.
+// Copyright (c) 2017-present Tantivy contributors.
+
+use std::fmt::{Display, Formatter};
+
+use crate::{Token, TokenStream, Tokenizer};
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct NgramError {
+    message: String,
+}
+
+impl NgramError {
+    fn invalid_argument(message: impl Into<String>) -> Self {
+        Self {
+            message: message.into(),
+        }
+    }
+}
+
+impl Display for NgramError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str(&self.message)
+    }
+}
+
+impl std::error::Error for NgramError {}
+
+#[derive(Clone, Debug)]
+pub struct NgramTokenizer {
+    min_gram: usize,
+    max_gram: usize,
+    prefix_only: bool,
+    token: Token,
+}
+
+impl NgramTokenizer {
+    pub fn new(min_gram: usize, max_gram: usize, prefix_only: bool) -> Result<Self, NgramError> {
+        if min_gram == 0 {
+            return Err(NgramError::invalid_argument(
+                "min_gram must be greater than 0",
+            ));
+        }
+        if min_gram > max_gram {
+            return Err(NgramError::invalid_argument(
+                "min_gram must not be greater than max_gram",
+            ));
+        }
+        Ok(Self {
+            min_gram,
+            max_gram,
+            prefix_only,
+            token: Token::default(),
+        })
+    }
+
+    pub fn all_ngrams(min_gram: usize, max_gram: usize) -> Result<Self, NgramError> {
+        Self::new(min_gram, max_gram, false)
+    }
+
+    pub fn prefix_only(min_gram: usize, max_gram: usize) -> Result<Self, NgramError> {
+        Self::new(min_gram, max_gram, true)
+    }
+}
+
+pub struct NgramTokenStream<'a> {
+    ngram_charidx_iterator: StutteringIterator<CodepointFrontiers<'a>>,
+    prefix_only: bool,
+    text: &'a str,
+    token: &'a mut Token,
+}
+
+impl Tokenizer for NgramTokenizer {
+    type TokenStream<'a> = NgramTokenStream<'a>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        self.token.reset();
+        NgramTokenStream {
+            ngram_charidx_iterator: StutteringIterator::new(
+                CodepointFrontiers::for_str(text),
+                self.min_gram,
+                self.max_gram,
+            ),
+            prefix_only: self.prefix_only,
+            text,
+            token: &mut self.token,
+        }
+    }
+}
+
+impl TokenStream for NgramTokenStream<'_> {
+    fn advance(&mut self) -> bool {
+        if let Some((offset_from, offset_to)) = self.ngram_charidx_iterator.next() {
+            if self.prefix_only && offset_from > 0 {
+                return false;
+            }
+            self.token.position = 0;
+            self.token.offset_from = offset_from;
+            self.token.offset_to = offset_to;
+            self.token.text.clear();
+            self.token.text.push_str(&self.text[offset_from..offset_to]);
+            true
+        } else {
+            false
+        }
+    }
+
+    fn token(&self) -> &Token {
+        self.token
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.token
+    }
+}
+
+struct StutteringIterator<T> {
+    underlying: T,
+    min_gram: usize,
+    max_gram: usize,
+    memory: Vec<usize>,
+    cursor: usize,
+    gram_len: usize,
+}
+
+impl<T> StutteringIterator<T>
+where
+    T: Iterator<Item = usize>,
+{
+    fn new(mut underlying: T, min_gram: usize, max_gram: usize) -> Self {
+        debug_assert!(min_gram > 0, "min_gram must be positive");
+        let memory: Vec<usize> = (&mut underlying).take(max_gram + 1).collect();
+        if memory.len() <= min_gram {
+            Self {
+                underlying,
+                min_gram: 1,
+                max_gram: 0,
+                memory,
+                cursor: 0,
+                gram_len: 0,
+            }
+        } else {
+            Self {
+                underlying,
+                min_gram,
+                max_gram: memory.len() - 1,
+                memory,
+                cursor: 0,
+                gram_len: min_gram,
+            }
+        }
+    }
+}
+
+impl<T> Iterator for StutteringIterator<T>
+where
+    T: Iterator<Item = usize>,
+{
+    type Item = (usize, usize);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.gram_len > self.max_gram {
+            self.gram_len = self.min_gram;
+            if let Some(next_val) = self.underlying.next() {
+                self.memory[self.cursor] = next_val;
+            } else {
+                self.max_gram -= 1;
+            }
+            self.cursor += 1;
+            if self.cursor >= self.memory.len() {
+                self.cursor = 0;
+            }
+        }
+        if self.max_gram < self.min_gram {
+            return None;
+        }
+        let start = self.memory[self.cursor % self.memory.len()];
+        let stop = self.memory[(self.cursor + self.gram_len) % self.memory.len()];
+        self.gram_len += 1;
+        Some((start, stop))
+    }
+}
+
+struct CodepointFrontiers<'a> {
+    text: &'a str,
+    next_offset: Option<usize>,
+}
+
+impl<'a> CodepointFrontiers<'a> {
+    fn for_str(text: &'a str) -> Self {
+        Self {
+            text,
+            next_offset: Some(0),
+        }
+    }
+}
+
+impl Iterator for CodepointFrontiers<'_> {
+    type Item = usize;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let offset = self.next_offset?;
+        if self.text.is_empty() {
+            self.next_offset = None;
+        } else {
+            let width = utf8_codepoint_width(self.text.as_bytes()[0]);
+            self.text = &self.text[width..];
+            self.next_offset = Some(offset + width);
+        }
+        Some(offset)
+    }
+}
+
+const CODEPOINT_UTF8_WIDTH: [u8; 16] = [1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 4];
+
+fn utf8_codepoint_width(byte: u8) -> usize {
+    CODEPOINT_UTF8_WIDTH[(byte as usize) >> 4] as usize
+}
diff --git a/rust/lance-tokenizer/src/raw_tokenizer.rs b/rust/lance-tokenizer/src/raw_tokenizer.rs
new file mode 100644
index 00000000000..31a684e2a84
--- /dev/null
+++ b/rust/lance-tokenizer/src/raw_tokenizer.rs
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+// SPDX-License-Identifier: MIT
+// Adapted from Tantivy v0.24.2 raw tokenizer.
+// Copyright (c) 2017-present Tantivy contributors.
+
+use crate::{Token, TokenStream, Tokenizer};
+
+#[derive(Clone, Default)]
+pub struct RawTokenizer {
+    token: Token,
+}
+
+pub struct RawTokenStream<'a> {
+    token: &'a mut Token,
+    has_token: bool,
+}
+
+impl Tokenizer for RawTokenizer {
+    type TokenStream<'a> = RawTokenStream<'a>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        self.token.reset();
+        self.token.position = 0;
+        self.token.position_length = 1;
+        self.token.offset_from = 0;
+        self.token.offset_to = text.len();
+        self.token.text.clear();
+        self.token.text.push_str(text);
+        RawTokenStream {
+            token: &mut self.token,
+            has_token: true,
+        }
+    }
+}
+
+impl TokenStream for RawTokenStream<'_> {
+    fn advance(&mut self) -> bool {
+        let has_token = self.has_token;
+        self.has_token = false;
+        has_token
+    }
+
+    fn token(&self) -> &Token {
+        self.token
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.token
+    }
+}
diff --git a/rust/lance-tokenizer/src/remove_long.rs b/rust/lance-tokenizer/src/remove_long.rs
new file mode 100644
index 00000000000..4bc6e5e7887
--- /dev/null
+++ b/rust/lance-tokenizer/src/remove_long.rs
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+// SPDX-License-Identifier: MIT
+// Adapted from Tantivy v0.24.2 remove-long filter.
+// Copyright (c) 2017-present Tantivy contributors.
+
+use crate::{Token, TokenFilter, TokenStream, Tokenizer};
+
+#[derive(Clone)]
+pub struct RemoveLongFilter {
+    length_limit: usize,
+}
+
+impl RemoveLongFilter {
+    pub fn limit(length_limit: usize) -> Self {
+        Self { length_limit }
+    }
+}
+
+impl TokenFilter for RemoveLongFilter {
+    type Tokenizer<T: Tokenizer> = RemoveLongFilterWrapper<T>;
+
+    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T> {
+        RemoveLongFilterWrapper {
+            length_limit: self.length_limit,
+            inner: tokenizer,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct RemoveLongFilterWrapper<T: Tokenizer> {
+    length_limit: usize,
+    inner: T,
+}
+
+impl<T: Tokenizer> Tokenizer for RemoveLongFilterWrapper<T> {
+    type TokenStream<'a> = RemoveLongFilterStream<T::TokenStream<'a>>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        RemoveLongFilterStream {
+            token_length_limit: self.length_limit,
+            tail: self.inner.token_stream(text),
+        }
+    }
+}
+
+pub struct RemoveLongFilterStream<T> {
+    token_length_limit: usize,
+    tail: T,
+}
+
+impl<T> RemoveLongFilterStream<T> {
+    fn predicate(&self, token: &Token) -> bool {
+        token.text.len() < self.token_length_limit
+    }
+}
+
+impl<T: TokenStream> TokenStream for RemoveLongFilterStream<T> {
+    fn advance(&mut self) -> bool {
+        while self.tail.advance() {
+            if self.predicate(self.tail.token()) {
+                return true;
+            }
+        }
+        false
+    }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+}
diff --git a/rust/lance-tokenizer/src/simple_tokenizer.rs b/rust/lance-tokenizer/src/simple_tokenizer.rs
new file mode 100644
index 00000000000..e2be3dc5dbd
--- /dev/null
+++ b/rust/lance-tokenizer/src/simple_tokenizer.rs
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+// SPDX-License-Identifier: MIT
+// Adapted from Tantivy v0.24.2 simple tokenizer.
+// Copyright (c) 2017-present Tantivy contributors.
+
+use std::str::CharIndices;
+
+use crate::{Token, TokenStream, Tokenizer};
+
+#[derive(Clone, Default)]
+pub struct SimpleTokenizer {
+    token: Token,
+}
+
+pub struct SimpleTokenStream<'a> {
+    text: &'a str,
+    chars: CharIndices<'a>,
+    token: &'a mut Token,
+}
+
+impl Tokenizer for SimpleTokenizer {
+    type TokenStream<'a> = SimpleTokenStream<'a>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        self.token.reset();
+        SimpleTokenStream {
+            text,
+            chars: text.char_indices(),
+            token: &mut self.token,
+        }
+    }
+}
+
+impl SimpleTokenStream<'_> {
+    fn search_token_end(&mut self) -> usize {
+        (&mut self.chars)
+            .filter(|(_, c)| !c.is_alphanumeric())
+            .map(|(offset, _)| offset)
+            .next()
+            .unwrap_or(self.text.len())
+    }
+}
+
+impl TokenStream for SimpleTokenStream<'_> {
+    fn advance(&mut self) -> bool {
+        self.token.text.clear();
+        self.token.position = self.token.position.wrapping_add(1);
+        while let Some((offset_from, c)) = self.chars.next() {
+            if c.is_alphanumeric() {
+                let offset_to = self.search_token_end();
+                self.token.offset_from = offset_from;
+                self.token.offset_to = offset_to;
+                self.token.text.push_str(&self.text[offset_from..offset_to]);
+                return true;
+            }
+        }
+        false
+    }
+
+    fn token(&self) -> &Token {
+        self.token
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.token
+    }
+}
diff --git a/rust/lance-tokenizer/src/stemmer.rs b/rust/lance-tokenizer/src/stemmer.rs
new file mode 100644
index 00000000000..03fcf118019
--- /dev/null
+++ b/rust/lance-tokenizer/src/stemmer.rs
@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+// SPDX-License-Identifier: MIT
+// Adapted from Tantivy v0.24.2 stemmer filter.
+// Copyright (c) 2017-present Tantivy contributors.
+
+use std::borrow::Cow;
+use std::mem;
+
+use rust_stemmers::Algorithm;
+use serde::{Deserialize, Serialize};
+
+use crate::{Token, TokenFilter, TokenStream, Tokenizer};
+
+#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Copy, Clone)]
+pub enum Language {
+    Arabic,
+    Danish,
+    Dutch,
+    English,
+    Finnish,
+    French,
+    German,
+    Greek,
+    Hungarian,
+    Italian,
+    Norwegian,
+    Portuguese,
+    Romanian,
+    Russian,
+    Spanish,
+    Swedish,
+    Tamil,
+    Turkish,
+}
+
+impl Language {
+    fn algorithm(self) -> Algorithm {
+        match self {
+            Self::Arabic => Algorithm::Arabic,
+            Self::Danish => Algorithm::Danish,
+            Self::Dutch => Algorithm::Dutch,
+            Self::English => Algorithm::English,
+            Self::Finnish => Algorithm::Finnish,
+            Self::French => Algorithm::French,
+            Self::German => Algorithm::German,
+            Self::Greek => Algorithm::Greek,
+            Self::Hungarian => Algorithm::Hungarian,
+            Self::Italian => Algorithm::Italian,
+            Self::Norwegian => Algorithm::Norwegian,
+            Self::Portuguese => Algorithm::Portuguese,
+            Self::Romanian => Algorithm::Romanian,
+            Self::Russian => Algorithm::Russian,
+            Self::Spanish => Algorithm::Spanish,
+            Self::Swedish => Algorithm::Swedish,
+            Self::Tamil => Algorithm::Tamil,
+            Self::Turkish => Algorithm::Turkish,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct Stemmer {
+    stemmer_algorithm: Algorithm,
+}
+
+impl Stemmer {
+    pub fn new(language: Language) -> Self {
+        Self {
+            stemmer_algorithm: language.algorithm(),
+        }
+    }
+}
+
+impl Default for Stemmer {
+    fn default() -> Self {
+        Self::new(Language::English)
+    }
+}
+
+impl TokenFilter for Stemmer {
+    type Tokenizer<T: Tokenizer> = StemmerFilter<T>;
+
+    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T> {
+        StemmerFilter {
+            stemmer_algorithm: self.stemmer_algorithm,
+            inner: tokenizer,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct StemmerFilter<T> {
+    stemmer_algorithm: Algorithm,
+    inner: T,
+}
+
+impl<T: Tokenizer> Tokenizer for StemmerFilter<T> {
+    type TokenStream<'a> = StemmerTokenStream<T::TokenStream<'a>>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        StemmerTokenStream {
+            tail: self.inner.token_stream(text),
+            stemmer: rust_stemmers::Stemmer::create(self.stemmer_algorithm),
+            buffer: String::new(),
+        }
+    }
+}
+
+pub struct StemmerTokenStream<T> {
+    tail: T,
+    stemmer: rust_stemmers::Stemmer,
+    buffer: String,
+}
+
+impl<T: TokenStream> TokenStream for StemmerTokenStream<T> {
+    fn advance(&mut self) -> bool {
+        if !self.tail.advance() {
+            return false;
+        }
+        let token = self.tail.token_mut();
+        let stemmed = self.stemmer.stem(&token.text);
+        match stemmed {
+            Cow::Owned(stemmed) => token.text = stemmed,
+            Cow::Borrowed(stemmed) => {
+                self.buffer.clear();
+                self.buffer.push_str(stemmed);
+                mem::swap(&mut token.text, &mut self.buffer);
+            }
+        }
+        true
+    }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+}
diff --git a/rust/lance-tokenizer/src/stop_word_filter.rs b/rust/lance-tokenizer/src/stop_word_filter.rs
new file mode 100644
index 00000000000..0c49330a619
--- /dev/null
+++ b/rust/lance-tokenizer/src/stop_word_filter.rs
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+// SPDX-License-Identifier: MIT
+// Adapted from Tantivy v0.24.2 stop-word filter.
+// Copyright (c) 2017-present Tantivy contributors.
+
+#[path = "stop_word_filter/stopwords.rs"]
+mod stopwords;
+
+use std::collections::HashSet;
+use std::sync::Arc;
+
+use crate::{Language, Token, TokenFilter, TokenStream, Tokenizer};
+
+#[derive(Clone)]
+pub struct StopWordFilter {
+    words: Arc<HashSet<String>>,
+}
+
+impl StopWordFilter {
+    pub fn new(language: Language) -> Option<Self> {
+        let words = match language {
+            Language::Danish => stopwords::DANISH,
+            Language::Dutch => stopwords::DUTCH,
+            Language::English => &[
+                "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into",
+                "is", "it", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then",
+                "there", "these", "they", "this", "to", "was", "will", "with",
+            ],
+            Language::Finnish => stopwords::FINNISH,
+            Language::French => stopwords::FRENCH,
+            Language::German => stopwords::GERMAN,
+            Language::Hungarian => stopwords::HUNGARIAN,
+            Language::Italian => stopwords::ITALIAN,
+            Language::Norwegian => stopwords::NORWEGIAN,
+            Language::Portuguese => stopwords::PORTUGUESE,
+            Language::Russian => stopwords::RUSSIAN,
+            Language::Spanish => stopwords::SPANISH,
+            Language::Swedish => stopwords::SWEDISH,
+            _ => return None,
+        };
+        Some(Self::remove(words.iter().map(|word| (*word).to_owned())))
+    }
+
+    pub fn remove<W: IntoIterator<Item = String>>(words: W) -> Self {
+        Self {
+            words: Arc::new(words.into_iter().collect()),
+        }
+    }
+}
+
+impl TokenFilter for StopWordFilter {
+    type Tokenizer<T: Tokenizer> = StopWordFilterWrapper<T>;
+
+    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T> {
+        StopWordFilterWrapper {
+            words: self.words,
+            inner: tokenizer,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct StopWordFilterWrapper<T> {
+    words: Arc<HashSet<String>>,
+    inner: T,
+}
+
+impl<T: Tokenizer> Tokenizer for StopWordFilterWrapper<T> {
+    type TokenStream<'a> = StopWordFilterStream<T::TokenStream<'a>>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        StopWordFilterStream {
+            words: self.words.clone(),
+            tail: self.inner.token_stream(text),
+        }
+    }
+}
+
+pub struct StopWordFilterStream<T> {
+    words: Arc<HashSet<String>>,
+    tail: T,
+}
+
+impl<T> StopWordFilterStream<T> {
+    fn predicate(&self, token: &Token) -> bool {
+        !self.words.contains(&token.text)
+    }
+}
+
+impl<T: TokenStream> TokenStream for StopWordFilterStream<T> {
+    fn advance(&mut self) -> bool {
+        while self.tail.advance() {
+            if self.predicate(self.tail.token()) {
+                return true;
+            }
+        }
+        false
+    }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+}
diff --git a/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs b/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs
new file mode 100644
index 00000000000..2ac3f4a28aa
--- /dev/null
+++ b/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs
@@ -0,0 +1,1900 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+// SPDX-License-Identifier: BSD-3-Clause
+// Vendored stop-word lists from Tantivy v0.24.2 / Snowball.
+/*
+These stop word lists are from the Snowball project (https://snowballstem.org/)
+which carries the following copyright and license:
+
+Copyright (c) 2001, Dr Martin Porter
+Copyright (c) 2004,2005, Richard Boulton
+Copyright (c) 2013, Yoshiki Shibukawa
+Copyright (c) 2006,2007,2009,2010,2011,2014-2019, Olly Betts
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+  1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+  2. Redistributions in binary form must reproduce the above copyright notice,
+     this list of conditions and the following disclaimer in the documentation
+     and/or other materials provided with the distribution.
+  3. Neither the name of the Snowball project nor the names of its contributors
+     may be used to endorse or promote products derived from this software
+     without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+pub const DANISH: &[&str] = &[
+    "og", "i", "jeg", "det", "at", "en", "den", "til", "er", "som", "på", "de", "med", "han", "af",
+    "for", "ikke", "der", "var", "mig", "sig", "men", "et", "har", "om", "vi", "min", "havde",
+    "ham", "hun", "nu", "over", "da", "fra", "du", "ud", "sin", "dem", "os", "op", "man", "hans",
+    "hvor", "eller", "hvad", "skal", "selv", "her", "alle", "vil", "blev", "kunne", "ind", "når",
+    "være", "dog", "noget", "ville", "jo", "deres", "efter", "ned", "skulle", "denne", "end",
+    "dette", "mit", "også", "under", "have", "dig", "anden", "hende", "mine", "alt", "meget",
+    "sit", "sine", "vor", "mod", "disse", "hvis", "din", "nogle", "hos", "blive", "mange", "ad",
+    "bliver", "hendes", "været", "thi", "jer", "sådan",
+];
+
+pub const DUTCH: &[&str] = &[
+    "de", "en", "van", "ik", "te", "dat", "die", "in", "een", "hij", "het", "niet", "zijn", "is",
+    "was", "op", "aan", "met", "als", "voor", "had", "er", "maar", "om", "hem", "dan", "zou", "of",
+    "wat", "mijn", "men", "dit", "zo", "door", "over", "ze", "zich", "bij", "ook", "tot", "je",
+    "mij", "uit", "der", "daar", "haar", "naar", "heb", "hoe", "heeft", "hebben", "deze", "u",
+    "want", "nog", "zal", "me", "zij", "nu", "ge", "geen", "omdat", "iets", "worden", "toch", "al",
+    "waren", "veel", "meer", "doen", "toen", "moet", "ben", "zonder", "kan", "hun", "dus", "alles",
+    "onder", "ja", "eens", "hier", "wie", "werd", "altijd", "doch", "wordt", "wezen", "kunnen",
+    "ons", "zelf", "tegen", "na", "reeds", "wil", "kon", "niets", "uw", "iemand", "geweest",
+    "andere",
+];
+
+pub const FINNISH: &[&str] = &[
+    "olla",
+    "olen",
+    "olet",
+    "on",
+    "olemme",
+    "olette",
+    "ovat",
+    "ole",
+    "oli",
+    "olisi",
+    "olisit",
+    "olisin",
+    "olisimme",
+    "olisitte",
+    "olisivat",
+    "olit",
+    "olin",
+    "olimme",
+    "olitte",
+    "olivat",
+    "ollut",
+    "olleet",
+    "en",
+    "et",
+    "ei",
+    "emme",
+    "ette",
+    "eivät",
+    "minä",
+    "minun",
+    "minut",
+    "minua",
+    "minussa",
+    "minusta",
+    "minuun",
+    "minulla",
+    "minulta",
+    "minulle",
+    "sinä",
+    "sinun",
+    "sinut",
+    "sinua",
+    "sinussa",
+    "sinusta",
+    "sinuun",
+    "sinulla",
+    "sinulta",
+    "sinulle",
+    "hän",
+    "hänen",
+    "hänet",
+    "häntä",
+    "hänessä",
+    "hänestä",
+    "häneen",
+    "hänellä",
+    "häneltä",
+    "hänelle",
+    "me",
+    "meidän",
+    "meidät",
+    "meitä",
+    "meissä",
+    "meistä",
+    "meihin",
+    "meillä",
+    "meiltä",
+    "meille",
+    "te",
+    "teidän",
+    "teidät",
+    "teitä",
+    "teissä",
+    "teistä",
+    "teihin",
+    "teillä",
+    "teiltä",
+    "teille",
+    "he",
+    "heidän",
+    "heidät",
+    "heitä",
+    "heissä",
+    "heistä",
+    "heihin",
+    "heillä",
+    "heiltä",
+    "heille",
+    "tämä",
+    "tämän",
+    "tätä",
+    "tässä",
+    "tästä",
+    "tähän",
+    "tällä",
+    "tältä",
+    "tälle",
+    "tänä",
+    "täksi",
+    "tuo",
+    "tuon",
+    "tuota",
+    "tuossa",
+    "tuosta",
+    "tuohon",
+    "tuolla",
+    "tuolta",
+    "tuolle",
+    "tuona",
+    "tuoksi",
+    "se",
+    "sen",
+    "sitä",
+    "siinä",
+    "siitä",
+    "siihen",
+    "sillä",
+    "siltä",
+    "sille",
+    "sinä",
+    "siksi",
+    "nämä",
+    "näiden",
+    "näitä",
+    "näissä",
+    "näistä",
+    "näihin",
+    "näillä",
+    "näiltä",
+    "näille",
+    "näinä",
+    "näiksi",
+    "nuo",
+    "noiden",
+    "noita",
+    "noissa",
+    "noista",
+    "noihin",
+    "noilla",
+    "noilta",
+    "noille",
+    "noina",
+    "noiksi",
+    "ne",
+    "niiden",
+    "niitä",
+    "niissä",
+    "niistä",
+    "niihin",
+    "niillä",
+    "niiltä",
+    "niille",
+    "niinä",
+    "niiksi",
+    "kuka",
+    "kenen",
+    "kenet",
+    "ketä",
+    "kenessä",
+    "kenestä",
+    "keneen",
+    "kenellä",
+    "keneltä",
+    "kenelle",
+    "kenenä",
+    "keneksi",
+    "ketkä",
+    "keiden",
+    "ketkä",
+    "keitä",
+    "keissä",
+    "keistä",
+    "keihin",
+    "keillä",
+    "keiltä",
+    "keille",
+    "keinä",
+    "keiksi",
+    "mikä",
+    "minkä",
+    "minkä",
+    "mitä",
+    "missä",
+    "mistä",
+    "mihin",
+    "millä",
+    "miltä",
+    "mille",
+    "minä",
+    "miksi",
+    "mitkä",
+    "joka",
+    "jonka",
+    "jota",
+    "jossa",
+    "josta",
+    "johon",
+    "jolla",
+    "jolta",
+    "jolle",
+    "jona",
+    "joksi",
+    "jotka",
+    "joiden",
+    "joita",
+    "joissa",
+    "joista",
+    "joihin",
+    "joilla",
+    "joilta",
+    "joille",
+    "joina",
+    "joiksi",
+    "että",
+    "ja",
+    "jos",
+    "koska",
+    "kuin",
+    "mutta",
+    "niin",
+    "sekä",
+    "sillä",
+    "tai",
+    "vaan",
+    "vai",
+    "vaikka",
+    "kanssa",
+    "mukaan",
+    "noin",
+    "poikki",
+    "yli",
+    "kun",
+    "nyt",
+    "itse",
+];
+
+pub const FRENCH: &[&str] = &[
+    "au", "aux", "avec", "ce", "ces", "dans", "de", "des", "du", "elle", "en", "et", "eux", "il",
+    "je", "la", "le", "leur", "lui", "ma", "mais", "me", "même", "mes", "moi", "mon", "ne", "nos",
+    "notre", "nous", "on", "ou", "par", "pas", "pour", "qu", "que", "qui", "sa", "se", "ses",
+    "sur", "ta", "te", "tes", "toi", "ton", "tu", "un", "une", "vos", "votre", "vous", "c", "d",
+    "j", "l", "à", "m", "n", "s", "t", "y", "étée", "étées", "étant", "suis", "es", "êtes", "sont",
+    "serai", "seras", "sera", "serons", "serez", "seront", "serais", "serait", "serions", "seriez",
+    "seraient", "étais", "était", "étions", "étiez", "étaient", "fus", "fut", "fûmes", "fûtes",
+    "furent", "sois", "soit", "soyons", "soyez", "soient", "fusse", "fusses", "fussions",
+    "fussiez", "fussent", "ayant", "eu", "eue", "eues", "eus", "ai", "avons", "avez", "ont",
+    "aurai", "aurons", "aurez", "auront", "aurais", "aurait", "aurions", "auriez", "auraient",
+    "avais", "avait", "aviez", "avaient", "eut", "eûmes", "eûtes", "eurent", "aie", "aies", "ait",
+    "ayons", "ayez", "aient", "eusse", "eusses", "eût", "eussions", "eussiez", "eussent", "ceci",
+    "cela", "celà", "cet", "cette", "ici", "ils", "les", "leurs", "quel", "quels", "quelle",
+    "quelles", "sans", "soi",
+];
+
+pub const GERMAN: &[&str] = &[
+    "aber",
+    "alle",
+    "allem",
+    "allen",
+    "aller",
+    "alles",
+    "als",
+    "also",
+    "am",
+    "an",
+    "ander",
+    "andere",
+    "anderem",
+    "anderen",
+    "anderer",
+    "anderes",
+    "anderm",
+    "andern",
+    "anderr",
+    "anders",
+    "auch",
+    "auf",
+    "aus",
+    "bei",
+    "bin",
+    "bis",
+    "bist",
+    "da",
+    "damit",
+    "dann",
+    "der",
+    "den",
+    "des",
+    "dem",
+    "die",
+    "das",
+    "daß",
+    "derselbe",
+    "derselben",
+    "denselben",
+    "desselben",
+    "demselben",
+    "dieselbe",
+    "dieselben",
+    "dasselbe",
+    "dazu",
+    "dein",
+    "deine",
+    "deinem",
+    "deinen",
+    "deiner",
+    "deines",
+    "denn",
+    "derer",
+    "dessen",
+    "dich",
+    "dir",
+    "du",
+    "dies",
+    "diese",
+    "diesem",
+    "diesen",
+    "dieser",
+    "dieses",
+    "doch",
+    "dort",
+    "durch",
+    "ein",
+    "eine",
+    "einem",
+    "einen",
+    "einer",
+    "eines",
+    "einig",
+    "einige",
+    "einigem",
+    "einigen",
+    "einiger",
+    "einiges",
+    "einmal",
+    "er",
+    "ihn",
+    "ihm",
+    "es",
+    "etwas",
+    "euer",
+    "eure",
+    "eurem",
+    "euren",
+    "eurer",
+    "eures",
+    "für",
+    "gegen",
+    "gewesen",
+    "hab",
+    "habe",
+    "haben",
+    "hat",
+    "hatte",
+    "hatten",
+    "hier",
+    "hin",
+    "hinter",
+    "ich",
+    "mich",
+    "mir",
+    "ihr",
+    "ihre",
+    "ihrem",
+    "ihren",
+    "ihrer",
+    "ihres",
+    "euch",
+    "im",
+    "in",
+    "indem",
+    "ins",
+    "ist",
+    "jede",
+    "jedem",
+    "jeden",
+    "jeder",
+    "jedes",
+    "jene",
+    "jenem",
+    "jenen",
+    "jener",
+    "jenes",
+    "jetzt",
+    "kann",
+    "kein",
+    "keine",
+    "keinem",
+    "keinen",
+    "keiner",
+    "keines",
+    "können",
+    "könnte",
+    "machen",
+    "man",
+    "manche",
+    "manchem",
+    "manchen",
+    "mancher",
+    "manches",
+    "mein",
+    "meine",
+    "meinem",
+    "meinen",
+    "meiner",
+    "meines",
+    "mit",
+    "muss",
+    "musste",
+    "nach",
+    "nicht",
+    "nichts",
+    "noch",
+    "nun",
+    "nur",
+    "ob",
+    "oder",
+    "ohne",
+    "sehr",
+    "sein",
+    "seine",
+    "seinem",
+    "seinen",
+    "seiner",
+    "seines",
+    "selbst",
+    "sich",
+    "sie",
+    "ihnen",
+    "sind",
+    "so",
+    "solche",
+    "solchem",
+    "solchen",
+    "solcher",
+    "solches",
+    "soll",
+    "sollte",
+    "sondern",
+    "sonst",
+    "über",
+    "um",
+    "und",
+    "uns",
+    "unse",
+    "unsem",
+    "unsen",
+    "unser",
+    "unses",
+    "unter",
+    "viel",
+    "vom",
+    "von",
+    "vor",
+    "während",
+    "war",
+    "waren",
+    "warst",
+    "was",
+    "weg",
+    "weil",
+    "weiter",
+    "welche",
+    "welchem",
+    "welchen",
+    "welcher",
+    "welches",
+    "wenn",
+    "werde",
+    "werden",
+    "wie",
+    "wieder",
+    "will",
+    "wir",
+    "wird",
+    "wirst",
+    "wo",
+    "wollen",
+    "wollte",
+    "würde",
+    "würden",
+    "zu",
+    "zum",
+    "zur",
+    "zwar",
+    "zwischen",
+];
+
+pub const HUNGARIAN: &[&str] = &[
+    "a",
+    "ahogy",
+    "ahol",
+    "aki",
+    "akik",
+    "akkor",
+    "alatt",
+    "által",
+    "általában",
+    "amely",
+    "amelyek",
+    "amelyekben",
+    "amelyeket",
+    "amelyet",
+    "amelynek",
+    "ami",
+    "amit",
+    "amolyan",
+    "amíg",
+    "amikor",
+    "át",
+    "abban",
+    "ahhoz",
+    "annak",
+    "arra",
+    "arról",
+    "az",
+    "azok",
+    "azon",
+    "azt",
+    "azzal",
+    "azért",
+    "aztán",
+    "azután",
+    "azonban",
+    "bár",
+    "be",
+    "belül",
+    "benne",
+    "cikk",
+    "cikkek",
+    "cikkeket",
+    "csak",
+    "de",
+    "e",
+    "eddig",
+    "egész",
+    "egy",
+    "egyes",
+    "egyetlen",
+    "egyéb",
+    "egyik",
+    "egyre",
+    "ekkor",
+    "el",
+    "elég",
+    "ellen",
+    "elő",
+    "először",
+    "előtt",
+    "első",
+    "én",
+    "éppen",
+    "ebben",
+    "ehhez",
+    "emilyen",
+    "ennek",
+    "erre",
+    "ez",
+    "ezt",
+    "ezek",
+    "ezen",
+    "ezzel",
+    "ezért",
+    "és",
+    "fel",
+    "felé",
+    "hanem",
+    "hiszen",
+    "hogy",
+    "hogyan",
+    "igen",
+    "így",
+    "illetve",
+    "ill.",
+    "ill",
+    "ilyen",
+    "ilyenkor",
+    "ison",
+    "ismét",
+    "itt",
+    "jó",
+    "jól",
+    "jobban",
+    "kell",
+    "kellett",
+    "keresztül",
+    "keressünk",
+    "ki",
+    "kívül",
+    "között",
+    "közül",
+    "legalább",
+    "lehet",
+    "lehetett",
+    "legyen",
+    "lenne",
+    "lenni",
+    "lesz",
+    "lett",
+    "maga",
+    "magát",
+    "majd",
+    "majd",
+    "már",
+    "más",
+    "másik",
+    "meg",
+    "még",
+    "mellett",
+    "mert",
+    "mely",
+    "melyek",
+    "mi",
+    "mit",
+    "míg",
+    "miért",
+    "milyen",
+    "mikor",
+    "minden",
+    "mindent",
+    "mindenki",
+    "mindig",
+    "mint",
+    "mintha",
+    "mivel",
+    "most",
+    "nagy",
+    "nagyobb",
+    "nagyon",
+    "ne",
+    "néha",
+    "nekem",
+    "neki",
+    "nem",
+    "néhány",
+    "nélkül",
+    "nincs",
+    "olyan",
+    "ott",
+    "össze",
+    "ő",
+    "ők",
+    "őket",
+    "pedig",
+    "persze",
+    "rá",
+    "s",
+    "saját",
+    "sem",
+    "semmi",
+    "sok",
+    "sokat",
+    "sokkal",
+    "számára",
+    "szemben",
+    "szerint",
+    "szinte",
+    "talán",
+    "tehát",
+    "teljes",
+    "tovább",
+    "továbbá",
+    "több",
+    "úgy",
+    "ugyanis",
+    "új",
+    "újabb",
+    "újra",
+    "után",
+    "utána",
+    "utolsó",
+    "vagy",
+    "vagyis",
+    "valaki",
+    "valami",
+    "valamint",
+    "való",
+    "vagyok",
+    "van",
+    "vannak",
+    "volt",
+    "voltam",
+    "voltak",
+    "voltunk",
+    "vissza",
+    "vele",
+    "viszont",
+    "volna",
+];
+
+pub const ITALIAN: &[&str] = &[
+    "ad",
+    "al",
+    "allo",
+    "ai",
+    "agli",
+    "all",
+    "agl",
+    "alla",
+    "alle",
+    "con",
+    "col",
+    "coi",
+    "da",
+    "dal",
+    "dallo",
+    "dai",
+    "dagli",
+    "dall",
+    "dagl",
+    "dalla",
+    "dalle",
+    "di",
+    "del",
+    "dello",
+    "dei",
+    "degli",
+    "dell",
+    "degl",
+    "della",
+    "delle",
+    "in",
+    "nel",
+    "nello",
+    "nei",
+    "negli",
+    "nell",
+    "negl",
+    "nella",
+    "nelle",
+    "su",
+    "sul",
+    "sullo",
+    "sui",
+    "sugli",
+    "sull",
+    "sugl",
+    "sulla",
+    "sulle",
+    "per",
+    "tra",
+    "contro",
+    "io",
+    "tu",
+    "lui",
+    "lei",
+    "noi",
+    "voi",
+    "loro",
+    "mio",
+    "mia",
+    "miei",
+    "mie",
+    "tuo",
+    "tua",
+    "tuoi",
+    "tue",
+    "suo",
+    "sua",
+    "suoi",
+    "sue",
+    "nostro",
+    "nostra",
+    "nostri",
+    "nostre",
+    "vostro",
+    "vostra",
+    "vostri",
+    "vostre",
+    "mi",
+    "ti",
+    "ci",
+    "vi",
+    "lo",
+    "la",
+    "li",
+    "le",
+    "gli",
+    "ne",
+    "il",
+    "un",
+    "uno",
+    "una",
+    "ma",
+    "ed",
+    "se",
+    "perché",
+    "anche",
+    "come",
+    "dov",
+    "dove",
+    "che",
+    "chi",
+    "cui",
+    "non",
+    "più",
+    "quale",
+    "quanto",
+    "quanti",
+    "quanta",
+    "quante",
+    "quello",
+    "quelli",
+    "quella",
+    "quelle",
+    "questo",
+    "questi",
+    "questa",
+    "queste",
+    "si",
+    "tutto",
+    "tutti",
+    "a",
+    "c",
+    "e",
+    "i",
+    "l",
+    "o",
+    "ho",
+    "hai",
+    "ha",
+    "abbiamo",
+    "avete",
+    "hanno",
+    "abbia",
+    "abbiate",
+    "abbiano",
+    "avrò",
+    "avrai",
+    "avrà",
+    "avremo",
+    "avrete",
+    "avranno",
+    "avrei",
+    "avresti",
+    "avrebbe",
+    "avremmo",
+    "avreste",
+    "avrebbero",
+    "avevo",
+    "avevi",
+    "aveva",
+    "avevamo",
+    "avevate",
+    "avevano",
+    "ebbi",
+    "avesti",
+    "ebbe",
+    "avemmo",
+    "aveste",
+    "ebbero",
+    "avessi",
+    "avesse",
+    "avessimo",
+    "avessero",
+    "avendo",
+    "avuto",
+    "avuta",
+    "avuti",
+    "avute",
+    "sono",
+    "sei",
+    "è",
+    "siamo",
+    "siete",
+    "sia",
+    "siate",
+    "siano",
+    "sarò",
+    "sarai",
+    "sarà",
+    "saremo",
+    "sarete",
+    "saranno",
+    "sarei",
+    "saresti",
+    "sarebbe",
+    "saremmo",
+    "sareste",
+    "sarebbero",
+    "ero",
+    "eri",
+    "era",
+    "eravamo",
+    "eravate",
+    "erano",
+    "fui",
+    "fosti",
+    "fu",
+    "fummo",
+    "foste",
+    "furono",
+    "fossi",
+    "fosse",
+    "fossimo",
+    "fossero",
+    "essendo",
+    "faccio",
+    "fai",
+    "facciamo",
+    "fanno",
+    "faccia",
+    "facciate",
+    "facciano",
+    "farò",
+    "farai",
+    "farà",
+    "faremo",
+    "farete",
+    "faranno",
+    "farei",
+    "faresti",
+    "farebbe",
+    "faremmo",
+    "fareste",
+    "farebbero",
+    "facevo",
+    "facevi",
+    "faceva",
+    "facevamo",
+    "facevate",
+    "facevano",
+    "feci",
+    "facesti",
+    "fece",
+    "facemmo",
+    "faceste",
+    "fecero",
+    "facessi",
+    "facesse",
+    "facessimo",
+    "facessero",
+    "facendo",
+    "sto",
+    "stai",
+    "sta",
+    "stiamo",
+    "stanno",
+    "stia",
+    "stiate",
+    "stiano",
+    "starò",
+    "starai",
+    "starà",
+    "staremo",
+    "starete",
+    "staranno",
+    "starei",
+    "staresti",
+    "starebbe",
+    "staremmo",
+    "stareste",
+    "starebbero",
+    "stavo",
+    "stavi",
+    "stava",
+    "stavamo",
+    "stavate",
+    "stavano",
+    "stetti",
+    "stesti",
+    "stette",
+    "stemmo",
+    "steste",
+    "stettero",
+    "stessi",
+    "stesse",
+    "stessimo",
+    "stessero",
+    "stando",
+];
+
+pub const NORWEGIAN: &[&str] = &[
+    "og",
+    "i",
+    "jeg",
+    "det",
+    "at",
+    "en",
+    "et",
+    "den",
+    "til",
+    "er",
+    "som",
+    "på",
+    "de",
+    "med",
+    "han",
+    "av",
+    "ikke",
+    "ikkje",
+    "der",
+    "så",
+    "var",
+    "meg",
+    "seg",
+    "men",
+    "ett",
+    "har",
+    "om",
+    "vi",
+    "min",
+    "mitt",
+    "ha",
+    "hadde",
+    "hun",
+    "nå",
+    "over",
+    "da",
+    "ved",
+    "fra",
+    "du",
+    "ut",
+    "sin",
+    "dem",
+    "oss",
+    "opp",
+    "man",
+    "kan",
+    "hans",
+    "hvor",
+    "eller",
+    "hva",
+    "skal",
+    "selv",
+    "sjøl",
+    "her",
+    "alle",
+    "vil",
+    "bli",
+    "ble",
+    "blei",
+    "blitt",
+    "kunne",
+    "inn",
+    "når",
+    "være",
+    "kom",
+    "noen",
+    "noe",
+    "ville",
+    "dere",
+    "deres",
+    "kun",
+    "ja",
+    "etter",
+    "ned",
+    "skulle",
+    "denne",
+    "for",
+    "deg",
+    "si",
+    "sine",
+    "sitt",
+    "mot",
+    "å",
+    "meget",
+    "hvorfor",
+    "dette",
+    "disse",
+    "uten",
+    "hvordan",
+    "ingen",
+    "din",
+    "ditt",
+    "blir",
+    "samme",
+    "hvilken",
+    "hvilke",
+    "sånn",
+    "inni",
+    "mellom",
+    "vår",
+    "hver",
+    "hvem",
+    "vors",
+    "hvis",
+    "både",
+    "bare",
+    "enn",
+    "fordi",
+    "før",
+    "mange",
+    "også",
+    "slik",
+    "vært",
+    "båe",
+    "begge",
+    "siden",
+    "dykk",
+    "dykkar",
+    "dei",
+    "deira",
+    "deires",
+    "deim",
+    "di",
+    "då",
+    "eg",
+    "ein",
+    "eit",
+    "eitt",
+    "elles",
+    "honom",
+    "hjå",
+    "ho",
+    "hoe",
+    "henne",
+    "hennar",
+    "hennes",
+    "hoss",
+    "hossen",
+    "ingi",
+    "inkje",
+    "korleis",
+    "korso",
+    "kva",
+    "kvar",
+    "kvarhelst",
+    "kven",
+    "kvi",
+    "kvifor",
+    "me",
+    "medan",
+    "mi",
+    "mine",
+    "mykje",
+    "no",
+    "nokon",
+    "noka",
+    "nokor",
+    "noko",
+    "nokre",
+    "sia",
+    "sidan",
+    "so",
+    "somt",
+    "somme",
+    "um",
+    "upp",
+    "vere",
+    "vore",
+    "verte",
+    "vort",
+    "varte",
+    "vart",
+];
+
+pub const PORTUGUESE: &[&str] = &[
+    "de",
+    "a",
+    "o",
+    "que",
+    "e",
+    "do",
+    "da",
+    "em",
+    "um",
+    "para",
+    "com",
+    "não",
+    "uma",
+    "os",
+    "no",
+    "se",
+    "na",
+    "por",
+    "mais",
+    "as",
+    "dos",
+    "como",
+    "mas",
+    "ao",
+    "ele",
+    "das",
+    "à",
+    "seu",
+    "sua",
+    "ou",
+    "quando",
+    "muito",
+    "nos",
+    "já",
+    "eu",
+    "também",
+    "só",
+    "pelo",
+    "pela",
+    "até",
+    "isso",
+    "ela",
+    "entre",
+    "depois",
+    "sem",
+    "mesmo",
+    "aos",
+    "seus",
+    "quem",
+    "nas",
+    "me",
+    "esse",
+    "eles",
+    "você",
+    "essa",
+    "num",
+    "nem",
+    "suas",
+    "meu",
+    "às",
+    "minha",
+    "numa",
+    "pelos",
+    "elas",
+    "qual",
+    "nós",
+    "lhe",
+    "deles",
+    "essas",
+    "esses",
+    "pelas",
+    "este",
+    "dele",
+    "tu",
+    "te",
+    "vocês",
+    "vos",
+    "lhes",
+    "meus",
+    "minhas",
+    "teu",
+    "tua",
+    "teus",
+    "tuas",
+    "nosso",
+    "nossa",
+    "nossos",
+    "nossas",
+    "dela",
+    "delas",
+    "esta",
+    "estes",
+    "estas",
+    "aquele",
+    "aquela",
+    "aqueles",
+    "aquelas",
+    "isto",
+    "aquilo",
+    "estou",
+    "está",
+    "estamos",
+    "estão",
+    "estive",
+    "esteve",
+    "estivemos",
+    "estiveram",
+    "estava",
+    "estávamos",
+    "estavam",
+    "estivera",
+    "estivéramos",
+    "esteja",
+    "estejamos",
+    "estejam",
+    "estivesse",
+    "estivéssemos",
+    "estivessem",
+    "estiver",
+    "estivermos",
+    "estiverem",
+    "hei",
+    "há",
+    "havemos",
+    "hão",
+    "houve",
+    "houvemos",
+    "houveram",
+    "houvera",
+    "houvéramos",
+    "haja",
+    "hajamos",
+    "hajam",
+    "houvesse",
+    "houvéssemos",
+    "houvessem",
+    "houver",
+    "houvermos",
+    "houverem",
+    "houverei",
+    "houverá",
+    "houveremos",
+    "houverão",
+    "houveria",
+    "houveríamos",
+    "houveriam",
+    "sou",
+    "somos",
+    "são",
+    "era",
+    "éramos",
+    "eram",
+    "fui",
+    "foi",
+    "fomos",
+    "foram",
+    "fora",
+    "fôramos",
+    "seja",
+    "sejamos",
+    "sejam",
+    "fosse",
+    "fôssemos",
+    "fossem",
+    "for",
+    "formos",
+    "forem",
+    "serei",
+    "será",
+    "seremos",
+    "serão",
+    "seria",
+    "seríamos",
+    "seriam",
+    "tenho",
+    "tem",
+    "temos",
+    "tém",
+    "tinha",
+    "tínhamos",
+    "tinham",
+    "tive",
+    "teve",
+    "tivemos",
+    "tiveram",
+    "tivera",
+    "tivéramos",
+    "tenha",
+    "tenhamos",
+    "tenham",
+    "tivesse",
+    "tivéssemos",
+    "tivessem",
+    "tiver",
+    "tivermos",
+    "tiverem",
+    "terei",
+    "terá",
+    "teremos",
+    "terão",
+    "teria",
+    "teríamos",
+    "teriam",
+];
+
+pub const RUSSIAN: &[&str] = &[
+    "и",
+    "в",
+    "во",
+    "не",
+    "что",
+    "он",
+    "на",
+    "я",
+    "с",
+    "со",
+    "как",
+    "а",
+    "то",
+    "все",
+    "она",
+    "так",
+    "его",
+    "но",
+    "да",
+    "ты",
+    "к",
+    "у",
+    "же",
+    "вы",
+    "за",
+    "бы",
+    "по",
+    "только",
+    "ее",
+    "мне",
+    "было",
+    "вот",
+    "от",
+    "меня",
+    "еще",
+    "нет",
+    "о",
+    "из",
+    "ему",
+    "теперь",
+    "когда",
+    "даже",
+    "ну",
+    "вдруг",
+    "ли",
+    "если",
+    "уже",
+    "или",
+    "ни",
+    "быть",
+    "был",
+    "него",
+    "до",
+    "вас",
+    "нибудь",
+    "опять",
+    "уж",
+    "вам",
+    "сказал",
+    "ведь",
+    "там",
+    "потом",
+    "себя",
+    "ничего",
+    "ей",
+    "может",
+    "они",
+    "тут",
+    "где",
+    "есть",
+    "надо",
+    "ней",
+    "для",
+    "мы",
+    "тебя",
+    "их",
+    "чем",
+    "была",
+    "сам",
+    "чтоб",
+    "без",
+    "будто",
+    "человек",
+    "чего",
+    "раз",
+    "тоже",
+    "себе",
+    "под",
+    "жизнь",
+    "будет",
+    "ж",
+    "тогда",
+    "кто",
+    "этот",
+    "говорил",
+    "того",
+    "потому",
+    "этого",
+    "какой",
+    "совсем",
+    "ним",
+    "здесь",
+    "этом",
+    "один",
+    "почти",
+    "мой",
+    "тем",
+    "чтобы",
+    "нее",
+    "кажется",
+    "сейчас",
+    "были",
+    "куда",
+    "зачем",
+    "сказать",
+    "всех",
+    "никогда",
+    "сегодня",
+    "можно",
+    "при",
+    "наконец",
+    "два",
+    "об",
+    "другой",
+    "хоть",
+    "после",
+    "над",
+    "больше",
+    "тот",
+    "через",
+    "эти",
+    "нас",
+    "про",
+    "всего",
+    "них",
+    "какая",
+    "много",
+    "разве",
+    "сказала",
+    "три",
+    "эту",
+    "моя",
+    "впрочем",
+    "хорошо",
+    "свою",
+    "этой",
+    "перед",
+    "иногда",
+    "лучше",
+    "чуть",
+    "том",
+    "нельзя",
+    "такой",
+    "им",
+    "более",
+    "всегда",
+    "конечно",
+    "всю",
+    "между",
+];
+
+pub const SPANISH: &[&str] = &[
+    "de",
+    "la",
+    "que",
+    "el",
+    "en",
+    "y",
+    "a",
+    "los",
+    "del",
+    "se",
+    "las",
+    "por",
+    "un",
+    "para",
+    "con",
+    "no",
+    "una",
+    "su",
+    "al",
+    "lo",
+    "como",
+    "más",
+    "pero",
+    "sus",
+    "le",
+    "ya",
+    "o",
+    "este",
+    "sí",
+    "porque",
+    "esta",
+    "entre",
+    "cuando",
+    "muy",
+    "sin",
+    "sobre",
+    "también",
+    "me",
+    "hasta",
+    "hay",
+    "donde",
+    "quien",
+    "desde",
+    "todo",
+    "nos",
+    "durante",
+    "todos",
+    "uno",
+    "les",
+    "ni",
+    "contra",
+    "otros",
+    "ese",
+    "eso",
+    "ante",
+    "ellos",
+    "e",
+    "esto",
+    "mí",
+    "antes",
+    "algunos",
+    "qué",
+    "unos",
+    "yo",
+    "otro",
+    "otras",
+    "otra",
+    "él",
+    "tanto",
+    "esa",
+    "estos",
+    "mucho",
+    "quienes",
+    "nada",
+    "muchos",
+    "cual",
+    "poco",
+    "ella",
+    "estar",
+    "estas",
+    "algunas",
+    "algo",
+    "nosotros",
+    "mi",
+    "mis",
+    "tú",
+    "te",
+    "ti",
+    "tu",
+    "tus",
+    "ellas",
+    "nosotras",
+    "vosotros",
+    "vosotras",
+    "os",
+    "mío",
+    "mía",
+    "míos",
+    "mías",
+    "tuyo",
+    "tuya",
+    "tuyos",
+    "tuyas",
+    "suyo",
+    "suya",
+    "suyos",
+    "suyas",
+    "nuestro",
+    "nuestra",
+    "nuestros",
+    "nuestras",
+    "vuestro",
+    "vuestra",
+    "vuestros",
+    "vuestras",
+    "esos",
+    "esas",
+    "estoy",
+    "estás",
+    "está",
+    "estamos",
+    "estáis",
+    "están",
+    "esté",
+    "estés",
+    "estemos",
+    "estéis",
+    "estén",
+    "estaré",
+    "estarás",
+    "estará",
+    "estaremos",
+    "estaréis",
+    "estarán",
+    "estaría",
+    "estarías",
+    "estaríamos",
+    "estaríais",
+    "estarían",
+    "estaba",
+    "estabas",
+    "estábamos",
+    "estabais",
+    "estaban",
+    "estuve",
+    "estuviste",
+    "estuvo",
+    "estuvimos",
+    "estuvisteis",
+    "estuvieron",
+    "estuviera",
+    "estuvieras",
+    "estuviéramos",
+    "estuvierais",
+    "estuvieran",
+    "estuviese",
+    "estuvieses",
+    "estuviésemos",
+    "estuvieseis",
+    "estuviesen",
+    "estando",
+    "estado",
+    "estada",
+    "estados",
+    "estadas",
+    "estad",
+    "he",
+    "has",
+    "ha",
+    "hemos",
+    "habéis",
+    "han",
+    "haya",
+    "hayas",
+    "hayamos",
+    "hayáis",
+    "hayan",
+    "habré",
+    "habrás",
+    "habrá",
+    "habremos",
+    "habréis",
+    "habrán",
+    "habría",
+    "habrías",
+    "habríamos",
+    "habríais",
+    "habrían",
+    "había",
+    "habías",
+    "habíamos",
+    "habíais",
+    "habían",
+    "hube",
+    "hubiste",
+    "hubo",
+    "hubimos",
+    "hubisteis",
+    "hubieron",
+    "hubiera",
+    "hubieras",
+    "hubiéramos",
+    "hubierais",
+    "hubieran",
+    "hubiese",
+    "hubieses",
+    "hubiésemos",
+    "hubieseis",
+    "hubiesen",
+    "habiendo",
+    "habido",
+    "habida",
+    "habidos",
+    "habidas",
+    "soy",
+    "eres",
+    "es",
+    "somos",
+    "sois",
+    "son",
+    "sea",
+    "seas",
+    "seamos",
+    "seáis",
+    "sean",
+    "seré",
+    "serás",
+    "será",
+    "seremos",
+    "seréis",
+    "serán",
+    "sería",
+    "serías",
+    "seríamos",
+    "seríais",
+    "serían",
+    "era",
+    "eras",
+    "éramos",
+    "erais",
+    "eran",
+    "fui",
+    "fuiste",
+    "fue",
+    "fuimos",
+    "fuisteis",
+    "fueron",
+    "fuera",
+    "fueras",
+    "fuéramos",
+    "fuerais",
+    "fueran",
+    "fuese",
+    "fueses",
+    "fuésemos",
+    "fueseis",
+    "fuesen",
+    "siendo",
+    "sido",
+    "tengo",
+    "tienes",
+    "tiene",
+    "tenemos",
+    "tenéis",
+    "tienen",
+    "tenga",
+    "tengas",
+    "tengamos",
+    "tengáis",
+    "tengan",
+    "tendré",
+    "tendrás",
+    "tendrá",
+    "tendremos",
+    "tendréis",
+    "tendrán",
+    "tendría",
+    "tendrías",
+    "tendríamos",
+    "tendríais",
+    "tendrían",
+    "tenía",
+    "tenías",
+    "teníamos",
+    "teníais",
+    "tenían",
+    "tuve",
+    "tuviste",
+    "tuvo",
+    "tuvimos",
+    "tuvisteis",
+    "tuvieron",
+    "tuviera",
+    "tuvieras",
+    "tuviéramos",
+    "tuvierais",
+    "tuvieran",
+    "tuviese",
+    "tuvieses",
+    "tuviésemos",
+    "tuvieseis",
+    "tuviesen",
+    "teniendo",
+    "tenido",
+    "tenida",
+    "tenidos",
+    "tenidas",
+    "tened",
+];
+
+pub const SWEDISH: &[&str] = &[
+    "och", "det", "att", "i", "en", "jag", "hon", "som", "han", "på", "den", "med", "var", "sig",
+    "för", "så", "till", "är", "men", "ett", "om", "hade", "de", "av", "icke", "mig", "du",
+    "henne", "då", "sin", "nu", "har", "inte", "hans", "honom", "skulle", "hennes", "där", "min",
+    "man", "ej", "vid", "kunde", "något", "från", "ut", "när", "efter", "upp", "vi", "dem", "vara",
+    "vad", "över", "än", "dig", "kan", "sina", "här", "ha", "mot", "alla", "under", "någon",
+    "eller", "allt", "mycket", "sedan", "ju", "denna", "själv", "detta", "åt", "utan", "varit",
+    "hur", "ingen", "mitt", "ni", "bli", "blev", "oss", "din", "dessa", "några", "deras", "blir",
+    "mina", "samma", "vilken", "er", "sådan", "vår", "blivit", "dess", "inom", "mellan", "sådant",
+    "varför", "varje", "vilka", "ditt", "vem", "vilket", "sitt", "sådana", "vart", "dina", "vars",
+    "vårt", "våra", "ert", "era", "vilkas",
+];
diff --git a/rust/lance-tokenizer/src/tokenizer_api.rs b/rust/lance-tokenizer/src/tokenizer_api.rs
new file mode 100644
index 00000000000..82dc781e798
--- /dev/null
+++ b/rust/lance-tokenizer/src/tokenizer_api.rs
@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+// SPDX-License-Identifier: MIT
+// Adapted from Tantivy v0.24.2 tokenizer API.
+// Copyright (c) 2017-present Tantivy contributors.
+
+use std::borrow::{Borrow, BorrowMut};
+use std::ops::{Deref, DerefMut};
+
+/// Token emitted by a tokenizer.
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct Token {
+    /// Byte offset of the first character.
+    pub offset_from: usize,
+    /// Byte offset after the last character.
+    pub offset_to: usize,
+    /// Logical token position.
+    pub position: usize,
+    /// Token text.
+    pub text: String,
+    /// Position length measured in original tokens.
+    pub position_length: usize,
+}
+
+impl Default for Token {
+    fn default() -> Self {
+        Self {
+            offset_from: 0,
+            offset_to: 0,
+            position: usize::MAX,
+            text: String::new(),
+            position_length: 1,
+        }
+    }
+}
+
+impl Token {
+    /// Reset the token to its default state.
+    pub fn reset(&mut self) {
+        self.offset_from = 0;
+        self.offset_to = 0;
+        self.position = usize::MAX;
+        self.text.clear();
+        self.position_length = 1;
+    }
+}
+
+/// Tokenizer splits text into a token stream.
+pub trait Tokenizer: 'static + Clone + Send + Sync {
+    /// Stream type emitted by the tokenizer.
+    type TokenStream<'a>: TokenStream;
+
+    /// Create a token stream for the provided text.
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a>;
+}
+
+/// Token stream object-safe wrapper.
+pub struct BoxTokenStream<'a>(Box<dyn TokenStream + 'a>);
+
+impl<'a> BoxTokenStream<'a> {
+    pub fn new<T: TokenStream + 'a>(token_stream: T) -> Self {
+        Self(Box::new(token_stream))
+    }
+}
+
+impl TokenStream for BoxTokenStream<'_> {
+    fn advance(&mut self) -> bool {
+        self.0.advance()
+    }
+
+    fn token(&self) -> &Token {
+        self.0.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.0.token_mut()
+    }
+}
+
+impl<'a> Deref for BoxTokenStream<'a> {
+    type Target = dyn TokenStream + 'a;
+
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}
+
+impl DerefMut for BoxTokenStream<'_> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut *self.0
+    }
+}
+
+impl<'a> TokenStream for Box<dyn TokenStream + 'a> {
+    fn advance(&mut self) -> bool {
+        let token_stream: &mut dyn TokenStream = self.borrow_mut();
+        token_stream.advance()
+    }
+
+    fn token(&self) -> &Token {
+        let token_stream: &(dyn TokenStream + 'a) = self.borrow();
+        token_stream.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        let token_stream: &mut (dyn TokenStream + 'a) = self.borrow_mut();
+        token_stream.token_mut()
+    }
+}
+
+/// Consumable token stream.
+pub trait TokenStream {
+    /// Advance to the next token.
+    fn advance(&mut self) -> bool;
+
+    /// Access the current token.
+    fn token(&self) -> &Token;
+
+    /// Mutate the current token.
+    fn token_mut(&mut self) -> &mut Token;
+
+    /// Iterate to the next token and return it.
+    fn next(&mut self) -> Option<&Token> {
+        if self.advance() {
+            Some(self.token())
+        } else {
+            None
+        }
+    }
+
+    /// Consume the remaining stream into the provided sink.
+    fn process(&mut self, sink: &mut dyn FnMut(&Token)) {
+        while self.advance() {
+            sink(self.token());
+        }
+    }
+}
+
+/// Filter that wraps a tokenizer with additional token-processing behavior.
+pub trait TokenFilter: 'static + Send + Sync {
+    /// Tokenizer produced by this filter.
+    type Tokenizer<T: Tokenizer>: Tokenizer;
+
+    /// Wrap the tokenizer.
+    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T>;
+}
diff --git a/rust/lance-tokenizer/src/whitespace_tokenizer.rs b/rust/lance-tokenizer/src/whitespace_tokenizer.rs
new file mode 100644
index 00000000000..80b793f2ed4
--- /dev/null
+++ b/rust/lance-tokenizer/src/whitespace_tokenizer.rs
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+// SPDX-License-Identifier: MIT
+// Adapted from Tantivy v0.24.2 whitespace tokenizer.
+// Copyright (c) 2017-present Tantivy contributors.
+
+use std::str::CharIndices;
+
+use crate::{Token, TokenStream, Tokenizer};
+
+#[derive(Clone, Default)]
+pub struct WhitespaceTokenizer {
+    token: Token,
+}
+
+pub struct WhitespaceTokenStream<'a> {
+    text: &'a str,
+    chars: CharIndices<'a>,
+    token: &'a mut Token,
+}
+
+impl Tokenizer for WhitespaceTokenizer {
+    type TokenStream<'a> = WhitespaceTokenStream<'a>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        self.token.reset();
+        WhitespaceTokenStream {
+            text,
+            chars: text.char_indices(),
+            token: &mut self.token,
+        }
+    }
+}
+
+impl WhitespaceTokenStream<'_> {
+    fn search_token_end(&mut self) -> usize {
+        (&mut self.chars)
+            .filter(|(_, c)| c.is_ascii_whitespace())
+            .map(|(offset, _)| offset)
+            .next()
+            .unwrap_or(self.text.len())
+    }
+}
+
+impl TokenStream for WhitespaceTokenStream<'_> {
+    fn advance(&mut self) -> bool {
+        self.token.text.clear();
+        self.token.position = self.token.position.wrapping_add(1);
+        while let Some((offset_from, c)) = self.chars.next() {
+            if !c.is_ascii_whitespace() {
+                let offset_to = self.search_token_end();
+                self.token.offset_from = offset_from;
+                self.token.offset_to = offset_to;
+                self.token.text.push_str(&self.text[offset_from..offset_to]);
+                return true;
+            }
+        }
+        false
+    }
+
+    fn token(&self) -> &Token {
+        self.token
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.token
+    }
+}
diff --git a/rust/lance/Cargo.toml b/rust/lance/Cargo.toml
index 02fb37d2cc0..f3dbb0ffd0a 100644
--- a/rust/lance/Cargo.toml
+++ b/rust/lance/Cargo.toml
@@ -25,11 +25,13 @@ lance-io = { workspace = true }
 lance-linalg = { workspace = true }
 lance-index = { workspace = true }
 lance-namespace = { workspace = true }
+lance-tokenizer = { workspace = true }
 lance-table = { workspace = true }
 lance-geo = { workspace = true, optional = true }
 arrow-arith = { workspace = true }
 arrow-array = { workspace = true }
 arrow-buffer = { workspace = true }
+arrow-cast = { workspace = true }
 arrow-ipc = { workspace = true }
 arrow-ord = { workspace = true }
 arrow-row = { workspace = true }
@@ -75,8 +77,7 @@ serde_json = { workspace = true }
 serde = { workspace = true }
 moka.workspace = true
 permutation = { version = "0.4.0" }
-tantivy.workspace = true
-aws-sdk-dynamodb = { workspace = true, optional = true }
+aws-sdk-dynamodb = { workspace = true, optional = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
 tracing.workspace = true
 humantime = { workspace = true }
 async_cell = "0.2.2"
@@ -84,6 +85,10 @@ semver.workspace = true
 tokio-stream = { workspace = true }
 tokio-util = { workspace = true }
 
+[build-dependencies]
+prost-build.workspace = true
+protobuf-src = { version = "2.1", optional = true }
+
 [target.'cfg(target_os = "linux")'.dev-dependencies]
 pprof.workspace = true
 # Need this so we can prevent dynamic linking in binaries (see cli feature)
@@ -97,6 +102,7 @@ libc = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 criterion = { workspace = true }
 approx.workspace = true
+rand_distr.workspace = true
 dirs = "5.0.0"
 all_asserts = "2.3.1"
 mock_instant.workspace = true
@@ -112,7 +118,7 @@ tracking-allocator = { version = "0.4", features = ["tracing-compat"] }
 paste = "1.0"
 # For S3 / DynamoDB tests
 aws-config = { workspace = true }
-aws-sdk-s3 = { workspace = true }
+aws-sdk-s3 = { workspace = true, default-features = false, features = ["default-https-client", "http-1x", "rt-tokio"] }
 geoarrow-array = { workspace = true }
 geoarrow-schema = { workspace = true }
 geo-types = { workspace = true }
@@ -127,6 +133,7 @@ dynamodb = ["lance-table/dynamodb", "dep:aws-sdk-dynamodb"]
 dynamodb_tests = ["dynamodb"]
 substrait = ["lance-datafusion/substrait"]
 protoc = [
+    "dep:protobuf-src",
     "lance-encoding/protoc",
     "lance-file/protoc",
     "lance-index/protoc",
@@ -182,6 +189,10 @@ harness = false
 name = "vector_throughput"
 harness = false
 
+[[bench]]
+name = "distributed_vector_build"
+harness = false
+
 [[bench]]
 name = "mem_wal_write"
 harness = false
diff --git a/rust/lance/benches/distributed_vector_build.rs b/rust/lance/benches/distributed_vector_build.rs
new file mode 100644
index 00000000000..768abd5aba2
--- /dev/null
+++ b/rust/lance/benches/distributed_vector_build.rs
@@ -0,0 +1,452 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use arrow_array::{ArrayRef, FixedSizeListArray, RecordBatch, RecordBatchIterator};
+use arrow_array::{cast::AsArray, types::Float32Type};
+use arrow_schema::{DataType, Field, FieldRef, Schema as ArrowSchema};
+use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
+#[cfg(target_os = "linux")]
+use pprof::criterion::{Output, PProfProfiler};
+use serde::Serialize;
+use uuid::Uuid;
+
+use lance::dataset::{Dataset, WriteMode, WriteParams};
+use lance::index::{DatasetIndexExt, vector::VectorIndexParams};
+use lance_arrow::FixedSizeListArrayExt;
+use lance_index::progress::noop_progress;
+use lance_index::vector::kmeans::{KMeansParams, train_kmeans};
+use lance_index::{
+    IndexType,
+    vector::{ivf::IvfBuildParams, pq::PQBuildParams},
+};
+use lance_linalg::distance::DistanceType;
+use lance_testing::datagen::generate_random_array;
+use tokio::runtime::Runtime;
+
+const NUM_FRAGMENTS: usize = 128;
+const ROWS_PER_FRAGMENT: usize = 1024;
+const DIM: i32 = 128;
+const NUM_SUB_VECTORS: usize = 16;
+const NUM_BITS: usize = 8;
+const MAX_ITERS: usize = 20;
+const SAMPLE_RATE: usize = 8;
+
+#[derive(Clone, Copy, Debug)]
+struct BenchCase {
+    num_shards: usize,
+    num_partitions: usize,
+}
+
+impl BenchCase {
+    fn label(&self) -> String {
+        format!(
+            "pq_shards_{}_partitions_{}",
+            self.num_shards, self.num_partitions
+        )
+    }
+}
+
+#[derive(Clone, Debug)]
+struct MergeFixture {
+    index_dir: PathBuf,
+    partial_aux_bytes: u64,
+    partial_dir_count: usize,
+}
+
+#[derive(Debug, Serialize)]
+struct CaseMetadata {
+    label: String,
+    num_shards: usize,
+    num_partitions: usize,
+    partial_dir_count: usize,
+    partial_aux_bytes: u64,
+    partial_aux_bytes_per_shard: u64,
+    total_rows: usize,
+    rows_per_shard: usize,
+}
+
+fn dataset_root() -> PathBuf {
+    std::env::temp_dir().join(format!(
+        "lance_bench_distributed_build_{}_{}_{}",
+        NUM_FRAGMENTS, ROWS_PER_FRAGMENT, DIM
+    ))
+}
+
+fn dataset_uri() -> String {
+    format!("file://{}", dataset_root().display())
+}
+
+fn workspace_root() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .ancestors()
+        .nth(2)
+        .unwrap()
+        .to_path_buf()
+}
+
+fn criterion_group_root() -> PathBuf {
+    workspace_root()
+        .join("target")
+        .join("criterion")
+        .join("distributed_merge_only_ivf_pq")
+}
+
+fn bench_cases() -> [BenchCase; 6] {
+    [
+        BenchCase {
+            num_shards: 8,
+            num_partitions: 256,
+        },
+        BenchCase {
+            num_shards: 32,
+            num_partitions: 256,
+        },
+        BenchCase {
+            num_shards: 128,
+            num_partitions: 256,
+        },
+        BenchCase {
+            num_shards: 8,
+            num_partitions: 1024,
+        },
+        BenchCase {
+            num_shards: 32,
+            num_partitions: 1024,
+        },
+        BenchCase {
+            num_shards: 128,
+            num_partitions: 1024,
+        },
+    ]
+}
+
+fn fixture_uuid(bench_case: BenchCase) -> Uuid {
+    Uuid::from_u128(
+        0x733a_0000_0000_0000_0000_0000_0000_0000
+            | ((bench_case.num_shards as u128) << 64)
+            | bench_case.num_partitions as u128,
+    )
+}
+
+fn working_uuid(bench_case: BenchCase) -> Uuid {
+    Uuid::from_u128(
+        0x733b_0000_0000_0000_0000_0000_0000_0000
+            | ((bench_case.num_shards as u128) << 64)
+            | bench_case.num_partitions as u128,
+    )
+}
+
+fn create_batches() -> (Arc<ArrowSchema>, Vec<RecordBatch>) {
+    let schema = Arc::new(ArrowSchema::new(vec![Field::new(
+        "vector",
+        DataType::FixedSizeList(
+            FieldRef::new(Field::new("item", DataType::Float32, true)),
+            DIM,
+        ),
+        false,
+    )]));
+
+    let batches = (0..NUM_FRAGMENTS)
+        .map(|_| {
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(
+                    FixedSizeListArray::try_new_from_values(
+                        generate_random_array(ROWS_PER_FRAGMENT * DIM as usize),
+                        DIM,
+                    )
+                    .unwrap(),
+                )],
+            )
+            .unwrap()
+        })
+        .collect::<Vec<_>>();
+
+    (schema, batches)
+}
+
+async fn create_or_open_dataset() -> Dataset {
+    let uri = dataset_uri();
+    if let Ok(dataset) = Dataset::open(&uri).await
+        && dataset.get_fragments().len() == NUM_FRAGMENTS
+    {
+        return dataset;
+    }
+
+    let dataset_path = dataset_root();
+    if dataset_path.exists() {
+        fs::remove_dir_all(&dataset_path).unwrap();
+    }
+
+    let (schema, batches) = create_batches();
+    let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
+    let write_params = WriteParams {
+        max_rows_per_file: ROWS_PER_FRAGMENT,
+        max_rows_per_group: ROWS_PER_FRAGMENT,
+        mode: WriteMode::Overwrite,
+        ..Default::default()
+    };
+
+    let dataset = Dataset::write(reader, &uri, Some(write_params))
+        .await
+        .unwrap();
+    assert_eq!(dataset.get_fragments().len(), NUM_FRAGMENTS);
+    dataset
+}
+
+async fn train_shared_ivf_pq(
+    dataset: &Dataset,
+    num_partitions: usize,
+) -> (IvfBuildParams, PQBuildParams) {
+    let batch = dataset
+        .scan()
+        .project(&["vector".to_string()])
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    let vectors = batch.column_by_name("vector").unwrap().as_fixed_size_list();
+    let dim = vectors.value_length() as usize;
+    let values = vectors.values().as_primitive::<Float32Type>();
+
+    let kmeans = train_kmeans::<Float32Type>(
+        values,
+        KMeansParams::new(None, MAX_ITERS as u32, 1, DistanceType::L2),
+        dim,
+        num_partitions,
+        SAMPLE_RATE,
+    )
+    .unwrap();
+
+    let centroids = Arc::new(
+        FixedSizeListArray::try_new_from_values(
+            kmeans.centroids.as_primitive::<Float32Type>().clone(),
+            dim as i32,
+        )
+        .unwrap(),
+    );
+    let mut ivf_params = IvfBuildParams::try_with_centroids(num_partitions, centroids).unwrap();
+    ivf_params.max_iters = MAX_ITERS;
+    ivf_params.sample_rate = SAMPLE_RATE;
+
+    let mut pq_train_params = PQBuildParams::new(NUM_SUB_VECTORS, NUM_BITS);
+    pq_train_params.max_iters = MAX_ITERS;
+    pq_train_params.sample_rate = SAMPLE_RATE;
+
+    let pq = pq_train_params.build(vectors, DistanceType::L2).unwrap();
+    let codebook: ArrayRef = Arc::new(pq.codebook.values().as_primitive::<Float32Type>().clone());
+
+    let mut pq_params = PQBuildParams::with_codebook(NUM_SUB_VECTORS, NUM_BITS, codebook);
+    pq_params.max_iters = MAX_ITERS;
+    pq_params.sample_rate = SAMPLE_RATE;
+
+    (ivf_params, pq_params)
+}
+
+fn contiguous_fragment_groups(dataset: &Dataset, num_shards: usize) -> Vec<Vec<u32>> {
+    assert_eq!(NUM_FRAGMENTS % num_shards, 0);
+    let fragments = dataset.get_fragments();
+    let group_size = fragments.len() / num_shards;
+    fragments
+        .chunks(group_size)
+        .map(|group| {
+            group
+                .iter()
+                .map(|frag| frag.id() as u32)
+                .collect::<Vec<_>>()
+        })
+        .collect()
+}
+
+async fn build_partial_fixture(dataset: &mut Dataset, bench_case: BenchCase) -> MergeFixture {
+    let fixture_uuid = fixture_uuid(bench_case);
+    let index_dir = dataset_root()
+        .join("_indices")
+        .join(fixture_uuid.to_string());
+
+    if has_partial_dirs(&index_dir) {
+        return MergeFixture {
+            partial_aux_bytes: sum_partial_auxiliary_bytes(&index_dir),
+            partial_dir_count: count_partial_dirs(&index_dir),
+            index_dir,
+        };
+    }
+
+    if index_dir.exists() {
+        fs::remove_dir_all(&index_dir).unwrap();
+    }
+
+    let fragment_groups = contiguous_fragment_groups(dataset, bench_case.num_shards);
+    let (ivf_params, pq_params) =
+        Box::pin(train_shared_ivf_pq(dataset, bench_case.num_partitions)).await;
+    let params = VectorIndexParams::with_ivf_pq_params(DistanceType::L2, ivf_params, pq_params);
+
+    for fragments in fragment_groups {
+        let mut builder = dataset.create_index_builder(&["vector"], IndexType::Vector, &params);
+        builder = builder
+            .name("distributed_merge_only".to_string())
+            .fragments(fragments)
+            .index_uuid(fixture_uuid.to_string());
+        Box::pin(builder.execute_uncommitted()).await.unwrap();
+    }
+
+    MergeFixture {
+        partial_aux_bytes: sum_partial_auxiliary_bytes(&index_dir),
+        partial_dir_count: count_partial_dirs(&index_dir),
+        index_dir,
+    }
+}
+
+fn has_partial_dirs(index_dir: &Path) -> bool {
+    fs::read_dir(index_dir)
+        .ok()
+        .into_iter()
+        .flatten()
+        .flatten()
+        .any(|entry| {
+            entry.file_type().map(|t| t.is_dir()).unwrap_or(false)
+                && entry.file_name().to_string_lossy().starts_with("partial_")
+        })
+}
+
+fn count_partial_dirs(index_dir: &Path) -> usize {
+    fs::read_dir(index_dir)
+        .unwrap()
+        .flatten()
+        .filter(|entry| {
+            entry.file_type().map(|t| t.is_dir()).unwrap_or(false)
+                && entry.file_name().to_string_lossy().starts_with("partial_")
+        })
+        .count()
+}
+
+fn sum_partial_auxiliary_bytes(index_dir: &Path) -> u64 {
+    fs::read_dir(index_dir)
+        .unwrap()
+        .flatten()
+        .filter(|entry| {
+            entry.file_type().map(|t| t.is_dir()).unwrap_or(false)
+                && entry.file_name().to_string_lossy().starts_with("partial_")
+        })
+        .map(|entry| entry.path().join("auxiliary.idx"))
+        .filter_map(|path| fs::metadata(path).ok())
+        .map(|metadata| metadata.len())
+        .sum()
+}
+
+fn copy_dir_recursive(source: &Path, target: &Path) {
+    fs::create_dir_all(target).unwrap();
+    for entry in fs::read_dir(source).unwrap().flatten() {
+        let source_path = entry.path();
+        let target_path = target.join(entry.file_name());
+        let file_type = entry.file_type().unwrap();
+        if file_type.is_dir() {
+            copy_dir_recursive(&source_path, &target_path);
+        } else {
+            fs::copy(&source_path, &target_path).unwrap();
+        }
+    }
+}
+
+fn prepare_iteration_target(source: &Path, target: &Path) {
+    if target.exists() {
+        fs::remove_dir_all(target).unwrap();
+    }
+    copy_dir_recursive(source, target);
+}
+
+fn write_case_metadata(fixtures: &[(BenchCase, MergeFixture)]) {
+    let output_dir = criterion_group_root();
+    fs::create_dir_all(&output_dir).unwrap();
+    let metadata = fixtures
+        .iter()
+        .map(|(bench_case, fixture)| CaseMetadata {
+            label: bench_case.label(),
+            num_shards: bench_case.num_shards,
+            num_partitions: bench_case.num_partitions,
+            partial_dir_count: fixture.partial_dir_count,
+            partial_aux_bytes: fixture.partial_aux_bytes,
+            partial_aux_bytes_per_shard: fixture.partial_aux_bytes
+                / fixture.partial_dir_count as u64,
+            total_rows: NUM_FRAGMENTS * ROWS_PER_FRAGMENT,
+            rows_per_shard: (NUM_FRAGMENTS * ROWS_PER_FRAGMENT) / bench_case.num_shards,
+        })
+        .collect::<Vec<_>>();
+    let payload = serde_json::to_vec_pretty(&metadata).unwrap();
+    fs::write(output_dir.join("case_metadata.json"), payload).unwrap();
+}
+
+fn bench_distributed_merge_only(c: &mut Criterion) {
+    let rt = Runtime::new().unwrap();
+    let mut dataset = rt.block_on(create_or_open_dataset());
+    let mut fixtures = Vec::new();
+
+    for bench_case in bench_cases() {
+        fixtures.push((
+            bench_case,
+            rt.block_on(build_partial_fixture(&mut dataset, bench_case)),
+        ));
+    }
+    write_case_metadata(&fixtures);
+
+    let dataset = Arc::new(dataset);
+    let mut group = c.benchmark_group("distributed_merge_only_ivf_pq");
+    group.sample_size(10);
+
+    for (bench_case, fixture) in fixtures {
+        let target_uuid = working_uuid(bench_case);
+        let target_index_dir_fs = dataset_root()
+            .join("_indices")
+            .join(target_uuid.to_string());
+        let source_index_dir_fs = fixture.index_dir.clone();
+
+        group.throughput(Throughput::Bytes(fixture.partial_aux_bytes));
+        group.bench_with_input(
+            BenchmarkId::new("finalize_only", bench_case.label()),
+            &bench_case,
+            |b, _| {
+                let dataset = dataset.clone();
+                let target_index_dir_fs = target_index_dir_fs.clone();
+                let source_index_dir_fs = source_index_dir_fs.clone();
+                b.iter_batched(
+                    || prepare_iteration_target(&source_index_dir_fs, &target_index_dir_fs),
+                    |_| {
+                        rt.block_on(dataset.merge_index_metadata(
+                            &target_uuid.to_string(),
+                            IndexType::IvfPq,
+                            None,
+                            noop_progress(),
+                        ))
+                        .unwrap();
+                    },
+                    BatchSize::PerIteration,
+                );
+            },
+        );
+    }
+
+    group.finish();
+}
+
+#[cfg(target_os = "linux")]
+criterion_group!(
+    name = benches;
+    config = Criterion::default()
+        .significance_level(0.1)
+        .sample_size(10)
+        .with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+    targets = bench_distributed_merge_only
+);
+
+#[cfg(not(target_os = "linux"))]
+criterion_group!(
+    name = benches;
+    config = Criterion::default().significance_level(0.1).sample_size(10);
+    targets = bench_distributed_merge_only
+);
+
+criterion_main!(benches);
diff --git a/rust/lance/benches/ivf_pq.rs b/rust/lance/benches/ivf_pq.rs
index ebe70a32d4d..ae92b406168 100644
--- a/rust/lance/benches/ivf_pq.rs
+++ b/rust/lance/benches/ivf_pq.rs
@@ -9,13 +9,14 @@ use arrow_array::{FixedSizeListArray, RecordBatch, RecordBatchIterator};
 use arrow_schema::{DataType, Field, FieldRef, Schema};
 use criterion::{Criterion, criterion_group, criterion_main};
 
+use lance::index::DatasetIndexExt;
 use lance::{
     Dataset,
     dataset::{WriteMode, WriteParams},
     index::vector::VectorIndexParams,
 };
 use lance_arrow::*;
-use lance_index::{DatasetIndexExt, IndexType};
+use lance_index::IndexType;
 use lance_linalg::distance::MetricType;
 use lance_testing::datagen::generate_random_array;
 #[cfg(target_os = "linux")]
diff --git a/rust/lance/benches/mem_wal_read.rs b/rust/lance/benches/mem_wal_read.rs
index c2293c1eb26..80501606d8a 100644
--- a/rust/lance/benches/mem_wal_read.rs
+++ b/rust/lance/benches/mem_wal_read.rs
@@ -59,9 +59,9 @@ use datafusion::prelude::SessionContext;
 use futures::TryStreamExt;
 use lance::dataset::mem_wal::scanner::{
     ActiveMemTableRef, LsmDataSourceCollector, LsmPointLookupPlanner, LsmScanner,
-    LsmVectorSearchPlanner, RegionSnapshot,
+    LsmVectorSearchPlanner, ShardSnapshot,
 };
-use lance::dataset::mem_wal::{DatasetMemWalExt, MemWalConfig, RegionWriterConfig};
+use lance::dataset::mem_wal::{DatasetMemWalExt, MemWalConfig, ShardWriterConfig};
 use lance::dataset::{Dataset, WriteParams};
 use lance_linalg::distance::DistanceType;
 #[cfg(target_os = "linux")]
@@ -170,8 +170,8 @@ struct BenchContext {
     base_dataset: Arc<Dataset>,
     /// Dataset with MemWAL for LSM scan.
     lsm_dataset: Arc<Dataset>,
-    /// Region snapshots with flushed generations.
-    region_snapshots: Vec<RegionSnapshot>,
+    /// Shard snapshots with flushed generations.
+    shard_snapshots: Vec<ShardSnapshot>,
     /// Active memtable reference.
     active_memtable: Option<(Uuid, ActiveMemTableRef)>,
     /// Total rows across all sources.
@@ -232,7 +232,7 @@ async fn setup_benchmark(
     // Initialize MemWAL
     lsm_dataset
         .initialize_mem_wal(MemWalConfig {
-            region_spec: None,
+            shard_spec: None,
             maintained_indexes: vec![],
         })
         .await
@@ -240,22 +240,22 @@ async fn setup_benchmark(
 
     let lsm_dataset = Arc::new(lsm_dataset);
 
-    // Create RegionWriter with small memtable size to trigger flushes
-    let region_id = Uuid::new_v4();
-    let config = RegionWriterConfig {
-        region_id,
-        region_spec_id: 0,
+    // Create ShardWriter with small memtable size to trigger flushes
+    let shard_id = Uuid::new_v4();
+    let config = ShardWriterConfig {
+        shard_id,
+        shard_spec_id: 0,
         durable_write: false,
         sync_indexed_write: false,
         max_memtable_size: memtable_rows * 50, // ~50 bytes per row, triggers flush after memtable_rows
         max_memtable_rows: memtable_rows,
         max_wal_flush_interval: Some(Duration::from_secs(60)), // Long interval to avoid time-based flushes
-        ..RegionWriterConfig::default()
+        ..ShardWriterConfig::default()
     };
 
     let writer = lsm_dataset
         .as_ref()
-        .mem_wal_writer(region_id, config)
+        .mem_wal_writer(shard_id, config)
         .await
         .unwrap();
 
@@ -309,13 +309,12 @@ async fn setup_benchmark(
     // Get active memtable reference
     let active_memtable_ref = writer.active_memtable_ref().await;
 
-    // Build region snapshot
-    let mut region_snapshot = RegionSnapshot::new(region_id);
+    // Build shard snapshot
+    let mut shard_snapshot = ShardSnapshot::new(shard_id);
     if let Some(ref m) = manifest {
-        region_snapshot = region_snapshot.with_current_generation(m.current_generation);
+        shard_snapshot = shard_snapshot.with_current_generation(m.current_generation);
         for fg in &m.flushed_generations {
-            region_snapshot =
-                region_snapshot.with_flushed_generation(fg.generation, fg.path.clone());
+            shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone());
         }
     }
 
@@ -346,8 +345,8 @@ async fn setup_benchmark(
     BenchContext {
         base_dataset,
         lsm_dataset,
-        region_snapshots: vec![region_snapshot],
-        active_memtable: Some((region_id, active_memtable_ref)),
+        shard_snapshots: vec![shard_snapshot],
+        active_memtable: Some((shard_id, active_memtable_ref)),
         total_rows: base_rows + memtable_rows * 2 + gen3_rows,
         pk_columns,
     }
@@ -408,14 +407,14 @@ fn bench_scan(c: &mut Criterion) {
         &(),
         |b, _| {
             let dataset = ctx.lsm_dataset.clone();
-            let region_snapshots = ctx.region_snapshots.clone();
+            let shard_snapshots = ctx.shard_snapshots.clone();
             let pk_columns = ctx.pk_columns.clone();
             b.to_async(&rt).iter(|| {
                 let dataset = dataset.clone();
-                let region_snapshots = region_snapshots.clone();
+                let shard_snapshots = shard_snapshots.clone();
                 let pk_columns = pk_columns.clone();
                 async move {
-                    let scanner = LsmScanner::new(dataset, region_snapshots, pk_columns);
+                    let scanner = LsmScanner::new(dataset, shard_snapshots, pk_columns);
                     let batches: Vec<RecordBatch> = scanner
                         .try_into_stream()
                         .await
@@ -431,20 +430,20 @@ fn bench_scan(c: &mut Criterion) {
     );
 
     // LSM scan: base + flushed + active memtable
-    if let Some((region_id, ref active_memtable)) = ctx.active_memtable {
+    if let Some((shard_id, ref active_memtable)) = ctx.active_memtable {
         group.bench_with_input(BenchmarkId::new("LSM_Full", &label), &(), |b, _| {
             let dataset = ctx.lsm_dataset.clone();
-            let region_snapshots = ctx.region_snapshots.clone();
+            let shard_snapshots = ctx.shard_snapshots.clone();
             let pk_columns = ctx.pk_columns.clone();
             let active = active_memtable.clone();
             b.to_async(&rt).iter(|| {
                 let dataset = dataset.clone();
-                let region_snapshots = region_snapshots.clone();
+                let shard_snapshots = shard_snapshots.clone();
                 let pk_columns = pk_columns.clone();
                 let active = active.clone();
                 async move {
-                    let scanner = LsmScanner::new(dataset, region_snapshots, pk_columns)
-                        .with_active_memtable(region_id, active);
+                    let scanner = LsmScanner::new(dataset, shard_snapshots, pk_columns)
+                        .with_active_memtable(shard_id, active);
                     let batches: Vec<RecordBatch> = scanner
                         .try_into_stream()
                         .await
@@ -510,23 +509,23 @@ fn bench_scan_with_projection(c: &mut Criterion) {
     );
 
     // LSM scan with projection
-    if let Some((region_id, ref active_memtable)) = ctx.active_memtable {
+    if let Some((shard_id, ref active_memtable)) = ctx.active_memtable {
         group.bench_with_input(
             BenchmarkId::new("LSM_Full_Projected", &label),
             &(),
             |b, _| {
                 let dataset = ctx.lsm_dataset.clone();
-                let region_snapshots = ctx.region_snapshots.clone();
+                let shard_snapshots = ctx.shard_snapshots.clone();
                 let pk_columns = ctx.pk_columns.clone();
                 let active = active_memtable.clone();
                 b.to_async(&rt).iter(|| {
                     let dataset = dataset.clone();
-                    let region_snapshots = region_snapshots.clone();
+                    let shard_snapshots = shard_snapshots.clone();
                     let pk_columns = pk_columns.clone();
                     let active = active.clone();
                     async move {
-                        let scanner = LsmScanner::new(dataset, region_snapshots, pk_columns)
-                            .with_active_memtable(region_id, active)
+                        let scanner = LsmScanner::new(dataset, shard_snapshots, pk_columns)
+                            .with_active_memtable(shard_id, active)
                             .project(&["id"]);
                         let batches: Vec<RecordBatch> = scanner
                             .try_into_stream()
@@ -609,7 +608,7 @@ fn bench_point_lookup(c: &mut Criterion) {
     );
 
     // LSM point lookup: key in base table
-    if let Some((region_id, ref active_memtable)) = ctx.active_memtable {
+    if let Some((shard_id, ref active_memtable)) = ctx.active_memtable {
         let arrow_schema: Arc<ArrowSchema> = Arc::new(ctx.lsm_dataset.schema().into());
 
         group.bench_with_input(
@@ -617,20 +616,20 @@ fn bench_point_lookup(c: &mut Criterion) {
             &(),
             |b, _| {
                 let dataset = ctx.lsm_dataset.clone();
-                let region_snapshots = ctx.region_snapshots.clone();
+                let shard_snapshots = ctx.shard_snapshots.clone();
                 let pk_columns = ctx.pk_columns.clone();
                 let schema = arrow_schema.clone();
                 let active = active_memtable.clone();
                 let lookup_id = base_lookup_id;
                 b.to_async(&rt).iter(|| {
                     let dataset = dataset.clone();
-                    let region_snapshots = region_snapshots.clone();
+                    let shard_snapshots = shard_snapshots.clone();
                     let pk_columns = pk_columns.clone();
                     let schema = schema.clone();
                     let active = active.clone();
                     async move {
-                        let collector = LsmDataSourceCollector::new(dataset, region_snapshots)
-                            .with_active_memtable(region_id, active);
+                        let collector = LsmDataSourceCollector::new(dataset, shard_snapshots)
+                            .with_active_memtable(shard_id, active);
                         let planner = LsmPointLookupPlanner::new(collector, pk_columns, schema);
                         let plan = planner
                             .plan_lookup(&[ScalarValue::Int64(Some(lookup_id))], None)
@@ -652,20 +651,20 @@ fn bench_point_lookup(c: &mut Criterion) {
             &(),
             |b, _| {
                 let dataset = ctx.lsm_dataset.clone();
-                let region_snapshots = ctx.region_snapshots.clone();
+                let shard_snapshots = ctx.shard_snapshots.clone();
                 let pk_columns = ctx.pk_columns.clone();
                 let schema = arrow_schema.clone();
                 let active = active_memtable.clone();
                 let lookup_id = flushed_lookup_id;
                 b.to_async(&rt).iter(|| {
                     let dataset = dataset.clone();
-                    let region_snapshots = region_snapshots.clone();
+                    let shard_snapshots = shard_snapshots.clone();
                     let pk_columns = pk_columns.clone();
                     let schema = schema.clone();
                     let active = active.clone();
                     async move {
-                        let collector = LsmDataSourceCollector::new(dataset, region_snapshots)
-                            .with_active_memtable(region_id, active);
+                        let collector = LsmDataSourceCollector::new(dataset, shard_snapshots)
+                            .with_active_memtable(shard_id, active);
                         let planner = LsmPointLookupPlanner::new(collector, pk_columns, schema);
                         let plan = planner
                             .plan_lookup(&[ScalarValue::Int64(Some(lookup_id))], None)
@@ -687,20 +686,20 @@ fn bench_point_lookup(c: &mut Criterion) {
             &(),
             |b, _| {
                 let dataset = ctx.lsm_dataset.clone();
-                let region_snapshots = ctx.region_snapshots.clone();
+                let shard_snapshots = ctx.shard_snapshots.clone();
                 let pk_columns = ctx.pk_columns.clone();
                 let schema = arrow_schema.clone();
                 let active = active_memtable.clone();
                 let lookup_id = active_lookup_id;
                 b.to_async(&rt).iter(|| {
                     let dataset = dataset.clone();
-                    let region_snapshots = region_snapshots.clone();
+                    let shard_snapshots = shard_snapshots.clone();
                     let pk_columns = pk_columns.clone();
                     let schema = schema.clone();
                     let active = active.clone();
                     async move {
-                        let collector = LsmDataSourceCollector::new(dataset, region_snapshots)
-                            .with_active_memtable(region_id, active);
+                        let collector = LsmDataSourceCollector::new(dataset, shard_snapshots)
+                            .with_active_memtable(shard_id, active);
                         let planner = LsmPointLookupPlanner::new(collector, pk_columns, schema);
                         let plan = planner
                             .plan_lookup(&[ScalarValue::Int64(Some(lookup_id))], None)
@@ -786,7 +785,7 @@ fn create_query_vector(dim: usize) -> FixedSizeListArray {
 struct VectorBenchContext {
     base_dataset: Arc<Dataset>,
     lsm_dataset: Arc<Dataset>,
-    region_snapshots: Vec<RegionSnapshot>,
+    shard_snapshots: Vec<ShardSnapshot>,
     active_memtable: Option<(Uuid, ActiveMemTableRef)>,
     total_rows: usize,
     pk_columns: Vec<String>,
@@ -842,7 +841,7 @@ async fn setup_vector_benchmark(
     // Initialize MemWAL
     lsm_dataset
         .initialize_mem_wal(MemWalConfig {
-            region_spec: None,
+            shard_spec: None,
             maintained_indexes: vec![],
         })
         .await
@@ -850,21 +849,21 @@ async fn setup_vector_benchmark(
 
     let lsm_dataset = Arc::new(lsm_dataset);
 
-    let region_id = Uuid::new_v4();
-    let config = RegionWriterConfig {
-        region_id,
-        region_spec_id: 0,
+    let shard_id = Uuid::new_v4();
+    let config = ShardWriterConfig {
+        shard_id,
+        shard_spec_id: 0,
         durable_write: false,
         sync_indexed_write: false,
         max_memtable_size: memtable_rows * (dim * 4 + 8),
         max_memtable_rows: memtable_rows,
         max_wal_flush_interval: Some(Duration::from_secs(60)),
-        ..RegionWriterConfig::default()
+        ..ShardWriterConfig::default()
     };
 
     let writer = lsm_dataset
         .as_ref()
-        .mem_wal_writer(region_id, config)
+        .mem_wal_writer(shard_id, config)
         .await
         .unwrap();
 
@@ -909,12 +908,11 @@ async fn setup_vector_benchmark(
     let manifest = writer.manifest().await.unwrap();
     let active_memtable_ref = writer.active_memtable_ref().await;
 
-    let mut region_snapshot = RegionSnapshot::new(region_id);
+    let mut shard_snapshot = ShardSnapshot::new(shard_id);
     if let Some(ref m) = manifest {
-        region_snapshot = region_snapshot.with_current_generation(m.current_generation);
+        shard_snapshot = shard_snapshot.with_current_generation(m.current_generation);
         for fg in &m.flushed_generations {
-            region_snapshot =
-                region_snapshot.with_flushed_generation(fg.generation, fg.path.clone());
+            shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone());
         }
     }
 
@@ -931,8 +929,8 @@ async fn setup_vector_benchmark(
     VectorBenchContext {
         base_dataset,
         lsm_dataset,
-        region_snapshots: vec![region_snapshot],
-        active_memtable: Some((region_id, active_memtable_ref)),
+        shard_snapshots: vec![shard_snapshot],
+        active_memtable: Some((shard_id, active_memtable_ref)),
         total_rows: base_rows + memtable_rows * 2 + gen3_rows,
         pk_columns,
         vector_dim: dim,
@@ -992,26 +990,26 @@ fn bench_vector_search(c: &mut Criterion) {
     });
 
     // LSM vector search
-    if let Some((region_id, ref active_memtable)) = ctx.active_memtable {
+    if let Some((shard_id, ref active_memtable)) = ctx.active_memtable {
         let arrow_schema: Arc<ArrowSchema> = Arc::new(ctx.lsm_dataset.schema().into());
 
         group.bench_with_input(BenchmarkId::new("LSM_KNN", &label), &(), |b, _| {
             let dataset = ctx.lsm_dataset.clone();
-            let region_snapshots = ctx.region_snapshots.clone();
+            let shard_snapshots = ctx.shard_snapshots.clone();
             let pk_columns = ctx.pk_columns.clone();
             let schema = arrow_schema.clone();
             let active = active_memtable.clone();
             let query = create_query_vector(ctx.vector_dim);
             b.to_async(&rt).iter(|| {
                 let dataset = dataset.clone();
-                let region_snapshots = region_snapshots.clone();
+                let shard_snapshots = shard_snapshots.clone();
                 let pk_columns = pk_columns.clone();
                 let schema = schema.clone();
                 let active = active.clone();
                 let query = query.clone();
                 async move {
-                    let collector = LsmDataSourceCollector::new(dataset, region_snapshots)
-                        .with_active_memtable(region_id, active);
+                    let collector = LsmDataSourceCollector::new(dataset, shard_snapshots)
+                        .with_active_memtable(shard_id, active);
                     let planner = LsmVectorSearchPlanner::new(
                         collector,
                         pk_columns,
diff --git a/rust/lance/benches/mem_wal_write.rs b/rust/lance/benches/mem_wal_write.rs
index 363767b519e..eafb981ae3a 100644
--- a/rust/lance/benches/mem_wal_write.rs
+++ b/rust/lance/benches/mem_wal_write.rs
@@ -31,9 +31,9 @@
 //! - `NUM_BATCHES`: Total number of batches to write (default: 1000)
 //! - `DURABLE_WRITE`: yes/no/both (default: no) - whether writes wait for WAL flush
 //! - `INDEXED_WRITE`: yes/no/both (default: no) - whether writes update indexes synchronously
-//! - `MAX_WAL_BUFFER_SIZE`: WAL buffer size in bytes (default: 1MB from RegionWriterConfig)
+//! - `MAX_WAL_BUFFER_SIZE`: WAL buffer size in bytes (default: 1MB from ShardWriterConfig)
 //! - `MAX_FLUSH_INTERVAL_MS`: WAL flush interval in milliseconds, 0 to disable (default: 1000ms)
-//! - `MAX_MEMTABLE_SIZE`: MemTable size threshold in bytes (default: 64MB from RegionWriterConfig)
+//! - `MAX_MEMTABLE_SIZE`: MemTable size threshold in bytes (default: 64MB from ShardWriterConfig)
 //! - `VECTOR_DIM`: Vector dimension for the vector column (default: 512)
 //! - `MEMWAL_MAINTAINED_INDEXES`: Comma-separated list of index names to maintain in MemWAL (default: id_btree)
 //!     - Available indexes: id_btree, text_fts, vector_ivfpq (all created on base table)
@@ -52,14 +52,15 @@ use arrow_array::{
 };
 use arrow_schema::{DataType, Field, Schema as ArrowSchema};
 use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
-use lance::dataset::mem_wal::{DatasetMemWalExt, MemWalConfig, RegionWriterConfig};
+use lance::dataset::mem_wal::{DatasetMemWalExt, MemWalConfig, ShardWriterConfig};
 use lance::dataset::{Dataset, WriteParams};
+use lance::index::DatasetIndexExt;
 use lance::index::vector::VectorIndexParams;
 use lance_arrow::FixedSizeListArrayExt;
+use lance_index::IndexType;
 use lance_index::scalar::ScalarIndexParams;
 use lance_index::vector::ivf::IvfBuildParams;
 use lance_index::vector::pq::PQBuildParams;
-use lance_index::{DatasetIndexExt, IndexType};
 use lance_linalg::distance::DistanceType;
 #[cfg(target_os = "linux")]
 use pprof::criterion::{Output, PProfProfiler};
@@ -392,7 +393,7 @@ async fn create_dataset(
     // Initialize MemWAL with specified maintained indexes
     dataset
         .initialize_mem_wal(MemWalConfig {
-            region_spec: None,
+            shard_spec: None,
             maintained_indexes: maintained_indexes.to_vec(),
         })
         .await
@@ -478,7 +479,7 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
     let total_bytes = (total_rows * row_size_bytes) as u64;
 
     // Get effective config values for display
-    let default_config = RegionWriterConfig::default();
+    let default_config = ShardWriterConfig::default();
     let effective_wal_buffer = max_wal_buffer_size.unwrap_or(default_config.max_wal_buffer_size);
     let effective_flush_interval =
         max_flush_interval.unwrap_or(default_config.max_wal_flush_interval);
@@ -529,7 +530,7 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
             let name_prefix = build_name_prefix(durable, indexed);
 
             // Create dataset ONCE before benchmark iterations
-            // Each iteration will use a different region on the same dataset
+            // Each iteration will use a different shard on the same dataset
             let dataset = rt.block_on(create_dataset(
                 &schema,
                 &name_prefix,
@@ -571,12 +572,12 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
                                 // Re-open dataset (cheap operation)
                                 let dataset = Dataset::open(&dataset_uri).await.unwrap();
 
-                                // Create a NEW region for each iteration
-                                let region_id = Uuid::new_v4();
-                                let default_config = RegionWriterConfig::default();
-                                let config = RegionWriterConfig {
-                                    region_id,
-                                    region_spec_id: 0,
+                                // Create a NEW shard for each iteration
+                                let shard_id = Uuid::new_v4();
+                                let default_config = ShardWriterConfig::default();
+                                let config = ShardWriterConfig {
+                                    shard_id,
+                                    shard_spec_id: 0,
                                     durable_write: durable,
                                     sync_indexed_write: indexed,
                                     max_wal_buffer_size: max_wal_buffer_size
@@ -602,7 +603,7 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
 
                                 // Get writer through Dataset API (index configs loaded automatically)
                                 let writer =
-                                    dataset.mem_wal_writer(region_id, config).await.unwrap();
+                                    dataset.mem_wal_writer(shard_id, config).await.unwrap();
 
                                 // Time writes (excluding close to measure pure put throughput)
                                 let start = Instant::now();
@@ -619,7 +620,7 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
                                 // Get stats after close to include all WAL flushes
                                 let stats = stats_handle.snapshot();
 
-                                total_duration += put_duration;
+                                total_duration += put_duration + close_duration;
 
                                 // Report stats once (first iteration of first sample only)
                                 if iter == 0 && !stats_printed.swap(true, Ordering::SeqCst) {
diff --git a/rust/lance/benches/memtable_read.rs b/rust/lance/benches/memtable_read.rs
index 240c90294e9..efd16dbb62a 100644
--- a/rust/lance/benches/memtable_read.rs
+++ b/rust/lance/benches/memtable_read.rs
@@ -38,15 +38,16 @@ use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_m
 use futures::TryStreamExt;
 use lance::dataset::mem_wal::write::{CacheConfig, IndexStore, MemTable};
 use lance::dataset::{Dataset, WriteParams};
+use lance::index::DatasetIndexExt;
 use lance::index::vector::VectorIndexParams;
 use lance_arrow::FixedSizeListArrayExt;
+use lance_index::IndexType;
 use lance_index::scalar::FullTextSearchQuery;
 use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
 use lance_index::vector::ivf::IvfBuildParams;
 use lance_index::vector::ivf::storage::IvfModel;
 use lance_index::vector::kmeans::{KMeansParams, train_kmeans};
 use lance_index::vector::pq::builder::PQBuildParams;
-use lance_index::{DatasetIndexExt, IndexType};
 use lance_linalg::distance::{DistanceType, MetricType};
 #[cfg(target_os = "linux")]
 use pprof::criterion::{Output, PProfProfiler};
diff --git a/rust/lance/benches/take.rs b/rust/lance/benches/take.rs
index 68d9c963ef9..ec078d0f636 100644
--- a/rust/lance/benches/take.rs
+++ b/rust/lance/benches/take.rs
@@ -376,7 +376,7 @@ fn bench_sample(c: &mut Criterion) {
                     let schema = schema.clone();
                     let dataset = dataset.clone();
                     async move {
-                        dataset.sample(sample_size, &schema).await.unwrap();
+                        dataset.sample(sample_size, &schema, None).await.unwrap();
                     }
                 })
             },
diff --git a/rust/lance/benches/vector_index.rs b/rust/lance/benches/vector_index.rs
index cf4106db8b4..21c9aa4e4aa 100644
--- a/rust/lance/benches/vector_index.rs
+++ b/rust/lance/benches/vector_index.rs
@@ -15,10 +15,11 @@ use pprof::criterion::{Output, PProfProfiler};
 use rand::Rng;
 
 use lance::dataset::{Dataset, WriteMode, WriteParams, builder::DatasetBuilder};
+use lance::index::DatasetIndexExt;
 use lance::index::vector::VectorIndexParams;
 use lance_arrow::{FixedSizeListArrayExt, as_fixed_size_list_array};
 use lance_index::{
-    DatasetIndexExt, IndexType,
+    IndexType,
     vector::{ivf::IvfBuildParams, pq::PQBuildParams},
 };
 use lance_linalg::distance::MetricType;
diff --git a/rust/lance/benches/vector_throughput.rs b/rust/lance/benches/vector_throughput.rs
index ff24c7771ba..9a04971684b 100644
--- a/rust/lance/benches/vector_throughput.rs
+++ b/rust/lance/benches/vector_throughput.rs
@@ -19,10 +19,11 @@ use pprof::criterion::{Output, PProfProfiler};
 use rand::Rng;
 
 use lance::dataset::{Dataset, WriteMode, WriteParams};
+use lance::index::DatasetIndexExt;
 use lance::index::vector::VectorIndexParams;
 use lance_arrow::FixedSizeListArrayExt;
 use lance_index::{
-    DatasetIndexExt, IndexType,
+    IndexType,
     vector::{ivf::IvfBuildParams, pq::PQBuildParams},
 };
 use lance_linalg::distance::MetricType;
diff --git a/rust/lance/build.rs b/rust/lance/build.rs
new file mode 100644
index 00000000000..4225007d46c
--- /dev/null
+++ b/rust/lance/build.rs
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::io::Result;
+
+fn main() -> Result<()> {
+    println!("cargo:rerun-if-changed=protos");
+
+    #[cfg(feature = "protoc")]
+    // Use vendored protobuf compiler if requested.
+    unsafe {
+        std::env::set_var("PROTOC", protobuf_src::protoc());
+    }
+
+    let mut prost_build = prost_build::Config::new();
+    prost_build.extern_path(".lance.table", "::lance_table::format::pb");
+    prost_build.extern_path(".lance.index.pb", "::lance_index::pb");
+    prost_build.extern_path(".lance.datafusion", "::lance_datafusion::pb");
+    prost_build.protoc_arg("--experimental_allow_proto3_optional");
+    prost_build.enable_type_names();
+    prost_build.compile_protos(&["./protos/ann.proto"], &["./protos"])?;
+
+    Ok(())
+}
diff --git a/rust/lance/src/bin/lq.rs b/rust/lance/src/bin/lq.rs
index d28121bbf55..afaa15c7f10 100644
--- a/rust/lance/src/bin/lq.rs
+++ b/rust/lance/src/bin/lq.rs
@@ -10,9 +10,9 @@ use futures::TryStreamExt;
 use futures::stream::StreamExt;
 
 use lance::dataset::Dataset;
+use lance::index::DatasetIndexExt;
 use lance::index::vector::VectorIndexParams;
 use lance::{Error, Result};
-use lance_index::DatasetIndexExt;
 use lance_linalg::distance::MetricType;
 
 #[derive(Parser)]
diff --git a/rust/lance/src/datafusion/dataframe.rs b/rust/lance/src/datafusion/dataframe.rs
index 0c0291d2d1b..00db9920bf9 100644
--- a/rust/lance/src/datafusion/dataframe.rs
+++ b/rust/lance/src/datafusion/dataframe.rs
@@ -258,7 +258,7 @@ impl SessionContextExt for SessionContext {
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
     use std::sync::Arc;
 
     use arrow::{
diff --git a/rust/lance/src/dataset.rs b/rust/lance/src/dataset.rs
index 6adefc7bf3d..8a7a9cf3636 100644
--- a/rust/lance/src/dataset.rs
+++ b/rust/lance/src/dataset.rs
@@ -15,6 +15,7 @@ use futures::{FutureExt, Stream};
 
 use crate::dataset::metadata::UpdateFieldMetadataBuilder;
 use crate::dataset::transaction::translate_schema_metadata_updates;
+use crate::index::DatasetIndexExt;
 use crate::session::caches::{DSMetadataCache, ManifestKey, TransactionKey};
 use crate::session::index_caches::DSIndexCache;
 use itertools::Itertools;
@@ -28,14 +29,17 @@ use lance_core::utils::tracing::{
 };
 use lance_datafusion::projection::ProjectionPlan;
 use lance_file::datatypes::populate_schema_dictionary;
-use lance_file::reader::FileReaderOptions;
+use lance_file::reader::{FileReader, FileReaderOptions};
 use lance_file::version::LanceFileVersion;
-use lance_index::{DatasetIndexExt, IndexType};
+use lance_index::{IndexType, progress::IndexBuildProgress};
 use lance_io::object_store::{
     LanceNamespaceStorageOptionsProvider, ObjectStore, ObjectStoreParams, StorageOptions,
     StorageOptionsAccessor, StorageOptionsProvider,
 };
-use lance_io::utils::{read_last_block, read_message, read_metadata_offset, read_struct};
+use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
+use lance_io::utils::{
+    CachedFileSize, read_last_block, read_message, read_metadata_offset, read_struct,
+};
 use lance_namespace::LanceNamespace;
 use lance_table::format::{
     DataFile, DataStorageFormat, DeletionFile, Fragment, IndexMetadata, Manifest, RowIdMeta, pb,
@@ -54,12 +58,12 @@ use roaring::RoaringBitmap;
 use rowids::get_row_id_index;
 use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
-use std::collections::{BTreeMap, HashMap, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::fmt::Debug;
+use std::num::NonZero;
 use std::ops::Range;
 use std::pin::Pin;
 use std::sync::Arc;
-use take::row_offsets_to_row_addresses;
 use tracing::{info, instrument};
 
 pub(crate) mod blob;
@@ -87,6 +91,8 @@ pub mod updater;
 mod utils;
 pub mod write;
 
+pub(crate) use take::row_offsets_to_row_addresses;
+
 use self::builder::DatasetBuilder;
 use self::cleanup::RemovalStats;
 use self::fragment::FileFragment;
@@ -107,7 +113,7 @@ use crate::io::commit::{
 use crate::session::Session;
 use crate::utils::temporal::{SystemTime, timestamp_to_nanos, utc_now};
 use crate::{Error, Result};
-pub use blob::BlobFile;
+pub use blob::{BlobFile, ReadBlob, ReadBlobsBuilder, ReadBlobsStream};
 use hash_joiner::HashJoiner;
 pub use lance_core::ROW_ID;
 use lance_core::box_error;
@@ -128,8 +134,8 @@ use crate::dataset::index::LanceIndexStoreExt;
 pub use write::update::{UpdateBuilder, UpdateJob};
 #[allow(deprecated)]
 pub use write::{
-    AutoCleanupParams, CommitBuilder, DeleteBuilder, DeleteResult, InsertBuilder, WriteDestination,
-    WriteMode, WriteParams, write_fragments,
+    AutoCleanupParams, CommitBuilder, DeleteBuilder, DeleteResult, ExternalBlobMode, InsertBuilder,
+    WriteDestination, WriteMode, WriteParams, WriteProgressFn, WriteStats, write_fragments,
 };
 
 pub(crate) const INDICES_DIR: &str = "_indices";
@@ -175,6 +181,8 @@ pub struct Dataset {
     /// Object store parameters used when opening this dataset.
     /// These are used when creating object stores for additional base paths.
     pub(crate) store_params: Option<Box<ObjectStoreParams>>,
+    /// Optional runtime-only object store parameters keyed by base path URI.
+    pub(crate) base_store_params: Option<Arc<HashMap<String, ObjectStoreParams>>>,
 }
 
 impl std::fmt::Debug for Dataset {
@@ -184,6 +192,7 @@ impl std::fmt::Debug for Dataset {
             .field("base", &self.base)
             .field("version", &self.manifest.version)
             .field("cache_num_items", &self.session.approx_num_items())
+            .field("base_store_params", &self.base_store_params.is_some())
             .finish()
     }
 }
@@ -251,6 +260,9 @@ pub struct ReadParams {
     /// File reader options to use when reading data files.
     ///
     /// This allows control over features like caching repetition indices and validation.
+    /// Options set here act as dataset-level defaults and can be overridden on a
+    /// per-scan basis via [`Scanner::batch_size_bytes`](crate::dataset::scanner::Scanner::batch_size_bytes) or
+    /// [`Scanner::with_file_reader_options`](crate::dataset::scanner::Scanner::with_file_reader_options).
     pub file_reader_options: Option<FileReaderOptions>,
 }
 
@@ -575,6 +587,7 @@ impl Dataset {
             self.commit_handler.clone(),
             self.file_reader_options.clone(),
             self.store_params.as_deref().cloned(),
+            self.base_store_params.clone(),
         )
     }
 
@@ -693,6 +706,7 @@ impl Dataset {
         commit_handler: Arc<dyn CommitHandler>,
         file_reader_options: Option<FileReaderOptions>,
         store_params: Option<ObjectStoreParams>,
+        base_store_params: Option<Arc<HashMap<String, ObjectStoreParams>>>,
     ) -> Result<Self> {
         let refs = Refs::new(
             object_store.clone(),
@@ -720,6 +734,7 @@ impl Dataset {
             index_cache,
             file_reader_options,
             store_params: store_params.map(Box::new),
+            base_store_params,
         })
     }
 
@@ -743,20 +758,20 @@ impl Dataset {
             .await
     }
 
-    /// Write into a namespace-managed table with automatic credential vending.
+    /// Write into a namespace client-managed table with automatic credential vending.
     ///
     /// For CREATE mode, calls declare_table() to initialize the table.
-    /// For other modes, calls describe_table() and opens dataset with namespace credentials.
+    /// For other modes, calls describe_table() and opens dataset with namespace client credentials.
     ///
     /// # Arguments
     ///
     /// * `batches` - The record batches to write
-    /// * `namespace` - The namespace to use for table management
+    /// * `namespace_client` - The namespace client to use for table management
     /// * `table_id` - The table identifier
     /// * `params` - Write parameters
     pub async fn write_into_namespace(
         batches: impl RecordBatchReader + Send + 'static,
-        namespace: Arc<dyn LanceNamespace>,
+        namespace_client: Arc<dyn LanceNamespace>,
         table_id: Vec<String>,
         mut params: Option<WriteParams>,
     ) -> Result<Self> {
@@ -768,7 +783,7 @@ impl Dataset {
                     id: Some(table_id.clone()),
                     ..Default::default()
                 };
-                let response = namespace
+                let response = namespace_client
                     .declare_table(declare_request)
                     .await
                     .map_err(|e| Error::namespace_source(Box::new(e)))?;
@@ -782,7 +797,7 @@ impl Dataset {
                 // Set up commit handler when managed_versioning is enabled
                 if response.managed_versioning == Some(true) {
                     let external_store = LanceNamespaceExternalManifestStore::new(
-                        namespace.clone(),
+                        namespace_client.clone(),
                         table_id.clone(),
                     );
                     let commit_handler: Arc<dyn CommitHandler> =
@@ -792,13 +807,13 @@ impl Dataset {
                     write_params.commit_handler = Some(commit_handler);
                 }
 
-                // Set initial credentials and provider from namespace
+                // Set initial credentials and provider from namespace_client
                 if let Some(namespace_storage_options) = response.storage_options {
                     let provider: Arc<dyn StorageOptionsProvider> = Arc::new(
-                        LanceNamespaceStorageOptionsProvider::new(namespace, table_id),
+                        LanceNamespaceStorageOptionsProvider::new(namespace_client, table_id),
                     );
 
-                    // Merge namespace storage options with any existing options
+                    // Merge namespace client storage options with any existing options
                     let mut merged_options = write_params
                         .store_params
                         .as_ref()
@@ -825,7 +840,7 @@ impl Dataset {
                     id: Some(table_id.clone()),
                     ..Default::default()
                 };
-                let response = namespace
+                let response = namespace_client
                     .describe_table(request)
                     .await
                     .map_err(|e| Error::namespace_source(Box::new(e)))?;
@@ -839,7 +854,7 @@ impl Dataset {
                 // Set up commit handler when managed_versioning is enabled
                 if response.managed_versioning == Some(true) {
                     let external_store = LanceNamespaceExternalManifestStore::new(
-                        namespace.clone(),
+                        namespace_client.clone(),
                         table_id.clone(),
                     );
                     let commit_handler: Arc<dyn CommitHandler> =
@@ -849,15 +864,15 @@ impl Dataset {
                     write_params.commit_handler = Some(commit_handler);
                 }
 
-                // Set initial credentials and provider from namespace
+                // Set initial credentials and provider from namespace_client
                 if let Some(namespace_storage_options) = response.storage_options {
                     let provider: Arc<dyn StorageOptionsProvider> =
                         Arc::new(LanceNamespaceStorageOptionsProvider::new(
-                            namespace.clone(),
+                            namespace_client.clone(),
                             table_id.clone(),
                         ));
 
-                    // Merge namespace storage options with any existing options
+                    // Merge namespace client storage options with any existing options
                     let mut merged_options = write_params
                         .store_params
                         .as_ref()
@@ -1466,10 +1481,43 @@ impl Dataset {
         row_indices: &[u64],
         column: impl AsRef<str>,
     ) -> Result<Vec<BlobFile>> {
-        let row_addrs = row_offsets_to_row_addresses(self, row_indices).await?;
+        let fragments = self.get_fragments();
+        let row_addrs = row_offsets_to_row_addresses(&fragments, row_indices).await?;
         blob::take_blobs_by_addresses(self, &row_addrs, column.as_ref()).await
     }
 
+    /// Create a planned blob reader for a blob column.
+    ///
+    /// This API complements [`Self::take_blobs`]. `take_blobs` returns
+    /// [`BlobFile`] handles for caller-driven random access, while
+    /// `read_blobs` builds a streaming read plan for sequential or batched blob
+    /// retrieval.
+    ///
+    /// ```rust
+    /// # use std::sync::Arc;
+    /// # use futures::TryStreamExt;
+    /// # use lance::dataset::Dataset;
+    /// # use lance::Result;
+    /// # async fn example(dataset: Arc<Dataset>) -> Result<()> {
+    /// let blobs = dataset
+    ///     .read_blobs("images")?
+    ///     .with_row_indices(vec![0, 1, 2])
+    ///     .execute()
+    ///     .await?;
+    /// # let _ = blobs;
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn read_blobs(self: &Arc<Self>, column: impl AsRef<str>) -> Result<ReadBlobsBuilder> {
+        let column = column.as_ref();
+        let blob_field_id = blob::validate_blob_column(self, column)?;
+        Ok(ReadBlobsBuilder::new(
+            self.clone(),
+            column.to_string(),
+            blob_field_id,
+        ))
+    }
+
     /// Get a stream of batches based on iterator of ranges of row numbers.
     ///
     /// This is an experimental API. It may change at any time.
@@ -1484,14 +1532,74 @@ impl Dataset {
 
     /// Randomly sample `n` rows from the dataset.
     ///
+    /// If `fragment_ids` is provided, sampling is limited to rows from those
+    /// fragments in the current dataset version.
+    ///
     /// The returned rows are in row-id order (not random order), which allows
     /// the underlying take operation to use an efficient sorted code path.
-    pub async fn sample(&self, n: usize, projection: &Schema) -> Result<RecordBatch> {
+    pub async fn sample(
+        &self,
+        n: usize,
+        projection: &Schema,
+        fragment_ids: Option<&[u32]>,
+    ) -> Result<RecordBatch> {
         use rand::seq::IteratorRandom;
-        let num_rows = self.count_rows(None).await?;
-        let mut ids = (0..num_rows as u64).choose_multiple(&mut rand::rng(), n);
-        ids.sort_unstable();
-        self.take(&ids, projection.clone()).await
+
+        match fragment_ids {
+            None => {
+                let num_rows = self.count_rows(None).await?;
+                let mut ids = (0..num_rows as u64).choose_multiple(&mut rand::rng(), n);
+                ids.sort_unstable();
+                self.take(&ids, projection.clone()).await
+            }
+            Some(fragment_ids) => {
+                if fragment_ids.is_empty() {
+                    return Err(Error::invalid_input(
+                        "Dataset::sample does not accept an empty fragment_ids list".to_string(),
+                    ));
+                }
+
+                let selected_fragment_ids = fragment_ids.iter().copied().collect::<BTreeSet<_>>();
+                let selected_fragments = self
+                    .get_fragments()
+                    .into_iter()
+                    .filter(|fragment| selected_fragment_ids.contains(&(fragment.id() as u32)))
+                    .collect::<Vec<_>>();
+
+                if selected_fragments.len() != selected_fragment_ids.len() {
+                    let present_fragment_ids = selected_fragments
+                        .iter()
+                        .map(|fragment| fragment.id() as u32)
+                        .collect::<HashSet<_>>();
+                    let missing_fragment_ids = selected_fragment_ids
+                        .into_iter()
+                        .filter(|fragment_id| !present_fragment_ids.contains(fragment_id))
+                        .collect::<Vec<_>>();
+                    return Err(Error::invalid_input(format!(
+                        "Dataset::sample received fragment ids that are not part of the current dataset version: {missing_fragment_ids:?}",
+                    )));
+                }
+
+                let num_rows = stream::iter(selected_fragments.iter().cloned())
+                    .map(|fragment| async move { fragment.count_rows(None).await })
+                    .buffer_unordered(16)
+                    .try_fold(0_u64, |acc, rows| async move { Ok(acc + rows as u64) })
+                    .await?;
+
+                let mut offsets = (0..num_rows).choose_multiple(&mut rand::rng(), n);
+                offsets.sort_unstable();
+
+                let row_addrs = row_offsets_to_row_addresses(&selected_fragments, &offsets).await?;
+                let dataset = Arc::new(self.clone());
+                let projection = Arc::new(
+                    ProjectionRequest::from(projection.clone())
+                        .into_projection_plan(dataset.clone())?,
+                );
+                TakeBuilder::try_new_from_addresses(dataset, row_addrs, projection)?
+                    .execute()
+                    .await
+            }
+        }
     }
 
     /// Delete rows based on a predicate.
@@ -1568,6 +1676,23 @@ impl Dataset {
         cloned
     }
 
+    fn store_params_for_base(
+        &self,
+        base_path: Option<&lance_table::format::BasePath>,
+    ) -> ObjectStoreParams {
+        // Base-specific bindings are exact ObjectStoreParams keyed by
+        // `BasePath.path`. If a base has no explicit binding then reads fall back
+        // to the dataset-level default store params.
+        base_path
+            .and_then(|base_path| {
+                self.base_store_params
+                    .as_ref()
+                    .and_then(|params| params.get(&base_path.path))
+            })
+            .cloned()
+            .unwrap_or_else(|| self.store_params.as_deref().cloned().unwrap_or_default())
+    }
+
     /// Returns the initial storage options used when opening this dataset, if any.
     ///
     /// This returns the static initial options without triggering any refresh.
@@ -1651,14 +1776,116 @@ impl Dataset {
     }
 
     pub(crate) fn data_file_dir(&self, data_file: &DataFile) -> Result<Path> {
-        match data_file.base_id.as_ref() {
+        self.data_file_dir_for_base(data_file.base_id)
+    }
+
+    /// Create a [`DataFile`] by reading metadata from an existing lance file.
+    ///
+    /// This reads the file's schema and version information, matches columns to
+    /// the dataset's schema to determine field IDs, and calculates column indices.
+    /// This is useful for constructing `DataFile` metadata needed for operations
+    /// like [`Operation::DataReplacement`].
+    ///
+    /// # Arguments
+    ///
+    /// * `path` - The path to the data file, relative to the dataset's data directory.
+    /// * `base_id` - The base path ID if the file is outside the dataset directory.
+    pub async fn create_data_file(&self, path: &str, base_id: Option<u32>) -> Result<DataFile> {
+        let data_dir = self.data_file_dir_for_base(base_id)?;
+        let filepath = data_dir.child(path);
+
+        // Get file size
+        let file_size = self.object_store().size(&filepath).await?;
+
+        // Read file metadata
+        let scheduler = ScanScheduler::new(
+            self.object_store.clone(),
+            SchedulerConfig::new(2 * 1024 * 1024 * 1024),
+        );
+        let file = scheduler
+            .open_file(&filepath, &CachedFileSize::new(file_size))
+            .await?;
+        let file_metadata = FileReader::read_all_metadata(&file).await?;
+
+        let file_version = LanceFileVersion::try_from_major_minor(
+            file_metadata.major_version as u32,
+            file_metadata.minor_version as u32,
+        )?;
+
+        // Get top-level column names from file schema in file order
+        let column_names: Vec<&str> = file_metadata
+            .file_schema
+            .fields
+            .iter()
+            .map(|f| f.name.as_str())
+            .collect();
+
+        // Project dataset schema by file column names to get dataset field IDs
+        let projected_ds_schema = self.schema().project(&column_names)?;
+
+        // Walk both schemas in parallel to build fields and column_indices
+        let is_structural = file_version >= LanceFileVersion::V2_1;
+        let ds_fields: Vec<_> = projected_ds_schema.fields_pre_order().collect();
+        let file_fields: Vec<_> = file_metadata.file_schema.fields_pre_order().collect();
+
+        if ds_fields.len() != file_fields.len() {
+            return Err(Error::invalid_input(format!(
+                "Schema mismatch: dataset projection has {} fields but file has {} fields",
+                ds_fields.len(),
+                file_fields.len()
+            )));
+        }
+
+        let mut fields = Vec::new();
+        let mut column_indices = Vec::new();
+        let mut curr_column_idx: i32 = 0;
+        let mut packed_struct_fields_num: usize = 0;
+
+        for (ds_field, file_field) in ds_fields.iter().zip(file_fields.iter()) {
+            if ds_field.name != file_field.name {
+                return Err(Error::invalid_input(format!(
+                    "Schema mismatch: expected field '{}' but file has '{}'",
+                    ds_field.name, file_field.name
+                )));
+            }
+
+            if packed_struct_fields_num > 0 {
+                packed_struct_fields_num -= 1;
+                continue;
+            }
+
+            if file_field.is_packed_struct() {
+                fields.push(ds_field.id);
+                column_indices.push(curr_column_idx);
+                curr_column_idx += 1;
+                packed_struct_fields_num = file_field.children.len();
+            } else if file_field.children.is_empty() || !is_structural {
+                fields.push(ds_field.id);
+                column_indices.push(curr_column_idx);
+                curr_column_idx += 1;
+            }
+        }
+
+        let file_size_nz = NonZero::new(file_size);
+        Ok(DataFile::new(
+            path,
+            fields,
+            column_indices,
+            file_metadata.major_version as u32,
+            file_metadata.minor_version as u32,
+            file_size_nz,
+            base_id,
+        ))
+    }
+
+    /// Resolve the data directory for a given base_id.
+    ///
+    /// If `base_id` is `None`, returns the default data directory.
+    fn data_file_dir_for_base(&self, base_id: Option<u32>) -> Result<Path> {
+        match base_id {
             Some(base_id) => {
-                let base_paths = &self.manifest.base_paths;
-                let base_path = base_paths.get(base_id).ok_or_else(|| {
-                    Error::invalid_input(format!(
-                        "base_path id {} not found for data_file {}",
-                        base_id, data_file.path
-                    ))
+                let base_path = self.manifest.base_paths.get(&base_id).ok_or_else(|| {
+                    Error::invalid_input(format!("base_path id {} not found", base_id))
                 })?;
                 let path = base_path.extract_path(self.session.store_registry())?;
                 if base_path.is_dataset_root {
@@ -1676,11 +1903,12 @@ impl Dataset {
         let base_path = self.manifest.base_paths.get(&base_id).ok_or_else(|| {
             Error::invalid_input(format!("Dataset base path with ID {} not found", base_id))
         })?;
+        let store_params = self.store_params_for_base(Some(base_path));
 
         let (store, _) = ObjectStore::from_uri_and_params(
             self.session.store_registry(),
             &base_path.path,
-            &self.store_params.as_deref().cloned().unwrap_or_default(),
+            &store_params,
         )
         .await?;
 
@@ -1737,6 +1965,18 @@ impl Dataset {
         self.session.clone()
     }
 
+    /// Get the currently checked-out version id.
+    ///
+    /// This is a cheap accessor that reads the id directly from the loaded
+    /// manifest without constructing the full [Version] summary.
+    pub fn version_id(&self) -> u64 {
+        self.manifest.version
+    }
+
+    /// Get the currently checked-out version details.
+    ///
+    /// This constructs a full [Version], including summary metadata derived
+    /// from the loaded manifest fragments.
     pub fn version(&self) -> Version {
         Version::from(self.manifest.as_ref())
     }
@@ -1776,6 +2016,27 @@ impl Dataset {
         Ok(versions)
     }
 
+    /// List all detached manifest locations.
+    ///
+    /// Detached manifests are versions that are not part of the main version history.
+    /// They are created by `commit_detached` and can be used for staging changes.
+    ///
+    /// To read transaction properties from a detached manifest:
+    /// ```ignore
+    /// let detached = dataset.list_detached_manifests().await?;
+    /// for location in detached {
+    ///     let ds = dataset.checkout_version(location.version).await?;
+    ///     let tx = ds.read_transaction().await?;
+    ///     // Access tx.transaction_properties
+    /// }
+    /// ```
+    pub async fn list_detached_manifests(&self) -> Result<Vec<ManifestLocation>> {
+        self.commit_handler
+            .list_detached_manifest_locations(&self.base, &self.object_store)
+            .try_collect()
+            .await
+    }
+
     /// Get the latest version of the dataset
     /// This is meant to be a fast path for checking if a dataset has changed. This is why
     /// we don't return the full version struct.
@@ -1817,6 +2078,11 @@ impl Dataset {
             .collect()
     }
 
+    /// Iterate over manifest fragments without allocating [`FileFragment`] wrappers.
+    pub fn iter_fragments(&self) -> impl Iterator<Item = &Fragment> {
+        self.manifest.fragments.iter()
+    }
+
     pub fn get_fragment(&self, fragment_id: usize) -> Option<FileFragment> {
         let dataset = Arc::new(self.clone());
         let fragment = self
@@ -2463,6 +2729,7 @@ pub(crate) fn load_new_transactions(dataset: &Dataset) -> NewTransactionResult<'
                         dataset.commit_handler.clone(),
                         dataset.file_reader_options.clone(),
                         dataset.store_params.as_deref().cloned(),
+                        dataset.base_store_params.clone(),
                     )?;
                     let loaded =
                         Arc::new(dataset_version.read_transaction().await?.ok_or_else(|| {
@@ -2494,6 +2761,7 @@ pub(crate) fn load_new_transactions(dataset: &Dataset) -> NewTransactionResult<'
                 dataset.commit_handler.clone(),
                 dataset.file_reader_options.clone(),
                 dataset.store_params.as_deref().cloned(),
+                dataset.base_store_params.clone(),
             )
         } else {
             // If we didn't get the latest manifest, we can still return the dataset
@@ -2658,11 +2926,14 @@ impl Dataset {
         self.merge_impl(stream, left_on, right_on).await
     }
 
+    /// Merge a distributed scalar index into a single root artifact and report
+    /// progress via the supplied callback.
     pub async fn merge_index_metadata(
         &self,
         index_uuid: &str,
         index_type: IndexType,
         batch_readhead: Option<usize>,
+        progress: Arc<dyn IndexBuildProgress>,
     ) -> Result<()> {
         let store = LanceIndexStore::from_dataset_for_new(self, index_uuid)?;
         let index_dir = self.indices_dir().child(index_uuid);
@@ -2673,6 +2944,7 @@ impl Dataset {
                     self.object_store(),
                     &index_dir,
                     Arc::new(store),
+                    progress,
                 )
                 .await
             }
@@ -2683,19 +2955,17 @@ impl Dataset {
                     &index_dir,
                     Arc::new(store),
                     batch_readhead,
+                    progress,
                 )
                 .await
             }
-            // Precise vector index types: IVF_FLAT, IVF_PQ, IVF_SQ
             IndexType::IvfFlat | IndexType::IvfPq | IndexType::IvfSq | IndexType::Vector => {
-                // Merge distributed vector index partials and finalize root index via Lance IVF helper
-                crate::index::vector::ivf::finalize_distributed_merge(
-                    self.object_store(),
-                    &index_dir,
-                    Some(index_type),
-                )
-                .await?;
-                Ok(())
+                Err(Error::invalid_input(
+                    "Vector distributed indexing no longer supports merge_index_metadata; \
+                     build segments, optionally merge groups with merge_existing_index_segments(...), \
+                     and commit with commit_existing_index_segments(...)"
+                        .to_string(),
+                ))
             }
             _ => Err(Error::invalid_input_source(Box::new(std::io::Error::new(
                 std::io::ErrorKind::InvalidInput,
diff --git a/rust/lance/src/dataset/blob.rs b/rust/lance/src/dataset/blob.rs
index 428c71a3458..a828f3fc336 100644
--- a/rust/lance/src/dataset/blob.rs
+++ b/rust/lance/src/dataset/blob.rs
@@ -1,7 +1,14 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::{collections::HashMap, future::Future, ops::DerefMut, sync::Arc};
+use std::{
+    collections::{BTreeMap, HashMap, VecDeque},
+    future::Future,
+    ops::{DerefMut, Range},
+    panic::AssertUnwindSafe,
+    sync::Arc,
+    task::Poll,
+};
 
 use arrow::array::AsArray;
 use arrow::datatypes::{UInt8Type, UInt32Type, UInt64Type};
@@ -9,25 +16,30 @@ use arrow_array::Array;
 use arrow_array::RecordBatch;
 use arrow_array::builder::{LargeBinaryBuilder, PrimitiveBuilder, StringBuilder};
 use arrow_schema::DataType as ArrowDataType;
+use bytes::Bytes;
+use futures::stream::BoxStream;
+use futures::{FutureExt, StreamExt, TryStreamExt, stream};
 use lance_arrow::{BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, FieldExt};
 use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry};
+use lance_io::scheduler::{FileScheduler, ScanScheduler, SchedulerConfig};
 use object_store::path::Path;
 use tokio::io::AsyncWriteExt;
-use tokio::sync::Mutex;
+use tokio::sync::{Mutex, OnceCell, oneshot};
 use url::Url;
 
 use super::take::TakeBuilder;
+use super::write::ExternalBlobMode;
 use super::{Dataset, ProjectionRequest};
 use arrow_array::StructArray;
 use lance_core::datatypes::{BlobKind, BlobVersion};
 use lance_core::utils::blob::blob_path;
 use lance_core::{Error, Result, utils::address::RowAddress};
-use lance_io::traits::{Reader, Writer};
+use lance_io::traits::{Reader, WriteExt, Writer};
+use lance_io::utils::CachedFileSize;
 
 const INLINE_MAX: usize = 64 * 1024; // 64KB inline cutoff
 const DEDICATED_THRESHOLD: usize = 4 * 1024 * 1024; // 4MB dedicated cutoff
 const PACK_FILE_MAX_SIZE: usize = 1024 * 1024 * 1024; // 1GiB per .pack sidecar
-
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub(super) struct ResolvedExternalBase {
     pub base_id: u32,
@@ -39,25 +51,23 @@ pub(super) struct ExternalBaseCandidate {
     pub base_id: u32,
     pub store_prefix: String,
     pub base_path: Path,
+    pub store_params: ObjectStoreParams,
 }
 
 #[derive(Debug)]
 pub(super) struct ExternalBaseResolver {
     candidates: Vec<ExternalBaseCandidate>,
     store_registry: Arc<ObjectStoreRegistry>,
-    store_params: ObjectStoreParams,
 }
 
 impl ExternalBaseResolver {
     pub(super) fn new(
         candidates: Vec<ExternalBaseCandidate>,
         store_registry: Arc<ObjectStoreRegistry>,
-        store_params: ObjectStoreParams,
     ) -> Self {
         Self {
             candidates,
             store_registry,
-            store_params,
         }
     }
 
@@ -65,13 +75,13 @@ impl ExternalBaseResolver {
         &self,
         uri: &str,
     ) -> Result<Option<ResolvedExternalBase>> {
-        let uri_store_prefix = self
-            .store_registry
-            .calculate_object_store_prefix(uri, self.store_params.storage_options())?;
         let uri_path = ObjectStore::extract_path_from_uri(self.store_registry.clone(), uri)?;
 
         let mut best_match: Option<(usize, ResolvedExternalBase)> = None;
         for candidate in &self.candidates {
+            let uri_store_prefix = self
+                .store_registry
+                .calculate_object_store_prefix(uri, candidate.store_params.storage_options())?;
             if candidate.store_prefix != uri_store_prefix {
                 continue;
             }
@@ -151,12 +161,12 @@ impl PackWriter {
     async fn write_with_allocator<F>(
         &mut self,
         alloc_blob_id: &mut F,
-        data: &[u8],
+        source: BlobWriteSource<'_>,
     ) -> Result<(u32, u64)>
     where
         F: FnMut() -> u32,
     {
-        let len = data.len();
+        let len = source.size();
         if self
             .current_blob_id
             .map(|_| self.current_size + len > self.max_pack_size)
@@ -169,7 +179,7 @@ impl PackWriter {
 
         let writer = self.writer.as_mut().expect("pack writer is initialized");
         let position = self.current_size as u64;
-        writer.write_all(data).await?;
+        source.write_to(writer.as_mut()).await?;
         self.current_size += len;
         Ok((self.current_blob_id.expect("pack blob id"), position))
     }
@@ -200,9 +210,98 @@ pub struct BlobPreprocessor {
     writer_metadata: Vec<HashMap<String, String>>,
     external_base_resolver: Option<Arc<ExternalBaseResolver>>,
     allow_external_blob_outside_bases: bool,
+    external_blob_mode: ExternalBlobMode,
+    source_store_registry: Arc<ObjectStoreRegistry>,
+    source_store_params: ObjectStoreParams,
+}
+
+/// A logical slice of an external blob that can be materialized or streamed into Lance-managed
+/// storage.
+struct ExternalBlobSource {
+    reader: Box<dyn Reader>,
+    start: u64,
+    size: u64,
+}
+
+/// A blob payload source used by packed and dedicated writers.
+///
+/// Inline blobs still need an in-memory byte slice because they are embedded into the descriptor
+/// array, while external ingest can stream bytes from the source reader.
+enum BlobWriteSource<'a> {
+    Bytes(&'a [u8]),
+    External(&'a ExternalBlobSource),
+}
+
+impl ExternalBlobSource {
+    /// Return the logical payload size after applying any external slice.
+    fn size(&self) -> u64 {
+        self.size
+    }
+
+    /// Convert the logical slice into the current reader API's usize-based range.
+    fn reader_range(&self) -> Result<Range<usize>> {
+        let start = usize::try_from(self.start).map_err(|_| {
+            Error::invalid_input(format!(
+                "External blob position {} does not fit into usize",
+                self.start
+            ))
+        })?;
+        let size = usize::try_from(self.size).map_err(|_| {
+            Error::invalid_input(format!(
+                "External blob size {} does not fit into usize",
+                self.size
+            ))
+        })?;
+        let end = start.checked_add(size).ok_or_else(|| {
+            Error::invalid_input(format!(
+                "External blob range overflows usize: position={}, size={}",
+                self.start, self.size
+            ))
+        })?;
+        Ok(start..end)
+    }
+
+    /// Materialize the slice into memory for the inline blob path.
+    async fn read_all(&self) -> Result<bytes::Bytes> {
+        let range = self.reader_range()?;
+        self.reader.get_range(range).await.map_err(Into::into)
+    }
+
+    /// Stream the slice into a writer for packed or dedicated blob storage.
+    async fn copy_to_writer(&self, writer: &mut dyn Writer) -> Result<()> {
+        let range = self.reader_range()?;
+        writer
+            .copy_range_from_reader(self.reader.as_ref(), range)
+            .await?;
+        Ok(())
+    }
+}
+
+impl BlobWriteSource<'_> {
+    /// Return the payload size regardless of whether bytes come from memory or an external reader.
+    fn size(&self) -> usize {
+        match self {
+            Self::Bytes(data) => data.len(),
+            Self::External(source) => usize::try_from(source.size())
+                .expect("packed and inline external blobs must fit into usize"),
+        }
+    }
+
+    /// Write the payload into Lance-managed storage without forcing callers to branch on source
+    /// type.
+    async fn write_to(&self, writer: &mut dyn Writer) -> Result<()> {
+        match self {
+            Self::Bytes(data) => {
+                writer.write_all(data).await?;
+                Ok(())
+            }
+            Self::External(source) => source.copy_to_writer(writer).await,
+        }
+    }
 }
 
 impl BlobPreprocessor {
+    #[allow(clippy::too_many_arguments)]
     pub(super) fn new(
         object_store: ObjectStore,
         data_dir: Path,
@@ -210,12 +309,19 @@ impl BlobPreprocessor {
         schema: &lance_core::datatypes::Schema,
         external_base_resolver: Option<Arc<ExternalBaseResolver>>,
         allow_external_blob_outside_bases: bool,
+        external_blob_mode: ExternalBlobMode,
+        source_store_registry: Arc<ObjectStoreRegistry>,
+        source_store_params: ObjectStoreParams,
+        pack_file_size_threshold: Option<usize>,
     ) -> Self {
-        let pack_writer = PackWriter::new(
+        let mut pack_writer = PackWriter::new(
             object_store.clone(),
             data_dir.clone(),
             data_file_key.clone(),
         );
+        if let Some(max_bytes) = pack_file_size_threshold {
+            pack_writer.max_pack_size = max_bytes;
+        }
         let arrow_schema = arrow_schema::Schema::from(schema);
         let fields = arrow_schema.fields();
         let blob_v2_cols = fields.iter().map(|field| field.is_blob_v2()).collect();
@@ -239,6 +345,9 @@ impl BlobPreprocessor {
             writer_metadata,
             external_base_resolver,
             allow_external_blob_outside_bases,
+            external_blob_mode,
+            source_store_registry,
+            source_store_params,
         }
     }
 
@@ -248,15 +357,15 @@ impl BlobPreprocessor {
         id
     }
 
-    async fn write_dedicated(&mut self, blob_id: u32, data: &[u8]) -> Result<Path> {
+    async fn write_dedicated(&mut self, blob_id: u32, source: BlobWriteSource<'_>) -> Result<Path> {
         let path = blob_path(&self.data_dir, &self.data_file_key, blob_id);
         let mut writer = self.object_store.create(&path).await?;
-        writer.write_all(data).await?;
+        source.write_to(writer.as_mut()).await?;
         Writer::shutdown(&mut writer).await?;
         Ok(path)
     }
 
-    async fn write_packed(&mut self, data: &[u8]) -> Result<(u32, u64)> {
+    async fn write_packed(&mut self, source: BlobWriteSource<'_>) -> Result<(u32, u64)> {
         let (counter, pack_writer) = (&mut self.local_counter, &mut self.pack_writer);
         pack_writer
             .write_with_allocator(
@@ -265,7 +374,7 @@ impl BlobPreprocessor {
                     *counter += 1;
                     id
                 },
-                data,
+                source,
             )
             .await
     }
@@ -291,6 +400,48 @@ impl BlobPreprocessor {
         )))
     }
 
+    async fn open_external_source(
+        &mut self,
+        uri: &str,
+        position: Option<u64>,
+        size: Option<u64>,
+    ) -> Result<ExternalBlobSource> {
+        let (object_store, path) = ObjectStore::from_uri_and_params(
+            self.source_store_registry.clone(),
+            uri,
+            &self.source_store_params,
+        )
+        .await?;
+        let reader = object_store.open(&path).await?;
+        match (position, size) {
+            (Some(position), Some(size)) => {
+                position.checked_add(size).ok_or_else(|| {
+                    Error::invalid_input(format!(
+                        "External blob range overflows u64: position={}, size={}",
+                        position, size
+                    ))
+                })?;
+                Ok(ExternalBlobSource {
+                    reader,
+                    start: position,
+                    size,
+                })
+            }
+            (None, None) => {
+                let size = reader.size().await? as u64;
+                Ok(ExternalBlobSource {
+                    reader,
+                    start: 0,
+                    size,
+                })
+            }
+            _ => Err(Error::invalid_input(format!(
+                "External blob URI '{}' must set both position and size when slicing for ingest",
+                uri
+            ))),
+        }
+    }
+
     pub(crate) async fn preprocess_batch(&mut self, batch: &RecordBatch) -> Result<RecordBatch> {
         let expected_columns = self.blob_v2_cols.len();
         if batch.num_columns() != expected_columns {
@@ -374,7 +525,8 @@ impl BlobPreprocessor {
                 let dedicated_threshold = self.dedicated_thresholds[idx];
                 if has_data && data_len > dedicated_threshold {
                     let blob_id = self.next_blob_id();
-                    self.write_dedicated(blob_id, data_col.value(i)).await?;
+                    self.write_dedicated(blob_id, BlobWriteSource::Bytes(data_col.value(i)))
+                        .await?;
 
                     kind_builder.append_value(BlobKind::Dedicated as u8);
                     data_builder.append_null();
@@ -386,7 +538,9 @@ impl BlobPreprocessor {
                 }
 
                 if has_data && data_len > INLINE_MAX {
-                    let (pack_blob_id, position) = self.write_packed(data_col.value(i)).await?;
+                    let (pack_blob_id, position) = self
+                        .write_packed(BlobWriteSource::Bytes(data_col.value(i)))
+                        .await?;
 
                     kind_builder.append_value(BlobKind::Packed as u8);
                     data_builder.append_null();
@@ -399,6 +553,64 @@ impl BlobPreprocessor {
 
                 if has_uri {
                     let uri_val = uri_col.value(i);
+                    if self.external_blob_mode == ExternalBlobMode::Ingest {
+                        let position = if has_position {
+                            Some(
+                                position_col
+                                    .as_ref()
+                                    .expect("position column must exist")
+                                    .value(i),
+                            )
+                        } else {
+                            None
+                        };
+                        let size = if has_size {
+                            Some(size_col.as_ref().expect("size column must exist").value(i))
+                        } else {
+                            None
+                        };
+                        let source = self.open_external_source(uri_val, position, size).await?;
+                        let data_len = source.size();
+
+                        if data_len > dedicated_threshold as u64 {
+                            let blob_id = self.next_blob_id();
+                            self.write_dedicated(blob_id, BlobWriteSource::External(&source))
+                                .await?;
+
+                            kind_builder.append_value(BlobKind::Dedicated as u8);
+                            data_builder.append_null();
+                            uri_builder.append_null();
+                            blob_id_builder.append_value(blob_id);
+                            blob_size_builder.append_value(data_len);
+                            position_builder.append_null();
+                            continue;
+                        }
+
+                        if data_len > INLINE_MAX as u64 {
+                            let (pack_blob_id, position) = self
+                                .write_packed(BlobWriteSource::External(&source))
+                                .await?;
+
+                            kind_builder.append_value(BlobKind::Packed as u8);
+                            data_builder.append_null();
+                            uri_builder.append_null();
+                            blob_id_builder.append_value(pack_blob_id);
+                            blob_size_builder.append_value(data_len);
+                            position_builder.append_value(position);
+                            continue;
+                        }
+
+                        let data = source.read_all().await?;
+
+                        kind_builder.append_value(BlobKind::Inline as u8);
+                        data_builder.append_value(data.as_ref());
+                        uri_builder.append_null();
+                        blob_id_builder.append_null();
+                        blob_size_builder.append_null();
+                        position_builder.append_null();
+                        continue;
+                    }
+
                     let (external_base_id, external_uri_or_path) =
                         self.resolve_external_reference(uri_val).await?;
                     kind_builder.append_value(BlobKind::External as u8);
@@ -509,29 +721,267 @@ pub async fn preprocess_blob_batches(
     Ok(out)
 }
 
-/// Current state of the reader.  Held in a mutex for easy sharing
+/// Mutable state for a [`BlobFile`] cursor.
 ///
-/// The u64 is the cursor in the file that the reader is currently at
-/// (note that seeks are allowed before the file is opened)
+/// The cursor is logical to the blob slice, not the backing object. Once closed,
+/// subsequent cursor-based and range-based reads are rejected, but reads that
+/// were already in flight may still complete.
 #[derive(Debug)]
-enum ReaderState {
-    Uninitialized(u64),
-    Open((u64, Arc<dyn Reader>)),
+enum BlobFileState {
+    Open(u64),
     Closed,
 }
 
-/// A file-like object that represents a blob in a dataset
+/// Shared physical read context for blob handles that resolve to the same object.
+///
+/// Blob descriptors are logical slices over a backing object (data file, packed
+/// sidecar, dedicated sidecar, or external object). Multiple [`BlobFile`] values
+/// can point at different regions of that same object. This struct gives those
+/// handles a single lazy-open scheduler plus a lightweight pending queue so
+/// concurrent reads can be opportunistically grouped before reaching Lance's
+/// existing I/O scheduler.
 #[derive(Debug)]
-pub struct BlobFile {
+struct BlobSource {
     object_store: Arc<ObjectStore>,
     path: Path,
-    reader: Arc<Mutex<ReaderState>>,
+    file_size: CachedFileSize,
+    scheduler: OnceCell<FileScheduler>,
+    pending_reads: Mutex<PendingBlobReads>,
+}
+
+impl BlobSource {
+    /// Create a shared read context for one physical backing object.
+    fn new(object_store: Arc<ObjectStore>, path: Path) -> Self {
+        Self {
+            object_store,
+            path,
+            file_size: CachedFileSize::unknown(),
+            scheduler: OnceCell::new(),
+            pending_reads: Mutex::new(PendingBlobReads::default()),
+        }
+    }
+
+    /// Read one or more physical ranges from this source.
+    ///
+    /// Concurrent callers enqueue their requests into `pending_reads`. The first
+    /// caller in a drain cycle becomes the leader and spawns the batch drain task.
+    /// The mutex critical section only updates in-memory queue bookkeeping; the
+    /// actual scheduler submission happens after the lock is released.
+    async fn read_ranges(self: &Arc<Self>, ranges: Vec<Range<u64>>) -> Result<Vec<Bytes>> {
+        if ranges.is_empty() {
+            return Ok(Vec::new());
+        }
+
+        let scheduler = self
+            .scheduler
+            .get_or_try_init(|| async {
+                ScanScheduler::new(
+                    self.object_store.clone(),
+                    SchedulerConfig::max_bandwidth(self.object_store.as_ref()),
+                )
+                .open_file(&self.path, &self.file_size)
+                .await
+            })
+            .await?;
+
+        let (response_tx, response_rx) = oneshot::channel();
+        let should_spawn = {
+            let mut pending_reads = self.pending_reads.lock().await;
+            pending_reads.requests.push(PendingBlobRead {
+                ranges,
+                response: response_tx,
+            });
+            if pending_reads.is_draining {
+                false
+            } else {
+                pending_reads.is_draining = true;
+                true
+            }
+        };
+
+        if should_spawn {
+            let source = self.clone();
+            let scheduler = scheduler.clone();
+            tokio::spawn(async move {
+                let result = AssertUnwindSafe(source.clone().drain_pending_reads(scheduler))
+                    .catch_unwind()
+                    .await;
+                if let Err(panic) = result {
+                    let mut pending_reads = source.pending_reads.lock().await;
+                    pending_reads.is_draining = false;
+                    std::panic::resume_unwind(panic);
+                }
+            });
+        }
+
+        response_rx.await.map_err(|_| {
+            Error::internal("Blob source read task dropped the response".to_string())
+        })?
+    }
+
+    /// Drain currently queued requests and submit them as scheduler batches.
+    ///
+    /// Each loop iteration grabs the queued requests with a short mutex hold and
+    /// immediately releases the lock before any I/O is awaited.
+    async fn drain_pending_reads(self: Arc<Self>, scheduler: FileScheduler) {
+        loop {
+            let batch = {
+                let mut pending_reads = self.pending_reads.lock().await;
+                if pending_reads.requests.is_empty() {
+                    pending_reads.is_draining = false;
+                    return;
+                }
+                std::mem::take(&mut pending_reads.requests)
+            };
+            fulfill_pending_blob_reads(&scheduler, batch).await;
+        }
+    }
+}
+
+/// Queue of pending logical blob reads for one [`BlobSource`].
+///
+/// `is_draining` marks whether a leader task is already draining the queue.
+#[derive(Default, Debug)]
+struct PendingBlobReads {
+    requests: Vec<PendingBlobRead>,
+    is_draining: bool,
+}
+
+/// Pending logical blob reads waiting to be grouped into one scheduler batch.
+///
+/// This queue exists only to combine overlapping concurrent calls. The actual
+/// coalescing and physical I/O scheduling still happens in [`FileScheduler`].
+#[derive(Debug)]
+struct PendingBlobRead {
+    ranges: Vec<Range<u64>>,
+    response: oneshot::Sender<Result<Vec<Bytes>>>,
+}
+
+/// Submit one grouped batch of pending blob reads to Lance's [`FileScheduler`].
+///
+/// The function flattens all logical requests into one range list, preserves the
+/// caller-visible order for each request, and fans the bytes back out after the
+/// scheduler completes its own merge / split logic.
+async fn fulfill_pending_blob_reads(scheduler: &FileScheduler, batch: Vec<PendingBlobRead>) {
+    let total_ranges = batch
+        .iter()
+        .map(|request| request.ranges.len())
+        .sum::<usize>();
+    let mut request_ranges = Vec::with_capacity(total_ranges);
+    let mut response = batch
+        .iter()
+        .map(|request| vec![Bytes::new(); request.ranges.len()])
+        .collect::<Vec<_>>();
+
+    for (request_idx, request) in batch.iter().enumerate() {
+        for (range_idx, range) in request.ranges.iter().enumerate() {
+            if range.is_empty() {
+                continue;
+            }
+            request_ranges.push((range.clone(), request_idx, range_idx));
+        }
+    }
+
+    let result = if request_ranges.is_empty() {
+        Ok(())
+    } else {
+        request_ranges.sort_by_key(|(range, _, _)| (range.start, range.end));
+        let priority = request_ranges[0].0.start;
+        match scheduler
+            .submit_request(
+                request_ranges
+                    .iter()
+                    .map(|(range, _, _)| range.clone())
+                    .collect::<Vec<_>>(),
+                priority,
+            )
+            .await
+        {
+            Ok(bytes_vec) => {
+                for ((_, request_idx, range_idx), bytes) in
+                    request_ranges.into_iter().zip(bytes_vec)
+                {
+                    response[request_idx][range_idx] = bytes;
+                }
+                Ok(())
+            }
+            Err(err) => Err(err),
+        }
+    };
+
+    match result {
+        Ok(()) => {
+            for (request, bytes) in batch.into_iter().zip(response) {
+                let _ = request.response.send(Ok(bytes));
+            }
+        }
+        Err(err) => {
+            let message = format!(
+                "Failed to read blob source {}: {}",
+                scheduler.reader().path(),
+                err
+            );
+            for request in batch {
+                let _ = request.response.send(Err(Error::io(message.clone())));
+            }
+        }
+    }
+}
+
+/// Cache key for sharing one [`BlobSource`] across multiple blob descriptors.
+///
+/// We include the store prefix as well as the path so the same path string in
+/// different object stores is never conflated.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+struct BlobSourceKey {
+    store_prefix: String,
+    path: String,
+}
+
+impl BlobSourceKey {
+    /// Build the cache key for one shared [`BlobSource`].
+    fn new(source: &BlobSource) -> Self {
+        Self {
+            store_prefix: source.object_store.store_prefix.clone(),
+            path: source.path.to_string(),
+        }
+    }
+}
+
+/// Return a shared [`BlobSource`] for the given physical object.
+///
+/// This keeps all blob handles that resolve to the same `(store, path)` on a
+/// single lazy-open scheduler and pending-read queue.
+fn shared_blob_source(
+    source_cache: &mut HashMap<BlobSourceKey, Arc<BlobSource>>,
+    object_store: Arc<ObjectStore>,
+    path: &Path,
+) -> Arc<BlobSource> {
+    let key = BlobSourceKey {
+        store_prefix: object_store.store_prefix.clone(),
+        path: path.to_string(),
+    };
+    source_cache
+        .entry(key)
+        .or_insert_with(|| Arc::new(BlobSource::new(object_store, path.clone())))
+        .clone()
+}
+
+/// A file-like object that represents a blob in a dataset
+#[derive(Debug)]
+pub struct BlobFile {
+    source: Arc<BlobSource>,
+    state: Arc<Mutex<BlobFileState>>,
     position: u64,
     size: u64,
     kind: BlobKind,
     uri: Option<String>,
 }
 
+/// Base-aware physical location metadata used while resolving blob reads.
+///
+/// This is cached per fragment so repeated rows from the same fragment do not
+/// recompute the object store, data directory, and data file key.
 #[derive(Clone)]
 struct BlobReadLocation {
     object_store: Arc<ObjectStore>,
@@ -541,22 +991,20 @@ struct BlobReadLocation {
 }
 
 impl BlobFile {
-    fn with_location(
-        object_store: Arc<ObjectStore>,
-        path: Path,
+    fn with_source(
+        source: Arc<BlobSource>,
         position: u64,
         size: u64,
         kind: BlobKind,
         uri: Option<String>,
     ) -> Self {
         Self {
-            object_store,
-            path,
+            source,
             position,
             size,
             kind,
             uri,
-            reader: Arc::new(Mutex::new(ReaderState::Uninitialized(0))),
+            state: Arc::new(Mutex::new(BlobFileState::Open(0))),
         }
     }
 
@@ -577,7 +1025,13 @@ impl BlobFile {
         position: u64,
         size: u64,
     ) -> Self {
-        Self::with_location(object_store, path, position, size, BlobKind::Inline, None)
+        Self::with_source(
+            Arc::new(BlobSource::new(object_store, path)),
+            position,
+            size,
+            BlobKind::Inline,
+            None,
+        )
     }
 
     /// Create a dedicated blob reader backed by a sidecar `.blob` file.
@@ -591,7 +1045,13 @@ impl BlobFile {
     /// * `path` - Full path to the dedicated sidecar blob file.
     /// * `size` - Total byte length to expose from the sidecar file.
     pub fn new_dedicated(object_store: Arc<ObjectStore>, path: Path, size: u64) -> Self {
-        Self::with_location(object_store, path, 0, size, BlobKind::Dedicated, None)
+        Self::with_source(
+            Arc::new(BlobSource::new(object_store, path)),
+            0,
+            size,
+            BlobKind::Dedicated,
+            None,
+        )
     }
 
     /// Create a packed blob reader for a slice inside a shared sidecar `.blob` file.
@@ -611,7 +1071,13 @@ impl BlobFile {
         position: u64,
         size: u64,
     ) -> Self {
-        Self::with_location(object_store, path, position, size, BlobKind::Packed, None)
+        Self::with_source(
+            Arc::new(BlobSource::new(object_store, path)),
+            position,
+            size,
+            BlobKind::Packed,
+            None,
+        )
     }
 
     /// Create an external blob reader backed by a caller-resolved object location.
@@ -634,9 +1100,8 @@ impl BlobFile {
         position: u64,
         size: u64,
     ) -> Self {
-        Self::with_location(
-            object_store,
-            path,
+        Self::with_source(
+            Arc::new(BlobSource::new(object_store, path)),
             position,
             size,
             BlobKind::External,
@@ -646,55 +1111,118 @@ impl BlobFile {
 
     /// Close the blob file, releasing any associated resources
     pub async fn close(&self) -> Result<()> {
-        let mut reader = self.reader.lock().await;
-        *reader = ReaderState::Closed;
+        let mut state = self.state.lock().await;
+        *state = BlobFileState::Closed;
         Ok(())
     }
 
     /// Returns true if the blob file is closed
     pub async fn is_closed(&self) -> bool {
-        matches!(*self.reader.lock().await, ReaderState::Closed)
+        matches!(*self.state.lock().await, BlobFileState::Closed)
     }
 
-    async fn do_with_reader<
-        T,
-        Fut: Future<Output = Result<(u64, T)>>,
-        Func: FnOnce(u64, Arc<dyn Reader>) -> Fut,
-    >(
+    async fn do_with_cursor<T, Fut: Future<Output = Result<(u64, T)>>, Func: FnOnce(u64) -> Fut>(
         &self,
         func: Func,
     ) -> Result<T> {
-        let mut reader = self.reader.lock().await;
-        if let ReaderState::Uninitialized(cursor) = *reader {
-            let opened = self.object_store.open(&self.path).await?;
-            let opened = Arc::<dyn Reader>::from(opened);
-            *reader = ReaderState::Open((cursor, opened.clone()));
-        }
-        match reader.deref_mut() {
-            ReaderState::Open((cursor, reader)) => {
-                let (new_cursor, data) = func(*cursor, reader.clone()).await?;
+        let mut state = self.state.lock().await;
+        match state.deref_mut() {
+            BlobFileState::Open(cursor) => {
+                let (new_cursor, data) = func(*cursor).await?;
                 *cursor = new_cursor;
                 Ok(data)
             }
-            ReaderState::Closed => Err(Error::invalid_input(
+            BlobFileState::Closed => Err(Error::invalid_input(
+                "Blob file is already closed".to_string(),
+            )),
+        }
+    }
+
+    async fn ensure_open(&self) -> Result<()> {
+        let state = self.state.lock().await;
+        match *state {
+            BlobFileState::Open(_) => Ok(()),
+            BlobFileState::Closed => Err(Error::invalid_input(
                 "Blob file is already closed".to_string(),
             )),
-            _ => unreachable!(),
         }
     }
 
+    fn read_phys_range(&self, range: Range<u64>) -> Result<Range<u64>> {
+        if range.start > range.end {
+            return Err(Error::invalid_input(format!(
+                "Blob range start {} must be <= end {}",
+                range.start, range.end
+            )));
+        }
+        if range.end > self.size {
+            return Err(Error::invalid_input(format!(
+                "Blob range end {} exceeds blob size {}",
+                range.end, self.size
+            )));
+        }
+        let start = self.position.checked_add(range.start).ok_or_else(|| {
+            Error::invalid_input(format!(
+                "Blob range start overflowed physical position: base={} offset={}",
+                self.position, range.start
+            ))
+        })?;
+        let end = self.position.checked_add(range.end).ok_or_else(|| {
+            Error::invalid_input(format!(
+                "Blob range end overflowed physical position: base={} offset={}",
+                self.position, range.end
+            ))
+        })?;
+        Ok(start..end)
+    }
+
+    /// Read a byte range relative to the beginning of this blob without changing the cursor.
+    ///
+    /// The provided range is interpreted in blob-local coordinates, not object
+    /// coordinates. Empty ranges are allowed. This method is intended for random
+    /// access callers that want deterministic range semantics instead of the
+    /// stateful file-like cursor used by [`Self::read`] and [`Self::read_up_to`].
+    pub async fn read_range(&self, range: Range<u64>) -> Result<Bytes> {
+        let mut data = self.read_ranges(&[range]).await?;
+        Ok(data.pop().unwrap_or_default())
+    }
+
+    /// Read multiple ranges relative to the beginning of this blob without changing the cursor.
+    ///
+    /// Empty ranges are allowed and yield empty buffers. The result order always
+    /// matches the input order, even though the underlying physical requests may
+    /// be reordered, coalesced, or split for efficiency.
+    pub async fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
+        self.ensure_open().await?;
+        let physical_ranges = ranges
+            .iter()
+            .cloned()
+            .map(|range| self.read_phys_range(range))
+            .collect::<Result<Vec<_>>>()?;
+        self.source.read_ranges(physical_ranges).await
+    }
+
     /// Read the entire blob file from the current cursor position
     /// to the end of the file
     ///
     /// After this call the cursor will be pointing to the end of
     /// the file.
     pub async fn read(&self) -> Result<bytes::Bytes> {
-        let position = self.position;
         let size = self.size;
-        self.do_with_reader(|cursor, reader| async move {
-            let start = position as usize + cursor as usize;
-            let end = (position + size) as usize;
-            Ok((end as u64, reader.get_range(start..end).await?))
+        let source = self.source.clone();
+        let position = self.position;
+        self.do_with_cursor(move |cursor| {
+            let source = source.clone();
+            async move {
+                if cursor >= size {
+                    return Ok((size, Bytes::new()));
+                }
+                let physical = (position + cursor)..(position + size);
+                Ok((
+                    size,
+                    source.read_ranges(vec![physical]).await?.pop().unwrap(),
+                ))
+            }
         })
         .await
     }
@@ -704,45 +1232,47 @@ impl BlobFile {
     /// After this call the cursor will be pointing to the end of
     /// the read data.
     pub async fn read_up_to(&self, len: usize) -> Result<bytes::Bytes> {
-        let position = self.position;
         let size = self.size;
-        self.do_with_reader(|cursor, reader| async move {
-            let start = position as usize + cursor as usize;
-            let read_size = len.min((size - cursor) as usize);
-            let end = start + read_size;
-            let data = reader.get_range(start..end).await?;
-            Ok((end as u64 - position, data))
+        let source = self.source.clone();
+        let position = self.position;
+        self.do_with_cursor(move |cursor| {
+            let source = source.clone();
+            async move {
+                if cursor >= size || len == 0 {
+                    return Ok((size.min(cursor), Bytes::new()));
+                }
+                let read_size = len.min((size - cursor) as usize) as u64;
+                let start = position + cursor;
+                let end = start + read_size;
+                let data = source.read_ranges(vec![start..end]).await?.pop().unwrap();
+                Ok((cursor + read_size, data))
+            }
         })
         .await
     }
 
     /// Seek to a new cursor position in the file
     pub async fn seek(&self, new_cursor: u64) -> Result<()> {
-        let mut reader = self.reader.lock().await;
-        match reader.deref_mut() {
-            ReaderState::Open((cursor, _)) => {
+        let mut state = self.state.lock().await;
+        match state.deref_mut() {
+            BlobFileState::Open(cursor) => {
                 *cursor = new_cursor;
                 Ok(())
             }
-            ReaderState::Closed => Err(Error::invalid_input(
+            BlobFileState::Closed => Err(Error::invalid_input(
                 "Blob file is already closed".to_string(),
             )),
-            ReaderState::Uninitialized(cursor) => {
-                *cursor = new_cursor;
-                Ok(())
-            }
         }
     }
 
     /// Return the current cursor position in the file
     pub async fn tell(&self) -> Result<u64> {
-        let reader = self.reader.lock().await;
-        match *reader {
-            ReaderState::Open((cursor, _)) => Ok(cursor),
-            ReaderState::Closed => Err(Error::invalid_input(
+        let state = self.state.lock().await;
+        match *state {
+            BlobFileState::Open(cursor) => Ok(cursor),
+            BlobFileState::Closed => Err(Error::invalid_input(
                 "Blob file is already closed".to_string(),
             )),
-            ReaderState::Uninitialized(cursor) => Ok(cursor),
         }
     }
 
@@ -756,7 +1286,7 @@ impl BlobFile {
     }
 
     pub fn data_path(&self) -> &Path {
-        &self.path
+        &self.source.path
     }
 
     pub fn kind(&self) -> BlobKind {
@@ -768,34 +1298,361 @@ impl BlobFile {
     }
 }
 
-pub(super) async fn take_blobs(
-    dataset: &Arc<Dataset>,
-    row_ids: &[u64],
-    column: &str,
-) -> Result<Vec<BlobFile>> {
-    let projection = dataset.schema().project(&[column])?;
-    let blob_field = &projection.fields[0];
-    let blob_field_id = blob_field.id;
-    if !projection.fields[0].is_blob() {
-        return Err(Error::invalid_input_source(
-            format!("the column '{}' is not a blob column", column).into(),
-        ));
+/// Blob bytes materialized by [`ReadBlobsBuilder`].
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ReadBlob {
+    /// Row address of the blob that was read.
+    pub row_address: u64,
+    /// Blob payload bytes.
+    pub data: Bytes,
+}
+
+/// Stream returned by [`ReadBlobsBuilder::try_into_stream`].
+pub type ReadBlobsStream = BoxStream<'static, Result<ReadBlob>>;
+
+/// Row selector configured on [`ReadBlobsBuilder`].
+#[derive(Debug, Clone)]
+enum ReadBlobsSelection {
+    None,
+    RowIds(Vec<u64>),
+    RowIndices(Vec<u64>),
+    RowAddresses(Vec<u64>),
+}
+
+/// Planner knobs for [`ReadBlobsBuilder`].
+///
+/// Options that shape how `read_blobs` uses Lance's existing schedulers.
+#[derive(Debug, Clone)]
+struct ReadBlobsOptions {
+    io_buffer_size_bytes: Option<u64>,
+    preserve_order: bool,
+}
+
+impl Default for ReadBlobsOptions {
+    fn default() -> Self {
+        Self {
+            io_buffer_size_bytes: None,
+            preserve_order: true,
+        }
     }
-    let description_and_addr = dataset
-        .take_builder(row_ids, projection)?
-        .with_row_address(true)
-        .execute()
+}
+
+/// Builder for sequential / planned blob reads.
+///
+/// Unlike [`Dataset::take_blobs`], which returns [`BlobFile`] handles for
+/// caller-driven random access, this builder plans object-store reads across a
+/// selected row set and yields fully materialized blob payloads.
+#[derive(Debug, Clone)]
+pub struct ReadBlobsBuilder {
+    dataset: Arc<Dataset>,
+    column: String,
+    blob_field_id: u32,
+    selection: ReadBlobsSelection,
+    options: ReadBlobsOptions,
+}
+
+impl ReadBlobsBuilder {
+    pub(crate) fn new(dataset: Arc<Dataset>, column: String, blob_field_id: u32) -> Self {
+        Self {
+            dataset,
+            column,
+            blob_field_id,
+            selection: ReadBlobsSelection::None,
+            options: ReadBlobsOptions::default(),
+        }
+    }
+
+    /// Read blobs for the provided stable row ids.
+    pub fn with_row_ids(mut self, row_ids: impl Into<Vec<u64>>) -> Self {
+        self.selection = ReadBlobsSelection::RowIds(row_ids.into());
+        self
+    }
+
+    /// Read blobs for the provided row offsets in dataset order.
+    pub fn with_row_indices(mut self, row_indices: impl Into<Vec<u64>>) -> Self {
+        self.selection = ReadBlobsSelection::RowIndices(row_indices.into());
+        self
+    }
+
+    /// Read blobs for the provided physical row addresses.
+    pub fn with_row_addresses(mut self, row_addrs: impl Into<Vec<u64>>) -> Self {
+        self.selection = ReadBlobsSelection::RowAddresses(row_addrs.into());
+        self
+    }
+
+    /// Set the scheduler I/O buffer size used while materializing blobs.
+    pub fn with_io_buffer_size_bytes(mut self, bytes: u64) -> Self {
+        self.options.io_buffer_size_bytes = Some(bytes);
+        self
+    }
+
+    /// Whether results must follow the caller's requested row order.
+    pub fn preserve_order(mut self, preserve: bool) -> Self {
+        self.options.preserve_order = preserve;
+        self
+    }
+
+    /// Execute the planned blob read and return a stream of blob payloads.
+    ///
+    /// The stream yields one [`ReadBlob`] per selected non-null blob row.
+    pub async fn try_into_stream(self) -> Result<ReadBlobsStream> {
+        self.validate()?;
+        let entries = collect_blob_entries_for_selection(
+            &self.dataset,
+            self.blob_field_id,
+            &self.column,
+            &self.selection,
+        )
         .await?;
-    let descriptions = description_and_addr.column(0).as_struct();
-    let row_addrs = description_and_addr.column(1).as_primitive::<UInt64Type>();
-    let blob_field_id = blob_field_id as u32;
+        let expected_selection_indices = entries
+            .iter()
+            .map(|entry| entry.selection_index)
+            .collect::<VecDeque<_>>();
+        let plans = plan_blob_read_plans(entries);
+        let execution = Arc::new(ReadBlobsExecution::new(self.options.io_buffer_size_bytes));
+        if plans.is_empty() {
+            return Ok(stream::empty().boxed());
+        }
 
-    match blob_version_from_descriptions(descriptions)? {
-        BlobVersion::V1 => collect_blob_files_v1(dataset, blob_field_id, descriptions, row_addrs),
-        BlobVersion::V2 => {
-            collect_blob_files_v2(dataset, blob_field_id, descriptions, row_addrs).await
+        let plan_stream = stream::iter(plans.into_iter().map(move |plan| {
+            let execution = execution.clone();
+            execute_blob_read_plan(plan, execution)
+        }))
+        .buffer_unordered(self.dataset.object_store.io_parallelism().max(1));
+
+        if !self.options.preserve_order {
+            return Ok(plan_stream
+                .map_ok(|blobs| {
+                    stream::iter(blobs.into_iter().map(|blob| Ok(into_read_blob(blob))))
+                })
+                .try_flatten()
+                .boxed());
+        }
+
+        let mut plan_stream = plan_stream.boxed();
+        let mut expected_selection_indices = expected_selection_indices;
+        let mut ready = BTreeMap::<usize, ReadBlob>::new();
+
+        Ok(stream::poll_fn(move |cx| {
+            loop {
+                let Some(next_selection_index) = expected_selection_indices.front().copied() else {
+                    return Poll::Ready(None);
+                };
+
+                if let Some(blob) = ready.remove(&next_selection_index) {
+                    expected_selection_indices.pop_front();
+                    return Poll::Ready(Some(Ok(blob)));
+                }
+
+                match plan_stream.poll_next_unpin(cx) {
+                    Poll::Ready(Some(Ok(blobs))) => {
+                        for blob in blobs {
+                            ready.insert(blob.selection_index, into_read_blob(blob));
+                        }
+                    }
+                    Poll::Ready(Some(Err(err))) => {
+                        return Poll::Ready(Some(Err(err)));
+                    }
+                    Poll::Ready(None) => {
+                        let err = Error::internal(format!(
+                            "planned blob read stream completed before selection index {} was produced",
+                            next_selection_index
+                        ));
+                        return Poll::Ready(Some(Err(err)));
+                    }
+                    Poll::Pending => return Poll::Pending,
+                }
+            }
+        })
+        .boxed())
+    }
+
+    /// Execute the planned blob read and collect the full result in memory.
+    pub async fn execute(self) -> Result<Vec<ReadBlob>> {
+        self.try_into_stream().await?.try_collect().await
+    }
+
+    fn validate(&self) -> Result<()> {
+        match self.selection {
+            ReadBlobsSelection::None => Err(Error::invalid_input(
+                "ReadBlobsBuilder requires a row selection; call one of with_row_ids, with_row_indices, or with_row_addresses".to_string(),
+            )),
+            _ if self.options.io_buffer_size_bytes == Some(0) => Err(Error::invalid_input(
+                "ReadBlobsBuilder io_buffer_size must be greater than 0".to_string(),
+            )),
+            _ => Ok(()),
+        }
+    }
+}
+
+/// One logical blob selected for planned reading.
+#[derive(Debug)]
+struct BlobEntry {
+    selection_index: usize,
+    row_address: u64,
+    file: BlobFile,
+}
+
+/// Physical read input derived from one [`BlobEntry`].
+#[derive(Debug)]
+struct PlannedBlobRead {
+    selection_index: usize,
+    row_address: u64,
+    physical_range: Range<u64>,
+}
+
+/// One per-source read plan emitted by `read_blobs`.
+#[derive(Debug)]
+struct BlobReadPlan {
+    source_key: BlobSourceKey,
+    source: Arc<BlobSource>,
+    reads: Vec<PlannedBlobRead>,
+}
+
+/// Operation-scoped scheduler cache for one [`ReadBlobsBuilder`] execution.
+///
+/// We reuse one [`ScanScheduler`] per object store during a single `read_blobs`
+/// operation and still submit exactly one request per physical file.
+#[derive(Debug)]
+struct ReadBlobsExecution {
+    io_buffer_size_bytes: Option<u64>,
+    schedulers: std::sync::Mutex<HashMap<String, Arc<ScanScheduler>>>,
+}
+
+impl ReadBlobsExecution {
+    fn new(io_buffer_size_bytes: Option<u64>) -> Self {
+        Self {
+            io_buffer_size_bytes,
+            schedulers: std::sync::Mutex::new(HashMap::new()),
         }
     }
+
+    fn scheduler_for(&self, source: &BlobSource) -> Arc<ScanScheduler> {
+        let mut schedulers = self.schedulers.lock().unwrap();
+        schedulers
+            .entry(source.object_store.store_prefix.clone())
+            .or_insert_with(|| {
+                let config = self
+                    .io_buffer_size_bytes
+                    .map(SchedulerConfig::new)
+                    .unwrap_or_else(|| {
+                        SchedulerConfig::max_bandwidth(source.object_store.as_ref())
+                    });
+                ScanScheduler::new(source.object_store.clone(), config)
+            })
+            .clone()
+    }
+}
+
+/// Materialized blob bytes plus the original selection index used to restore
+/// caller ordering after per-source reads complete.
+#[derive(Debug)]
+struct IndexedReadBlob {
+    selection_index: usize,
+    row_address: u64,
+    data: Bytes,
+}
+
+fn into_read_blob(blob: IndexedReadBlob) -> ReadBlob {
+    ReadBlob {
+        row_address: blob.row_address,
+        data: blob.data,
+    }
+}
+
+/// Group selected blobs by physical source and sort each group's ranges by
+/// physical offset before handing them to the file scheduler.
+fn plan_blob_read_plans(entries: Vec<BlobEntry>) -> Vec<BlobReadPlan> {
+    let mut plan_indices = HashMap::<BlobSourceKey, usize>::new();
+    let mut plans = Vec::<BlobReadPlan>::new();
+
+    for entry in entries {
+        let source_key = BlobSourceKey::new(&entry.file.source);
+        let plan_index = if let Some(plan_index) = plan_indices.get(&source_key) {
+            *plan_index
+        } else {
+            let plan_index = plans.len();
+            plans.push(BlobReadPlan {
+                source_key: source_key.clone(),
+                source: entry.file.source.clone(),
+                reads: Vec::new(),
+            });
+            plan_indices.insert(source_key.clone(), plan_index);
+            plan_index
+        };
+
+        plans[plan_index].reads.push(PlannedBlobRead {
+            selection_index: entry.selection_index,
+            row_address: entry.row_address,
+            physical_range: entry.file.position..(entry.file.position + entry.file.size),
+        });
+    }
+
+    plans.sort_by(|left, right| {
+        left.source_key
+            .store_prefix
+            .cmp(&right.source_key.store_prefix)
+            .then_with(|| left.source_key.path.cmp(&right.source_key.path))
+    });
+
+    for plan in &mut plans {
+        plan.reads.sort_by(|left, right| {
+            left.physical_range
+                .start
+                .cmp(&right.physical_range.start)
+                .then_with(|| left.physical_range.end.cmp(&right.physical_range.end))
+                .then_with(|| left.selection_index.cmp(&right.selection_index))
+        });
+    }
+
+    plans
+}
+
+/// Execute one per-source blob read plan with a single scheduler submission.
+async fn execute_blob_read_plan(
+    task: BlobReadPlan,
+    execution: Arc<ReadBlobsExecution>,
+) -> Result<Vec<IndexedReadBlob>> {
+    let ranges = task
+        .reads
+        .iter()
+        .map(|read| read.physical_range.clone())
+        .collect::<Vec<_>>();
+    let scheduler = execution.scheduler_for(&task.source);
+    let file_scheduler = scheduler
+        .open_file(&task.source.path, &task.source.file_size)
+        .await?;
+    let priority = ranges[0].start;
+    let bytes = file_scheduler.submit_request(ranges, priority).await?;
+
+    Ok(task
+        .reads
+        .into_iter()
+        .zip(bytes)
+        .map(|(read, data)| IndexedReadBlob {
+            selection_index: read.selection_index,
+            row_address: read.row_address,
+            data,
+        })
+        .collect())
+}
+
+pub(super) async fn take_blobs(
+    dataset: &Arc<Dataset>,
+    row_ids: &[u64],
+    column: &str,
+) -> Result<Vec<BlobFile>> {
+    let blob_field_id = validate_blob_column(dataset, column)?;
+    Ok(collect_blob_entries_for_selection(
+        dataset,
+        blob_field_id,
+        column,
+        &ReadBlobsSelection::RowIds(row_ids.to_vec()),
+    )
+    .await?
+    .into_iter()
+    .map(|entry| entry.file)
+    .collect())
 }
 
 /// Take [BlobFile] by row addresses.
@@ -810,38 +1667,99 @@ pub async fn take_blobs_by_addresses(
     row_addrs: &[u64],
     column: &str,
 ) -> Result<Vec<BlobFile>> {
+    let blob_field_id = validate_blob_column(dataset, column)?;
+    Ok(collect_blob_entries_for_selection(
+        dataset,
+        blob_field_id,
+        column,
+        &ReadBlobsSelection::RowAddresses(row_addrs.to_vec()),
+    )
+    .await?
+    .into_iter()
+    .map(|entry| entry.file)
+    .collect())
+}
+
+/// Validate that `column` exists and is a blob column, returning its field id.
+pub(super) fn validate_blob_column(dataset: &Arc<Dataset>, column: &str) -> Result<u32> {
     let projection = dataset.schema().project(&[column])?;
     let blob_field = &projection.fields[0];
-    let blob_field_id = blob_field.id;
-    if !projection.fields[0].is_blob() {
+    if !blob_field.is_blob() {
         return Err(Error::invalid_input_source(
             format!("the column '{}' is not a blob column", column).into(),
         ));
     }
+    Ok(blob_field.id as u32)
+}
 
-    // Convert Schema to ProjectionPlan
+/// Load blob descriptor rows for a stable-row-id selection.
+async fn take_blob_descriptions_by_row_ids(
+    dataset: &Arc<Dataset>,
+    row_ids: &[u64],
+    column: &str,
+) -> Result<RecordBatch> {
+    let projection = dataset.schema().project(&[column])?;
+    dataset
+        .take_builder(row_ids, projection)?
+        .with_row_address(true)
+        .execute()
+        .await
+}
+
+/// Load blob descriptor rows for a physical-row-address selection.
+async fn take_blob_descriptions_by_row_addresses(
+    dataset: &Arc<Dataset>,
+    row_addrs: &[u64],
+    column: &str,
+) -> Result<RecordBatch> {
+    let projection = dataset.schema().project(&[column])?;
     let projection_request = ProjectionRequest::from(projection);
     let projection_plan = Arc::new(projection_request.into_projection_plan(dataset.clone())?);
+    TakeBuilder::try_new_from_addresses(dataset.clone(), row_addrs.to_vec(), projection_plan)?
+        .with_row_address(true)
+        .execute()
+        .await
+}
 
-    // Use try_new_from_addresses to bypass row ID index lookup.
-    // This is critical when enable_stable_row_ids=true because row addresses
-    // (fragment_id << 32 | row_offset) are different from row IDs (sequential integers).
-    let description_and_addr =
-        TakeBuilder::try_new_from_addresses(dataset.clone(), row_addrs.to_vec(), projection_plan)?
-            .with_row_address(true)
-            .execute()
-            .await?;
+/// Resolve a caller selection into [`BlobEntry`] values that share `BlobSource`
+/// instances by physical backing object.
+async fn collect_blob_entries_for_selection(
+    dataset: &Arc<Dataset>,
+    blob_field_id: u32,
+    column: &str,
+    selection: &ReadBlobsSelection,
+) -> Result<Vec<BlobEntry>> {
+    let description_and_addr = match selection {
+        ReadBlobsSelection::None => {
+            return Err(Error::invalid_input(
+                "Blob row selection is required".to_string(),
+            ));
+        }
+        ReadBlobsSelection::RowIds(row_ids) => {
+            take_blob_descriptions_by_row_ids(dataset, row_ids, column).await?
+        }
+        ReadBlobsSelection::RowIndices(row_indices) => {
+            let row_addrs =
+                super::take::row_offsets_to_row_addresses(&dataset.get_fragments(), row_indices)
+                    .await?;
+            take_blob_descriptions_by_row_addresses(dataset, &row_addrs, column).await?
+        }
+        ReadBlobsSelection::RowAddresses(row_addrs) => {
+            take_blob_descriptions_by_row_addresses(dataset, row_addrs, column).await?
+        }
+    };
+
+    if description_and_addr.num_rows() == 0 {
+        return Ok(Vec::new());
+    }
 
     let descriptions = description_and_addr.column(0).as_struct();
-    let row_addrs_result = description_and_addr.column(1).as_primitive::<UInt64Type>();
-    let blob_field_id = blob_field_id as u32;
+    let row_addrs = description_and_addr.column(1).as_primitive::<UInt64Type>();
 
     match blob_version_from_descriptions(descriptions)? {
-        BlobVersion::V1 => {
-            collect_blob_files_v1(dataset, blob_field_id, descriptions, row_addrs_result)
-        }
+        BlobVersion::V1 => collect_blob_entries_v1(dataset, blob_field_id, descriptions, row_addrs),
         BlobVersion::V2 => {
-            collect_blob_files_v2(dataset, blob_field_id, descriptions, row_addrs_result).await
+            collect_blob_entries_v2(dataset, blob_field_id, descriptions, row_addrs).await
         }
     }
 }
@@ -867,41 +1785,68 @@ fn blob_version_from_descriptions(descriptions: &StructArray) -> Result<BlobVers
     .into()))
 }
 
-fn collect_blob_files_v1(
+/// Convert blob v1 descriptors into logical blob entries.
+fn collect_blob_entries_v1(
     dataset: &Arc<Dataset>,
     blob_field_id: u32,
     descriptions: &StructArray,
     row_addrs: &arrow::array::PrimitiveArray<UInt64Type>,
-) -> Result<Vec<BlobFile>> {
+) -> Result<Vec<BlobEntry>> {
     let positions = descriptions.column(0).as_primitive::<UInt64Type>();
     let sizes = descriptions.column(1).as_primitive::<UInt64Type>();
-
-    Ok(row_addrs
+    let mut source_cache = HashMap::<BlobSourceKey, Arc<BlobSource>>::new();
+    row_addrs
         .values()
         .iter()
         .zip(positions.iter())
         .zip(sizes.iter())
-        .filter_map(|((row_addr, position), size)| {
+        .enumerate()
+        .filter_map(|(selection_index, ((row_addr, position), size))| {
             let position = position?;
             let size = size?;
-            Some((*row_addr, position, size))
+            Some((selection_index, *row_addr, position, size))
         })
-        .map(|(row_addr, position, size)| {
+        .map(|(selection_index, row_addr, position, size)| {
             let frag_id = RowAddress::from(row_addr).fragment_id();
-            let frag = dataset.get_fragment(frag_id as usize).unwrap();
-            let data_file = frag.data_file_for_field(blob_field_id).unwrap();
+            let frag = dataset.get_fragment(frag_id as usize).ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "Blob row address {} references missing fragment {}",
+                    row_addr, frag_id
+                ))
+            })?;
+            let data_file = frag.data_file_for_field(blob_field_id).ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "Blob field {} has no data file in fragment {} for row address {}",
+                    blob_field_id, frag_id, row_addr
+                ))
+            })?;
             let data_file_path = dataset.data_dir().child(data_file.path.as_str());
-            BlobFile::new_inline(dataset.object_store.clone(), data_file_path, position, size)
+            Ok(BlobEntry {
+                selection_index,
+                row_address: row_addr,
+                file: BlobFile::with_source(
+                    shared_blob_source(
+                        &mut source_cache,
+                        dataset.object_store.clone(),
+                        &data_file_path,
+                    ),
+                    position,
+                    size,
+                    BlobKind::Inline,
+                    None,
+                ),
+            })
         })
-        .collect())
+        .collect()
 }
 
-async fn collect_blob_files_v2(
+/// Convert blob v2 descriptors into logical blob entries.
+async fn collect_blob_entries_v2(
     dataset: &Arc<Dataset>,
     blob_field_id: u32,
     descriptions: &StructArray,
     row_addrs: &arrow::array::PrimitiveArray<UInt64Type>,
-) -> Result<Vec<BlobFile>> {
+) -> Result<Vec<BlobEntry>> {
     let kinds = descriptions.column(0).as_primitive::<UInt8Type>();
     let positions = descriptions.column(1).as_primitive::<UInt64Type>();
     let sizes = descriptions.column(2).as_primitive::<UInt64Type>();
@@ -912,7 +1857,9 @@ async fn collect_blob_files_v2(
     let mut fragment_cache = HashMap::<u32, BlobReadLocation>::new();
     let mut store_cache = HashMap::<u32, Arc<ObjectStore>>::new();
     let mut external_base_path_cache = HashMap::<u32, Path>::new();
-    for (idx, row_addr) in row_addrs.values().iter().enumerate() {
+    let mut source_cache = HashMap::<BlobSourceKey, Arc<BlobSource>>::new();
+    for (selection_index, row_addr) in row_addrs.values().iter().enumerate() {
+        let idx = selection_index;
         let kind = BlobKind::try_from(kinds.value(idx))?;
 
         // Struct is non-nullable; null rows are encoded as inline with zero position/size and empty uri
@@ -932,12 +1879,16 @@ async fn collect_blob_files_v2(
                     &mut store_cache,
                 )
                 .await?;
-                files.push(BlobFile::new_inline(
+                let source = shared_blob_source(
+                    &mut source_cache,
                     location.object_store,
-                    location.data_file_path,
-                    position,
-                    size,
-                ));
+                    &location.data_file_path,
+                );
+                files.push(BlobEntry {
+                    selection_index,
+                    row_address: *row_addr,
+                    file: BlobFile::with_source(source, position, size, BlobKind::Inline, None),
+                });
             }
             BlobKind::Dedicated => {
                 let blob_id = blob_ids.value(idx);
@@ -951,7 +1902,12 @@ async fn collect_blob_files_v2(
                 )
                 .await?;
                 let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id);
-                files.push(BlobFile::new_dedicated(location.object_store, path, size));
+                let source = shared_blob_source(&mut source_cache, location.object_store, &path);
+                files.push(BlobEntry {
+                    selection_index,
+                    row_address: *row_addr,
+                    file: BlobFile::with_source(source, 0, size, BlobKind::Dedicated, None),
+                });
             }
             BlobKind::Packed => {
                 let blob_id = blob_ids.value(idx);
@@ -966,12 +1922,12 @@ async fn collect_blob_files_v2(
                 )
                 .await?;
                 let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id);
-                files.push(BlobFile::new_packed(
-                    location.object_store,
-                    path,
-                    position,
-                    size,
-                ));
+                let source = shared_blob_source(&mut source_cache, location.object_store, &path);
+                files.push(BlobEntry {
+                    selection_index,
+                    row_address: *row_addr,
+                    file: BlobFile::with_source(source, position, size, BlobKind::Packed, None),
+                });
             }
             BlobKind::External => {
                 let uri_or_path = blob_uris.value(idx).to_string();
@@ -1015,13 +1971,18 @@ async fn collect_blob_files_v2(
                 } else {
                     object_store.size(&path).await?
                 };
-                files.push(BlobFile::new_external(
-                    object_store,
-                    path,
-                    uri_or_path,
-                    position,
-                    size,
-                ));
+                let source = shared_blob_source(&mut source_cache, object_store, &path);
+                files.push(BlobEntry {
+                    selection_index,
+                    row_address: *row_addr,
+                    file: BlobFile::with_source(
+                        source,
+                        position,
+                        size,
+                        BlobKind::External,
+                        Some(uri_or_path),
+                    ),
+                });
             }
         }
     }
@@ -1111,21 +2072,39 @@ fn data_file_key_from_path(path: &str) -> &str {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+    use std::ops::Range;
     use std::sync::Arc;
+    use std::time::Duration;
 
     use arrow::{
         array::AsArray,
         datatypes::{UInt8Type, UInt32Type, UInt64Type},
     };
     use arrow_array::RecordBatch;
-    use arrow_array::{RecordBatchIterator, UInt32Array};
+    use arrow_array::{
+        ArrayRef, RecordBatchIterator, StringArray, StructArray, UInt32Array, UInt64Array,
+    };
     use arrow_schema::{DataType, Field, Schema};
-    use futures::TryStreamExt;
-    use lance_arrow::{BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, DataTypeExt};
+    use async_trait::async_trait;
+    use bytes::Bytes;
+    use chrono::Utc;
+    use futures::{StreamExt, TryStreamExt, future::try_join_all};
+    use lance_arrow::{
+        ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_V2_EXT_NAME, DataTypeExt,
+    };
     use lance_core::datatypes::BlobKind;
-    use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry};
+    use lance_io::object_store::{
+        ObjectStore, ObjectStoreParams, ObjectStoreRegistry, StorageOptionsAccessor,
+    };
     use lance_io::stream::RecordBatchStream;
     use lance_table::format::BasePath;
+    use object_store::{
+        Attributes, GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload,
+        ObjectMeta, PutMultipartOptions, PutOptions, PutPayload, PutResult, path::Path,
+    };
+    use tokio::sync::Notify;
+    use url::Url;
 
     use lance_core::{
         Error, Result,
@@ -1134,24 +2113,417 @@ mod tests {
     use lance_datagen::{BatchCount, RowCount, array};
     use lance_file::version::LanceFileVersion;
 
-    use super::data_file_key_from_path;
-    use crate::{
-        Dataset,
-        blob::{BlobArrayBuilder, blob_field},
-        dataset::WriteParams,
-        utils::test::TestDatasetGenerator,
-    };
+    use super::{
+        BlobEntry, BlobFile, BlobSource, ExternalBaseCandidate, ExternalBaseResolver,
+        ReadBlobsExecution, collect_blob_entries_v1, data_file_key_from_path,
+        execute_blob_read_plan, plan_blob_read_plans,
+    };
+    use crate::{
+        Dataset,
+        blob::{BlobArrayBuilder, blob_field},
+        dataset::{ExternalBlobMode, WriteParams},
+        utils::test::TestDatasetGenerator,
+    };
+
+    struct BlobTestFixture {
+        _test_dir: TempStrDir,
+        dataset: Arc<Dataset>,
+        data: Vec<RecordBatch>,
+    }
+
+    struct MultiBaseBlobFixture {
+        _test_dir: TempDir,
+        dataset: Arc<Dataset>,
+        expected: Vec<u8>,
+    }
+
+    #[cfg(feature = "azure")]
+    fn azure_store_params(account_name: &str) -> ObjectStoreParams {
+        ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("account_name".to_string(), account_name.to_string()),
+                    ("account_key".to_string(), "dGVzdA==".to_string()),
+                ]),
+            ))),
+            ..Default::default()
+        }
+    }
+
+    #[derive(Debug)]
+    struct RejectEmptyRangeObjectStore;
+
+    #[cfg(feature = "azure")]
+    #[tokio::test]
+    async fn test_external_base_resolver_uses_candidate_store_params() {
+        let store_registry = Arc::new(ObjectStoreRegistry::default());
+        let base_a = BasePath::new(
+            1,
+            "az://container/path-a".to_string(),
+            Some("base-a".to_string()),
+            false,
+        );
+        let base_b = BasePath::new(
+            2,
+            "az://container/path-b".to_string(),
+            Some("base-b".to_string()),
+            false,
+        );
+
+        let base_a_params = azure_store_params("account-a");
+        let base_b_params = azure_store_params("account-b");
+
+        let (store_a, extracted_a) =
+            ObjectStore::from_uri_and_params(store_registry.clone(), &base_a.path, &base_a_params)
+                .await
+                .unwrap();
+        let (store_b, extracted_b) =
+            ObjectStore::from_uri_and_params(store_registry.clone(), &base_b.path, &base_b_params)
+                .await
+                .unwrap();
+
+        let resolver = ExternalBaseResolver::new(
+            vec![
+                ExternalBaseCandidate {
+                    base_id: base_a.id,
+                    store_prefix: store_a.store_prefix.clone(),
+                    base_path: extracted_a,
+                    store_params: base_a_params,
+                },
+                ExternalBaseCandidate {
+                    base_id: base_b.id,
+                    store_prefix: store_b.store_prefix.clone(),
+                    base_path: extracted_b,
+                    store_params: base_b_params,
+                },
+            ],
+            store_registry,
+        );
+
+        let resolved_a = resolver
+            .resolve_external_uri("az://container/path-a/file.bin")
+            .await
+            .unwrap()
+            .unwrap();
+        let resolved_b = resolver
+            .resolve_external_uri("az://container/path-b/file.bin")
+            .await
+            .unwrap()
+            .unwrap();
+
+        assert_eq!(resolved_a.base_id, 1);
+        assert_eq!(resolved_a.relative_path, "file.bin");
+        assert_eq!(resolved_b.base_id, 2);
+        assert_eq!(resolved_b.relative_path, "file.bin");
+    }
+
+    impl std::fmt::Display for RejectEmptyRangeObjectStore {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "RejectEmptyRangeObjectStore")
+        }
+    }
+
+    #[async_trait]
+    impl object_store::ObjectStore for RejectEmptyRangeObjectStore {
+        async fn put(
+            &self,
+            _location: &Path,
+            _bytes: PutPayload,
+        ) -> object_store::Result<PutResult> {
+            unimplemented!("put is not used by these tests")
+        }
+
+        async fn put_opts(
+            &self,
+            _location: &Path,
+            _bytes: PutPayload,
+            _opts: PutOptions,
+        ) -> object_store::Result<PutResult> {
+            unimplemented!("put_opts is not used by these tests")
+        }
+
+        async fn put_multipart(
+            &self,
+            _location: &Path,
+        ) -> object_store::Result<Box<dyn MultipartUpload>> {
+            unimplemented!("put_multipart is not used by these tests")
+        }
+
+        async fn put_multipart_opts(
+            &self,
+            _location: &Path,
+            _opts: PutMultipartOptions,
+        ) -> object_store::Result<Box<dyn MultipartUpload>> {
+            unimplemented!("put_multipart_opts is not used by these tests")
+        }
+
+        async fn get(&self, _location: &Path) -> object_store::Result<GetResult> {
+            Err(object_store::Error::NotSupported {
+                source: "get is not used by these tests".into(),
+            })
+        }
+
+        async fn get_opts(
+            &self,
+            location: &Path,
+            options: GetOptions,
+        ) -> object_store::Result<GetResult> {
+            let Some(GetRange::Bounded(range)) = options.range else {
+                unreachable!("blob reads should always request a bounded range")
+            };
+            if range.start == range.end {
+                return Err(object_store::Error::Generic {
+                    store: "RejectEmptyRangeObjectStore",
+                    source: format!(
+                        "Range started at {} and ended at {}",
+                        range.start, range.end
+                    )
+                    .into(),
+                });
+            }
+            Err(object_store::Error::NotSupported {
+                source: format!("unexpected non-empty range {range:?} for {location}").into(),
+            })
+        }
+
+        async fn delete(&self, _location: &Path) -> object_store::Result<()> {
+            unimplemented!("delete is not used by these tests")
+        }
+
+        fn list(
+            &self,
+            _prefix: Option<&Path>,
+        ) -> futures::stream::BoxStream<'static, object_store::Result<ObjectMeta>> {
+            unimplemented!("list is not used by these tests")
+        }
+
+        async fn list_with_delimiter(
+            &self,
+            _prefix: Option<&Path>,
+        ) -> object_store::Result<ListResult> {
+            unimplemented!("list_with_delimiter is not used by these tests")
+        }
+
+        async fn copy(&self, _from: &Path, _to: &Path) -> object_store::Result<()> {
+            unimplemented!("copy is not used by these tests")
+        }
+
+        async fn copy_if_not_exists(&self, _from: &Path, _to: &Path) -> object_store::Result<()> {
+            unimplemented!("copy_if_not_exists is not used by these tests")
+        }
+    }
+
+    fn reject_empty_range_store() -> Arc<ObjectStore> {
+        Arc::new(ObjectStore::new(
+            Arc::new(RejectEmptyRangeObjectStore) as Arc<dyn object_store::ObjectStore>,
+            Url::parse("mock:///blob-tests").unwrap(),
+            None,
+            None,
+            false,
+            true,
+            lance_io::object_store::DEFAULT_LOCAL_IO_PARALLELISM,
+            lance_io::object_store::DEFAULT_DOWNLOAD_RETRY_COUNT,
+            None,
+        ))
+    }
+
+    #[derive(Debug)]
+    struct RecordingRangeObjectStore {
+        data: Bytes,
+        gate: Option<Arc<Notify>>,
+        requested_ranges: std::sync::Mutex<Vec<Range<u64>>>,
+    }
+
+    impl RecordingRangeObjectStore {
+        fn new(data: Bytes) -> Self {
+            Self {
+                data,
+                gate: None,
+                requested_ranges: std::sync::Mutex::new(Vec::new()),
+            }
+        }
+
+        fn with_gate(data: Bytes, gate: Arc<Notify>) -> Self {
+            Self {
+                data,
+                gate: Some(gate),
+                requested_ranges: std::sync::Mutex::new(Vec::new()),
+            }
+        }
+
+        fn requested_ranges(&self) -> Vec<Range<u64>> {
+            self.requested_ranges.lock().unwrap().clone()
+        }
+
+        fn object_meta(&self, location: &Path) -> ObjectMeta {
+            ObjectMeta {
+                location: location.clone(),
+                last_modified: Utc::now(),
+                size: self.data.len() as u64,
+                e_tag: None,
+                version: None,
+            }
+        }
+    }
+
+    impl std::fmt::Display for RecordingRangeObjectStore {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "RecordingRangeObjectStore")
+        }
+    }
+
+    #[async_trait]
+    impl object_store::ObjectStore for RecordingRangeObjectStore {
+        async fn put(
+            &self,
+            _location: &Path,
+            _bytes: PutPayload,
+        ) -> object_store::Result<PutResult> {
+            unimplemented!("put is not used by these tests")
+        }
+
+        async fn put_opts(
+            &self,
+            _location: &Path,
+            _bytes: PutPayload,
+            _opts: PutOptions,
+        ) -> object_store::Result<PutResult> {
+            unimplemented!("put_opts is not used by these tests")
+        }
+
+        async fn put_multipart(
+            &self,
+            _location: &Path,
+        ) -> object_store::Result<Box<dyn MultipartUpload>> {
+            unimplemented!("put_multipart is not used by these tests")
+        }
+
+        async fn put_multipart_opts(
+            &self,
+            _location: &Path,
+            _opts: PutMultipartOptions,
+        ) -> object_store::Result<Box<dyn MultipartUpload>> {
+            unimplemented!("put_multipart_opts is not used by these tests")
+        }
+
+        async fn get(&self, location: &Path) -> object_store::Result<GetResult> {
+            self.get_opts(location, GetOptions::default()).await
+        }
+
+        async fn get_opts(
+            &self,
+            location: &Path,
+            options: GetOptions,
+        ) -> object_store::Result<GetResult> {
+            let range = match options.range {
+                Some(GetRange::Bounded(range)) => range,
+                None => 0..self.data.len() as u64,
+                Some(other) => {
+                    return Err(object_store::Error::NotSupported {
+                        source: format!("unsupported range request {other:?}").into(),
+                    });
+                }
+            };
+            if let Some(gate) = &self.gate {
+                gate.notified().await;
+            }
+            self.requested_ranges.lock().unwrap().push(range.clone());
+            let bytes = self.data.slice(range.start as usize..range.end as usize);
+            Ok(GetResult {
+                payload: GetResultPayload::Stream(
+                    futures::stream::once(async move { Ok(bytes) }).boxed(),
+                ),
+                meta: self.object_meta(location),
+                range,
+                attributes: Attributes::default(),
+            })
+        }
+
+        async fn head(&self, location: &Path) -> object_store::Result<ObjectMeta> {
+            Ok(self.object_meta(location))
+        }
+
+        async fn delete(&self, _location: &Path) -> object_store::Result<()> {
+            unimplemented!("delete is not used by these tests")
+        }
 
-    struct BlobTestFixture {
-        _test_dir: TempStrDir,
-        dataset: Arc<Dataset>,
-        data: Vec<RecordBatch>,
+        fn list(
+            &self,
+            _prefix: Option<&Path>,
+        ) -> futures::stream::BoxStream<'static, object_store::Result<ObjectMeta>> {
+            unimplemented!("list is not used by these tests")
+        }
+
+        async fn list_with_delimiter(
+            &self,
+            _prefix: Option<&Path>,
+        ) -> object_store::Result<ListResult> {
+            unimplemented!("list_with_delimiter is not used by these tests")
+        }
+
+        async fn copy(&self, _from: &Path, _to: &Path) -> object_store::Result<()> {
+            unimplemented!("copy is not used by these tests")
+        }
+
+        async fn copy_if_not_exists(&self, _from: &Path, _to: &Path) -> object_store::Result<()> {
+            unimplemented!("copy_if_not_exists is not used by these tests")
+        }
     }
 
-    struct MultiBaseBlobFixture {
-        _test_dir: TempDir,
-        dataset: Arc<Dataset>,
-        expected: Vec<u8>,
+    fn recording_range_store_with_url(
+        data: Bytes,
+        url: &str,
+    ) -> (Arc<ObjectStore>, Arc<RecordingRangeObjectStore>) {
+        const TEST_RANGE_STORE_SIZE: usize = 128 * 1024;
+        let mut padded = vec![0; TEST_RANGE_STORE_SIZE.max(data.len())];
+        padded[..data.len()].copy_from_slice(data.as_ref());
+        let inner = Arc::new(RecordingRangeObjectStore::new(Bytes::from(padded)));
+        let store = Arc::new(ObjectStore::new(
+            inner.clone() as Arc<dyn object_store::ObjectStore>,
+            Url::parse(url).unwrap(),
+            None,
+            None,
+            false,
+            true,
+            lance_io::object_store::DEFAULT_LOCAL_IO_PARALLELISM,
+            lance_io::object_store::DEFAULT_DOWNLOAD_RETRY_COUNT,
+            None,
+        ));
+        (store, inner)
+    }
+
+    fn recording_range_store(data: Bytes) -> (Arc<ObjectStore>, Arc<RecordingRangeObjectStore>) {
+        recording_range_store_with_url(data, "mock://recording/blob-range-tests")
+    }
+
+    fn gated_range_store(
+        data: Bytes,
+        url: &str,
+    ) -> (
+        Arc<ObjectStore>,
+        Arc<RecordingRangeObjectStore>,
+        Arc<Notify>,
+    ) {
+        const TEST_RANGE_STORE_SIZE: usize = 128 * 1024;
+        let mut padded = vec![0; TEST_RANGE_STORE_SIZE.max(data.len())];
+        padded[..data.len()].copy_from_slice(data.as_ref());
+        let gate = Arc::new(Notify::new());
+        let inner = Arc::new(RecordingRangeObjectStore::with_gate(
+            Bytes::from(padded),
+            gate.clone(),
+        ));
+        let store = Arc::new(ObjectStore::new(
+            inner.clone() as Arc<dyn object_store::ObjectStore>,
+            Url::parse(url).unwrap(),
+            None,
+            None,
+            false,
+            true,
+            lance_io::object_store::DEFAULT_LOCAL_IO_PARALLELISM,
+            lance_io::object_store::DEFAULT_DOWNLOAD_RETRY_COUNT,
+            None,
+        ));
+        (store, inner, gate)
     }
 
     impl BlobTestFixture {
@@ -1283,6 +2655,46 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_read_blobs_requires_selection() {
+        let fixture = BlobTestFixture::new().await;
+
+        let err = fixture.dataset.read_blobs("blobs").unwrap().execute().await;
+
+        assert!(matches!(err, Err(Error::InvalidInput { .. })));
+        assert!(
+            err.unwrap_err()
+                .to_string()
+                .contains("requires a row selection")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_read_blobs_by_indices_execute() {
+        let fixture = BlobTestFixture::new().await;
+        let indices = vec![2, 12, 22];
+
+        let blobs = fixture
+            .dataset
+            .read_blobs("blobs")
+            .unwrap()
+            .with_row_indices(indices)
+            .execute()
+            .await
+            .unwrap();
+
+        assert_eq!(blobs.len(), 3);
+        for (actual_idx, (expected_batch_idx, expected_row_idx)) in
+            [(0, 2), (1, 2), (2, 2)].iter().enumerate()
+        {
+            let expected = fixture.data[*expected_batch_idx]
+                .column(1)
+                .as_binary::<i64>()
+                .value(*expected_row_idx);
+            assert_eq!(blobs[actual_idx].data.as_ref(), expected);
+        }
+    }
+
     #[tokio::test]
     pub async fn test_take_blobs_by_indices() {
         let fixture = BlobTestFixture::new().await;
@@ -1329,6 +2741,30 @@ mod tests {
         assert!(matches!(err, Err(Error::InvalidInput { .. })));
     }
 
+    #[tokio::test]
+    async fn test_collect_blob_entries_v1_rejects_missing_fragment() {
+        let fixture = BlobTestFixture::new().await;
+        let blob_field_id =
+            fixture.dataset.schema().project(&["blobs"]).unwrap().fields[0].id as u32;
+        let descriptions = StructArray::from(vec![
+            (
+                Arc::new(Field::new("position", DataType::UInt64, false)),
+                Arc::new(UInt64Array::from(vec![1])) as ArrayRef,
+            ),
+            (
+                Arc::new(Field::new("size", DataType::UInt64, false)),
+                Arc::new(UInt64Array::from(vec![3])) as ArrayRef,
+            ),
+        ]);
+        let row_addrs = UInt64Array::from(vec![(999_u64 << 32) | 7]);
+
+        let err =
+            collect_blob_entries_v1(&fixture.dataset, blob_field_id, &descriptions, &row_addrs)
+                .unwrap_err();
+
+        assert!(err.to_string().contains("references missing fragment"));
+    }
+
     #[tokio::test]
     pub async fn test_take_blob_not_blob_col() {
         let fixture = BlobTestFixture::new().await;
@@ -1518,6 +2954,184 @@ mod tests {
         assert_eq!(second.as_ref(), b"world");
     }
 
+    #[tokio::test]
+    async fn test_blob_file_read_empty_range_returns_empty_bytes() {
+        let store = reject_empty_range_store();
+        let path = Path::from("blobs/test.bin");
+
+        let empty_blob = BlobFile::new_packed(store.clone(), path.clone(), 1, 0);
+        assert!(empty_blob.read().await.unwrap().is_empty());
+        assert!(empty_blob.read_up_to(16).await.unwrap().is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_blob_file_read_tracks_relative_cursor() {
+        let test_dir = TempDir::default();
+        let file_path = test_dir.std_path().join("blob.bin");
+        std::fs::write(&file_path, b"abcd").unwrap();
+
+        let path = Path::from_absolute_path(file_path).unwrap();
+        let blob = BlobFile::new_packed(Arc::new(ObjectStore::local()), path, 1, 2);
+
+        assert_eq!(blob.read().await.unwrap().as_ref(), b"bc");
+        assert_eq!(blob.tell().await.unwrap(), 2);
+        assert!(blob.read().await.unwrap().is_empty());
+        assert!(blob.read_up_to(1).await.unwrap().is_empty());
+        assert_eq!(blob.tell().await.unwrap(), 2);
+    }
+
+    #[tokio::test]
+    async fn test_blob_file_read_range_does_not_change_cursor() {
+        let (store, _) = recording_range_store(Bytes::from_static(b"abcdefgh"));
+        let path = Path::from("blobs/test.bin");
+        let blob = BlobFile::new_packed(store, path, 1, 6);
+
+        let bytes = blob.read_range(2..5).await.unwrap();
+        assert_eq!(bytes.as_ref(), b"def");
+        assert_eq!(blob.tell().await.unwrap(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_blob_file_read_ranges_preserves_input_order() {
+        let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij"));
+        let path = Path::from("blobs/test.bin");
+        let blob = BlobFile::new_packed(store, path, 1, 6);
+
+        let chunks = blob.read_ranges(&[4..6, 0..2, 2..4, 2..2]).await.unwrap();
+        assert_eq!(chunks[0].as_ref(), b"fg");
+        assert_eq!(chunks[1].as_ref(), b"bc");
+        assert_eq!(chunks[2].as_ref(), b"de");
+        assert!(chunks[3].is_empty());
+        assert_eq!(inner.requested_ranges(), vec![1..7]);
+    }
+
+    #[tokio::test]
+    async fn test_blob_file_read_range_rejects_out_of_bounds() {
+        let (store, _) = recording_range_store(Bytes::from_static(b"abcdef"));
+        let path = Path::from("blobs/test.bin");
+        let blob = BlobFile::new_packed(store, path, 0, 4);
+
+        let err = blob.read_range(1..5).await.unwrap_err();
+        assert!(err.to_string().contains("exceeds blob size"));
+    }
+
+    #[tokio::test]
+    async fn test_blob_files_share_source_and_coalesce() {
+        let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij"));
+        let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin")));
+        let blob1 = BlobFile::with_source(source.clone(), 1, 3, BlobKind::Packed, None);
+        let blob2 = BlobFile::with_source(source, 4, 3, BlobKind::Packed, None);
+
+        let (data1, data2) = tokio::join!(blob1.read(), blob2.read());
+        assert_eq!(data1.unwrap().as_ref(), b"bcd");
+        assert_eq!(data2.unwrap().as_ref(), b"efg");
+        assert_eq!(inner.requested_ranges(), vec![1..7]);
+    }
+
+    #[tokio::test]
+    async fn test_read_blobs_plan_preserves_order_and_coalesces() {
+        let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij"));
+        let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin")));
+        let entries = vec![
+            BlobEntry {
+                selection_index: 0,
+                row_address: 10,
+                file: BlobFile::with_source(source.clone(), 4, 3, BlobKind::Packed, None),
+            },
+            BlobEntry {
+                selection_index: 1,
+                row_address: 11,
+                file: BlobFile::with_source(source, 1, 3, BlobKind::Packed, None),
+            },
+        ];
+        let execution = Arc::new(ReadBlobsExecution::new(None));
+        let blobs = try_join_all(
+            plan_blob_read_plans(entries)
+                .into_iter()
+                .map(|plan| execute_blob_read_plan(plan, execution.clone())),
+        )
+        .await
+        .unwrap();
+        let mut blobs = blobs.into_iter().flatten().collect::<Vec<_>>();
+        blobs.sort_by_key(|blob| blob.selection_index);
+
+        assert_eq!(blobs.len(), 2);
+        assert_eq!(blobs[0].row_address, 10);
+        assert_eq!(blobs[0].data.as_ref(), b"efg");
+        assert_eq!(blobs[1].row_address, 11);
+        assert_eq!(blobs[1].data.as_ref(), b"bcd");
+        assert_eq!(inner.requested_ranges(), vec![1..7]);
+    }
+
+    #[tokio::test]
+    async fn test_read_blobs_stream_emits_ready_plan_without_waiting_for_slower_ones() {
+        let (slow_store, _, slow_gate) = gated_range_store(
+            Bytes::from_static(b"abcdef"),
+            "mock://slow/blob-range-tests",
+        );
+        let (fast_store, _) = recording_range_store_with_url(
+            Bytes::from_static(b"uvwxyz"),
+            "mock://fast/blob-range-tests",
+        );
+        let entries = vec![
+            BlobEntry {
+                selection_index: 0,
+                row_address: 10,
+                file: BlobFile::with_source(
+                    Arc::new(BlobSource::new(slow_store, Path::from("blobs/slow.bin"))),
+                    0,
+                    3,
+                    BlobKind::Packed,
+                    None,
+                ),
+            },
+            BlobEntry {
+                selection_index: 1,
+                row_address: 11,
+                file: BlobFile::with_source(
+                    Arc::new(BlobSource::new(fast_store, Path::from("blobs/fast.bin"))),
+                    0,
+                    3,
+                    BlobKind::Packed,
+                    None,
+                ),
+            },
+        ];
+        let execution = Arc::new(ReadBlobsExecution::new(None));
+        let mut stream: super::ReadBlobsStream = futures::stream::iter(
+            plan_blob_read_plans(entries)
+                .into_iter()
+                .map(move |plan| execute_blob_read_plan(plan, execution.clone())),
+        )
+        .buffer_unordered(2)
+        .map_ok(|blobs: Vec<super::IndexedReadBlob>| {
+            futures::stream::iter(
+                blobs
+                    .into_iter()
+                    .map(|blob| Ok::<super::ReadBlob, Error>(super::into_read_blob(blob))),
+            )
+        })
+        .try_flatten()
+        .boxed();
+
+        let first = tokio::time::timeout(Duration::from_secs(1), stream.next())
+            .await
+            .unwrap()
+            .unwrap()
+            .unwrap();
+        assert_eq!(first.row_address, 11);
+        assert_eq!(first.data.as_ref(), b"uvw");
+
+        slow_gate.notify_one();
+        let second = tokio::time::timeout(Duration::from_secs(1), stream.next())
+            .await
+            .unwrap()
+            .unwrap()
+            .unwrap();
+        assert_eq!(second.row_address, 10);
+        assert_eq!(second.data.as_ref(), b"abc");
+    }
+
     #[tokio::test]
     async fn test_take_blob_v2_from_non_default_base_inline() {
         let fixture = create_multi_base_blob_v2_fixture(b"inline".to_vec(), None, true).await;
@@ -1776,6 +3390,200 @@ mod tests {
         assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"mapped");
     }
 
+    #[tokio::test]
+    async fn test_blob_v2_external_ingest_inline_slice() {
+        let dataset_dir = TempDir::default();
+        let external_dir = TempDir::default();
+        let external_path = external_dir.std_path().join("external.bin");
+        std::fs::write(&external_path, b"prefix-inline-suffix").unwrap();
+        let external_uri = format!("file://{}", external_path.display());
+
+        let metadata = [(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]
+            .into_iter()
+            .collect();
+        let blob_field = Field::new(
+            "blob",
+            DataType::Struct(
+                vec![
+                    Field::new("data", DataType::LargeBinary, true),
+                    Field::new("uri", DataType::Utf8, true),
+                    Field::new("position", DataType::UInt64, true),
+                    Field::new("size", DataType::UInt64, true),
+                ]
+                .into(),
+            ),
+            true,
+        )
+        .with_metadata(metadata);
+        let blob_array: ArrayRef = Arc::new(
+            StructArray::try_new(
+                vec![
+                    Field::new("data", DataType::LargeBinary, true),
+                    Field::new("uri", DataType::Utf8, true),
+                    Field::new("position", DataType::UInt64, true),
+                    Field::new("size", DataType::UInt64, true),
+                ]
+                .into(),
+                vec![
+                    Arc::new(arrow_array::LargeBinaryArray::from(vec![None::<&[u8]>])) as ArrayRef,
+                    Arc::new(StringArray::from(vec![Some(external_uri.as_str())])) as ArrayRef,
+                    Arc::new(UInt64Array::from(vec![Some(7)])) as ArrayRef,
+                    Arc::new(UInt64Array::from(vec![Some(6)])) as ArrayRef,
+                ],
+                None,
+            )
+            .unwrap(),
+        );
+        let schema = Arc::new(Schema::new(vec![blob_field]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![blob_array]).unwrap();
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+
+        let dataset = Arc::new(
+            Dataset::write(
+                reader,
+                &dataset_dir.path_str(),
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    external_blob_mode: ExternalBlobMode::Ingest,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let desc = dataset
+            .scan()
+            .project(&["blob"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap()
+            .column(0)
+            .as_struct()
+            .to_owned();
+        assert_eq!(
+            desc.column_by_name("kind")
+                .unwrap()
+                .as_primitive::<UInt8Type>()
+                .value(0),
+            BlobKind::Inline as u8
+        );
+
+        let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
+        assert_eq!(blobs.len(), 1);
+        assert_eq!(blobs[0].kind(), BlobKind::Inline);
+        assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"inline");
+    }
+
+    #[tokio::test]
+    async fn test_blob_v2_external_ingest_packed() {
+        let dataset_dir = TempDir::default();
+        let external_dir = TempDir::default();
+        let external_path = external_dir.std_path().join("external.bin");
+        let payload = vec![0x5A; super::INLINE_MAX + 1024];
+        std::fs::write(&external_path, &payload).unwrap();
+        let external_uri = format!("file://{}", external_path.display());
+
+        let mut blob_builder = BlobArrayBuilder::new(1);
+        blob_builder.push_uri(external_uri).unwrap();
+        let blob_array: arrow_array::ArrayRef = blob_builder.finish().unwrap();
+        let schema = Arc::new(Schema::new(vec![blob_field("blob", true)]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![blob_array]).unwrap();
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+
+        let dataset = Arc::new(
+            Dataset::write(
+                reader,
+                &dataset_dir.path_str(),
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    external_blob_mode: ExternalBlobMode::Ingest,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let desc = dataset
+            .scan()
+            .project(&["blob"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap()
+            .column(0)
+            .as_struct()
+            .to_owned();
+        assert_eq!(
+            desc.column_by_name("kind")
+                .unwrap()
+                .as_primitive::<UInt8Type>()
+                .value(0),
+            BlobKind::Packed as u8
+        );
+
+        let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
+        assert_eq!(blobs.len(), 1);
+        assert_eq!(blobs[0].kind(), BlobKind::Packed);
+        assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice());
+    }
+
+    #[tokio::test]
+    async fn test_blob_v2_external_ingest_dedicated() {
+        let dataset_dir = TempDir::default();
+        let external_dir = TempDir::default();
+        let external_path = external_dir.std_path().join("external.bin");
+        let payload = vec![0x7A; super::DEDICATED_THRESHOLD + 1024];
+        std::fs::write(&external_path, &payload).unwrap();
+        let external_uri = format!("file://{}", external_path.display());
+
+        let mut blob_builder = BlobArrayBuilder::new(1);
+        blob_builder.push_uri(external_uri).unwrap();
+        let blob_array: arrow_array::ArrayRef = blob_builder.finish().unwrap();
+        let schema = Arc::new(Schema::new(vec![blob_field("blob", true)]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![blob_array]).unwrap();
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+
+        let dataset = Arc::new(
+            Dataset::write(
+                reader,
+                &dataset_dir.path_str(),
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    external_blob_mode: ExternalBlobMode::Ingest,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let desc = dataset
+            .scan()
+            .project(&["blob"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap()
+            .column(0)
+            .as_struct()
+            .to_owned();
+        assert_eq!(
+            desc.column_by_name("kind")
+                .unwrap()
+                .as_primitive::<UInt8Type>()
+                .value(0),
+            BlobKind::Dedicated as u8
+        );
+
+        let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
+        assert_eq!(blobs.len(), 1);
+        assert_eq!(blobs[0].kind(), BlobKind::Dedicated);
+        assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice());
+    }
+
     #[tokio::test]
     async fn test_blob_v2_requires_v2_2() {
         let test_dir = TempStrDir::default();
@@ -1843,6 +3651,10 @@ mod tests {
             &writer_schema,
             None,
             false,
+            ExternalBlobMode::Reference,
+            Arc::new(ObjectStoreRegistry::default()),
+            ObjectStoreParams::default(),
+            None,
         );
 
         let mut blob_builder = BlobArrayBuilder::new(1);
diff --git a/rust/lance/src/dataset/builder.rs b/rust/lance/src/dataset/builder.rs
index 36ec0e18e66..393ff45c4ea 100644
--- a/rust/lance/src/dataset/builder.rs
+++ b/rust/lance/src/dataset/builder.rs
@@ -2,6 +2,8 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 use std::{collections::HashMap, sync::Arc, time::Duration};
 
+use lance_core::cache::CacheBackend;
+
 use super::refs::{Ref, Refs};
 use super::{DEFAULT_INDEX_CACHE_SIZE, DEFAULT_METADATA_CACHE_SIZE, ReadParams, WriteParams};
 use crate::dataset::branch_location::BranchLocation;
@@ -37,6 +39,8 @@ pub struct DatasetBuilder {
     /// Metadata cache size for the fragment metadata. If it is zero, metadata
     /// cache is disabled.
     metadata_cache_size_bytes: usize,
+    /// Custom index cache backend. If set, overrides `index_cache_size_bytes`.
+    index_cache_backend: Option<Arc<dyn CacheBackend>>,
     /// Optional pre-loaded manifest to avoid loading it again.
     manifest: Option<Manifest>,
     session: Option<Arc<Session>>,
@@ -45,8 +49,10 @@ pub struct DatasetBuilder {
     version: Option<Ref>,
     table_uri: String,
     file_reader_options: Option<FileReaderOptions>,
-    /// Storage options that override user-provided options (e.g., from namespace)
+    /// Storage options that override user-provided options (e.g., from namespace client)
     storage_options_override: Option<HashMap<String, String>>,
+    /// Runtime-only exact object store bindings keyed by base path URI.
+    base_store_params: HashMap<String, ObjectStoreParams>,
 }
 
 impl std::fmt::Debug for DatasetBuilder {
@@ -64,6 +70,7 @@ impl std::fmt::Debug for DatasetBuilder {
                 "storage_options_override",
                 &self.storage_options_override.is_some(),
             )
+            .field("base_store_params", &!self.base_store_params.is_empty())
             .finish()
     }
 }
@@ -73,6 +80,7 @@ impl DatasetBuilder {
         Self {
             index_cache_size_bytes: DEFAULT_INDEX_CACHE_SIZE,
             metadata_cache_size_bytes: DEFAULT_METADATA_CACHE_SIZE,
+            index_cache_backend: None,
             table_uri: table_uri.as_ref().to_string(),
             options: ObjectStoreParams::default(),
             commit_handler: None,
@@ -81,20 +89,21 @@ impl DatasetBuilder {
             manifest: None,
             file_reader_options: None,
             storage_options_override: None,
+            base_store_params: HashMap::new(),
         }
     }
 
-    /// Create a DatasetBuilder from a LanceNamespace
+    /// Create a DatasetBuilder from a LanceNamespace client
     ///
     /// This will automatically fetch the table location and storage options from the namespace
-    /// via `describe_table()`.
+    /// client via `describe_table()`.
     ///
-    /// Storage options from the namespace will override any user-provided storage options
-    /// set via `.with_storage_options()`. This ensures the namespace is always the source
+    /// Storage options from the namespace client will override any user-provided storage options
+    /// set via `.with_storage_options()`. This ensures the namespace client is always the source
     /// of truth for storage options.
     ///
     /// # Arguments
-    /// * `namespace` - The namespace implementation to fetch table info from
+    /// * `namespace_client` - The namespace client implementation to fetch table info from
     /// * `table_id` - The table identifier (e.g., vec!["my_table"])
     ///
     /// # Example
@@ -103,14 +112,14 @@ impl DatasetBuilder {
     /// use lance::dataset::DatasetBuilder;
     ///
     /// // Connect to a REST namespace
-    /// let namespace = ConnectBuilder::new("rest")
+    /// let namespace_client = ConnectBuilder::new("rest")
     ///     .property("uri", "http://localhost:8080")
     ///     .connect()
     ///     .await?;
     ///
-    /// // Load a dataset using storage options from namespace
+    /// // Load a dataset using storage options from namespace client
     /// let dataset = DatasetBuilder::from_namespace(
-    ///     namespace,
+    ///     namespace_client,
     ///     vec!["my_table".to_string()],
     /// )
     /// .await?
@@ -119,7 +128,7 @@ impl DatasetBuilder {
     /// ```
     #[allow(deprecated)]
     pub async fn from_namespace(
-        namespace: Arc<dyn LanceNamespace>,
+        namespace_client: Arc<dyn LanceNamespace>,
         table_id: Vec<String>,
     ) -> Result<Self> {
         let request = DescribeTableRequest {
@@ -127,7 +136,7 @@ impl DatasetBuilder {
             ..Default::default()
         };
 
-        let response = namespace
+        let response = namespace_client
             .describe_table(request)
             .await
             .map_err(|e| Error::namespace_source(Box::new(e)))?;
@@ -142,8 +151,10 @@ impl DatasetBuilder {
 
         // Check managed_versioning flag to determine if namespace-managed commits should be used
         if response.managed_versioning == Some(true) {
-            let external_store =
-                LanceNamespaceExternalManifestStore::new(namespace.clone(), table_id.clone());
+            let external_store = LanceNamespaceExternalManifestStore::new(
+                namespace_client.clone(),
+                table_id.clone(),
+            );
             let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
                 external_manifest_store: Arc::new(external_store),
             });
@@ -157,7 +168,7 @@ impl DatasetBuilder {
 
         if let Some(initial_opts) = namespace_storage_options {
             let provider: Arc<dyn lance_io::object_store::StorageOptionsProvider> = Arc::new(
-                LanceNamespaceStorageOptionsProvider::new(namespace, table_id),
+                LanceNamespaceStorageOptionsProvider::new(namespace_client, table_id),
             );
             builder.options.storage_options_accessor = Some(Arc::new(
                 StorageOptionsAccessor::with_initial_and_provider(initial_opts, provider),
@@ -177,6 +188,15 @@ impl DatasetBuilder {
         self
     }
 
+    /// Use a custom index cache backend.
+    ///
+    /// When set, this overrides `with_index_cache_size_bytes` — the custom
+    /// backend is responsible for its own capacity management.
+    pub fn with_index_cache_backend(mut self, backend: Arc<dyn CacheBackend>) -> Self {
+        self.index_cache_backend = Some(backend);
+        self
+    }
+
     /// Set the cache size for indices. Set to zero, to disable the cache.
     #[deprecated(since = "0.30.0", note = "Use `with_index_cache_size_bytes` instead")]
     pub fn with_index_cache_size(mut self, cache_size: usize) -> Self {
@@ -337,10 +357,10 @@ impl DatasetBuilder {
         self
     }
 
-    /// Enable credential vending from a LanceNamespace
+    /// Enable credential vending from a LanceNamespace client
     ///
-    /// Credentials will be automatically refreshed from the namespace
-    /// before they expire. The namespace should return `expires_at_millis`
+    /// Credentials will be automatically refreshed from the namespace client
+    /// before they expire. The namespace client should return `expires_at_millis`
     /// in the storage_options from `describe_table()`.
     ///
     /// Use `with_s3_credentials_refresh_offset()` to configure how early
@@ -357,14 +377,14 @@ impl DatasetBuilder {
     /// use lance_io::object_store::{StorageOptionsProvider, LanceNamespaceStorageOptionsProvider};
     ///
     /// // Connect to a REST namespace
-    /// let namespace = ConnectBuilder::new("rest")
+    /// let namespace_client = ConnectBuilder::new("rest")
     ///     .property("uri", "http://localhost:8080")
     ///     .connect()
     ///     .await?;
     ///
-    /// // Create a storage options provider from namespace
+    /// // Create a storage options provider from namespace client
     /// let provider = Arc::new(LanceNamespaceStorageOptionsProvider::new(
-    ///     namespace,
+    ///     namespace_client,
     ///     vec!["my_table".to_string()],
     /// ));
     ///
@@ -432,6 +452,22 @@ impl DatasetBuilder {
         self
     }
 
+    /// Set runtime-only object store params for a specific registered base path.
+    ///
+    /// These params are not persisted in the manifest. They are used as-is
+    /// whenever the dataset resolves an object store for the given
+    /// `BasePath.path`. Dataset-level store params remain the fallback for bases
+    /// without an explicit binding.
+    pub fn with_base_store_params(
+        mut self,
+        base_path: impl AsRef<str>,
+        store_params: ObjectStoreParams,
+    ) -> Self {
+        self.base_store_params
+            .insert(base_path.as_ref().to_string(), store_params);
+        self
+    }
+
     /// Set options based on [ReadParams].
     pub fn with_read_params(mut self, read_params: ReadParams) -> Self {
         self = self
@@ -480,6 +516,12 @@ impl DatasetBuilder {
         self
     }
 
+    /// Set exact object store params used as the dataset-level default binding.
+    pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self {
+        self.options = store_params;
+        self
+    }
+
     /// Build a lance object store for the given config
     pub async fn build_object_store(
         self,
@@ -550,39 +592,58 @@ impl DatasetBuilder {
         }
     }
 
+    // Runtime per-base overrides are supplied as storage options, but the dataset
+    // ultimately resolves object stores from ObjectStoreParams. Normalize once in
+    // the builder so reads only need to look up the prepared params by base path.
+    fn merge_store_params_with_storage_options(
+        params: &ObjectStoreParams,
+        override_options: &HashMap<String, String>,
+    ) -> ObjectStoreParams {
+        if override_options.is_empty() {
+            return params.clone();
+        }
+
+        let mut merged_params = params.clone();
+        let mut merged_options = merged_params.storage_options().cloned().unwrap_or_default();
+        merged_options.extend(override_options.clone());
+
+        let storage_options_accessor = match merged_params
+            .storage_options_accessor
+            .as_ref()
+            .and_then(|accessor| accessor.provider().cloned())
+        {
+            Some(provider) => Arc::new(StorageOptionsAccessor::with_initial_and_provider(
+                merged_options,
+                provider,
+            )),
+            None => Arc::new(StorageOptionsAccessor::with_static_options(merged_options)),
+        };
+        merged_params.storage_options_accessor = Some(storage_options_accessor);
+        merged_params
+    }
+
     async fn load_impl(mut self) -> Result<Dataset> {
-        // Apply storage_options_override to merge namespace options with any existing accessor
+        // Apply storage_options_override to merge namespace client options with any existing accessor
         if let Some(override_opts) = self.storage_options_override.take() {
-            // Get existing options and merge
-            let mut merged_opts = self.options.storage_options().cloned().unwrap_or_default();
-            // Override with namespace storage options - they take precedence
-            merged_opts.extend(override_opts);
-
-            // Update accessor with merged options
-            if let Some(accessor) = &self.options.storage_options_accessor {
-                if let Some(provider) = accessor.provider().cloned() {
-                    self.options.storage_options_accessor = Some(Arc::new(
-                        StorageOptionsAccessor::with_initial_and_provider(merged_opts, provider),
-                    ));
-                } else {
-                    self.options.storage_options_accessor = Some(Arc::new(
-                        StorageOptionsAccessor::with_static_options(merged_opts),
-                    ));
-                }
-            } else {
-                self.options.storage_options_accessor = Some(Arc::new(
-                    StorageOptionsAccessor::with_static_options(merged_opts),
-                ));
-            }
+            self.options =
+                Self::merge_store_params_with_storage_options(&self.options, &override_opts);
         }
 
+        let index_cache_backend = self.index_cache_backend.take();
         let session = match self.session.as_ref() {
             Some(session) => session.clone(),
-            None => Arc::new(Session::new(
-                self.index_cache_size_bytes,
-                self.metadata_cache_size_bytes,
-                Default::default(),
-            )),
+            None => match index_cache_backend {
+                Some(backend) => Arc::new(Session::with_index_cache_backend(
+                    backend,
+                    self.metadata_cache_size_bytes,
+                    Default::default(),
+                )),
+                None => Arc::new(Session::new(
+                    self.index_cache_size_bytes,
+                    self.metadata_cache_size_bytes,
+                    Default::default(),
+                )),
+            },
         };
 
         let target_ref = self.version.clone();
@@ -593,6 +654,8 @@ impl DatasetBuilder {
 
         let file_reader_options = self.file_reader_options.clone();
         let store_params = self.options.clone();
+        let base_store_params = (!self.base_store_params.is_empty())
+            .then(|| Arc::new(std::mem::take(&mut self.base_store_params)));
         let (object_store, base_path, commit_handler) = self.build_object_store().await?;
 
         // Two cases that need to check out after loading the manifest:
@@ -645,6 +708,7 @@ impl DatasetBuilder {
             base_path,
             commit_handler,
             Some(store_params),
+            base_store_params,
         )
         .await?;
 
@@ -682,6 +746,7 @@ impl DatasetBuilder {
         base_path: Path,
         commit_handler: Arc<dyn CommitHandler>,
         store_params: Option<ObjectStoreParams>,
+        base_store_params: Option<Arc<HashMap<String, ObjectStoreParams>>>,
     ) -> Result<Dataset> {
         let (manifest, location) = if let Some(mut manifest) = manifest {
             let location = commit_handler
@@ -717,7 +782,12 @@ impl DatasetBuilder {
                 None => commit_handler
                     .resolve_latest_location(&base_path, &object_store)
                     .await
-                    .map_err(|e| Error::dataset_not_found(base_path.to_string(), Box::new(e)))?,
+                    .map_err(|e| match &e {
+                        Error::NotFound { .. } => {
+                            Error::dataset_not_found(base_path.to_string(), Box::new(e))
+                        }
+                        _ => e,
+                    })?,
             };
             let manifest = Dataset::load_manifest(
                 &object_store,
@@ -739,6 +809,7 @@ impl DatasetBuilder {
             commit_handler,
             file_reader_options,
             store_params,
+            base_store_params,
         )
     }
 }
diff --git a/rust/lance/src/dataset/cleanup.rs b/rust/lance/src/dataset/cleanup.rs
index 464f735e2f6..0c6b78253b9 100644
--- a/rust/lance/src/dataset/cleanup.rs
+++ b/rust/lance/src/dataset/cleanup.rs
@@ -1180,7 +1180,9 @@ mod tests {
 
     use super::*;
     use crate::blob::{BlobArrayBuilder, blob_field};
+    use crate::index::DatasetIndexExt;
     use crate::{
+        dataset::transaction::{Operation, Transaction},
         dataset::{ReadParams, WriteMode, WriteParams, builder::DatasetBuilder},
         index::vector::VectorIndexParams,
     };
@@ -1193,7 +1195,7 @@ mod tests {
     use datafusion::common::assert_contains;
     use lance_core::utils::tempfile::TempStrDir;
     use lance_core::utils::testing::{ProxyObjectStore, ProxyObjectStorePolicy};
-    use lance_index::{DatasetIndexExt, IndexType};
+    use lance_index::IndexType;
     use lance_io::object_store::{
         ObjectStore, ObjectStoreParams, ObjectStoreRegistry, WrappingObjectStore,
     };
@@ -1201,6 +1203,7 @@ mod tests {
     use lance_table::io::commit::RenameCommitHandler;
     use lance_testing::datagen::{BatchGenerator, IncrementingInt32, RandomVector, some_batch};
     use mock_instant::thread_local::MockClock;
+    use uuid::Uuid;
 
     #[derive(Debug)]
     struct MockObjectStore {
@@ -1529,6 +1532,59 @@ mod tests {
         }
     }
 
+    async fn write_dummy_index_artifact(dataset: &Dataset, uuid: Uuid) -> Result<()> {
+        let index_dir = dataset.indices_dir().child(uuid.to_string());
+        dataset
+            .object_store()
+            .put(&index_dir.child("index.idx"), b"idx")
+            .await?;
+        dataset
+            .object_store()
+            .put(&index_dir.child("auxiliary.idx"), b"aux")
+            .await?;
+        Ok(())
+    }
+
+    async fn write_dummy_staging_partial(
+        dataset: &Dataset,
+        staging_uuid: Uuid,
+        shard_uuid: Uuid,
+    ) -> Result<()> {
+        let shard_dir = dataset
+            .indices_dir()
+            .child(staging_uuid.to_string())
+            .child(format!("partial_{}", shard_uuid));
+        dataset
+            .object_store()
+            .put(&shard_dir.child("index.idx"), b"idx")
+            .await?;
+        dataset
+            .object_store()
+            .put(&shard_dir.child("auxiliary.idx"), b"aux")
+            .await?;
+        Ok(())
+    }
+
+    fn dummy_index_metadata(
+        dataset: &Dataset,
+        field_id: i32,
+        uuid: Uuid,
+        fragment_bitmap: impl IntoIterator<Item = u32>,
+    ) -> IndexMetadata {
+        IndexMetadata {
+            uuid,
+            name: "some_index".to_string(),
+            fields: vec![field_id],
+            dataset_version: dataset.version().version,
+            fragment_bitmap: Some(fragment_bitmap.into_iter().collect()),
+            index_details: None,
+            index_version: IndexType::Vector.version(),
+            created_at: None,
+            base_id: None,
+            files: None,
+        }
+    }
+
     fn blob_v2_batch(blob_len: usize) -> Box<dyn RecordBatchReader + Send> {
         let mut blobs = BlobArrayBuilder::new(1);
         blobs.push_bytes(vec![0u8; blob_len]).unwrap();
@@ -2181,6 +2237,149 @@ mod tests {
         assert_eq!(before_count, after_count);
     }
 
+    #[tokio::test]
+    async fn cleanup_old_replaced_segment_keeps_still_referenced_segments() {
+        let fixture = MockDatasetFixture::try_new().unwrap();
+        fixture.create_some_data().await.unwrap();
+
+        let mut dataset = fixture.open().await.unwrap();
+        let field_id = dataset.schema().field("indexable").unwrap().id;
+
+        let seg_a = Uuid::new_v4();
+        let seg_b = Uuid::new_v4();
+        write_dummy_index_artifact(&dataset, seg_a).await.unwrap();
+        write_dummy_index_artifact(&dataset, seg_b).await.unwrap();
+
+        let index_a = dummy_index_metadata(&dataset, field_id, seg_a, [0_u32]);
+        let index_b = dummy_index_metadata(&dataset, field_id, seg_b, [1_u32]);
+        let initial_tx = Transaction::new(
+            dataset.manifest.version,
+            Operation::CreateIndex {
+                new_indices: vec![index_a.clone(), index_b.clone()],
+                removed_indices: vec![],
+            },
+            None,
+        );
+        dataset
+            .apply_commit(initial_tx, &Default::default(), &Default::default())
+            .await
+            .unwrap();
+
+        MockClock::set_system_time(TimeDelta::try_days(10).unwrap().to_std().unwrap());
+
+        let seg_c = Uuid::new_v4();
+        write_dummy_index_artifact(&dataset, seg_c).await.unwrap();
+        let index_c = dummy_index_metadata(&dataset, field_id, seg_c, [2_u32]);
+        let replace_tx = Transaction::new(
+            dataset.manifest.version,
+            Operation::CreateIndex {
+                new_indices: vec![index_c.clone()],
+                removed_indices: vec![index_a.clone()],
+            },
+            None,
+        );
+        dataset
+            .apply_commit(replace_tx, &Default::default(), &Default::default())
+            .await
+            .unwrap();
+
+        let removed = fixture
+            .run_cleanup(utc_now() - TimeDelta::try_days(7).unwrap())
+            .await
+            .unwrap();
+
+        assert_eq!(removed.index_files_removed, 2);
+        assert!(
+            !dataset
+                .object_store()
+                .exists(
+                    &dataset
+                        .indices_dir()
+                        .child(seg_a.to_string())
+                        .child("index.idx")
+                )
+                .await
+                .unwrap()
+        );
+        assert!(
+            dataset
+                .object_store()
+                .exists(
+                    &dataset
+                        .indices_dir()
+                        .child(seg_b.to_string())
+                        .child("index.idx")
+                )
+                .await
+                .unwrap()
+        );
+        assert!(
+            dataset
+                .object_store()
+                .exists(
+                    &dataset
+                        .indices_dir()
+                        .child(seg_c.to_string())
+                        .child("index.idx")
+                )
+                .await
+                .unwrap()
+        );
+    }
+
+    #[tokio::test]
+    async fn cleanup_old_uncommitted_index_artifacts() {
+        let fixture = MockDatasetFixture::try_new().unwrap();
+        fixture.create_some_data().await.unwrap();
+
+        let dataset = fixture.open().await.unwrap();
+        let staging_uuid = Uuid::new_v4();
+        let shard_uuid = Uuid::new_v4();
+        let built_segment_uuid = Uuid::new_v4();
+
+        write_dummy_staging_partial(&dataset, staging_uuid, shard_uuid)
+            .await
+            .unwrap();
+        write_dummy_index_artifact(&dataset, built_segment_uuid)
+            .await
+            .unwrap();
+
+        MockClock::set_system_time(TimeDelta::try_days(10).unwrap().to_std().unwrap());
+
+        let removed = fixture
+            .run_cleanup(utc_now() - TimeDelta::try_days(7).unwrap())
+            .await
+            .unwrap();
+
+        assert_eq!(removed.old_versions, 0);
+        assert_eq!(removed.index_files_removed, 4);
+        assert!(
+            !dataset
+                .object_store()
+                .exists(
+                    &dataset
+                        .indices_dir()
+                        .child(staging_uuid.to_string())
+                        .child(format!("partial_{}", shard_uuid))
+                        .child("index.idx"),
+                )
+                .await
+                .unwrap()
+        );
+        assert!(
+            !dataset
+                .object_store()
+                .exists(
+                    &dataset
+                        .indices_dir()
+                        .child(built_segment_uuid.to_string())
+                        .child("index.idx"),
+                )
+                .await
+                .unwrap()
+        );
+    }
+
     #[tokio::test]
     async fn cleanup_failed_commit_data_file() {
         // We should clean up data files that are written but the commit failed
@@ -2197,7 +2396,8 @@ mod tests {
         // deposited a data file
         assert_eq!(before_count.num_data_files, 2);
         assert_eq!(before_count.num_manifest_files, 1);
-        assert_eq!(before_count.num_tx_files, 2);
+        // Only 1 txn file: the failed commit's txn file was already cleaned up.
+        assert_eq!(before_count.num_tx_files, 1);
 
         // All of our manifests are newer than the threshold but temp files
         // should still be deleted.
@@ -2642,8 +2842,9 @@ mod tests {
         // Create a full-text index (Inverted) on the "text" column once.
         // We only create this on main during dataset creation. Branches inherit the index configuration.
         async fn create_text_index(&mut self) -> Result<()> {
+            use crate::index::DatasetIndexExt;
+            use lance_index::IndexType;
             use lance_index::scalar::InvertedIndexParams;
-            use lance_index::{DatasetIndexExt, IndexType};
             let params = InvertedIndexParams::default();
             self.dataset
                 .create_index(&["text"], IndexType::Inverted, None, &params, true)
@@ -2867,7 +3068,7 @@ mod tests {
         assert_eq!(setup.branch1.counts.num_data_files, 2);
         assert_eq!(setup.branch1.counts.num_tx_files, 1);
         assert_eq!(setup.branch1.counts.num_delete_files, 2);
-        assert_eq!(setup.branch1.counts.num_index_files, 8);
+        assert_eq!(setup.branch1.counts.num_index_files, 23);
         setup.assert_all_unchanged().await;
 
         setup.branch1.compact().await.unwrap();
@@ -2882,7 +3083,7 @@ mod tests {
         assert_eq!(setup.branch1.counts.num_data_files, 2);
         assert_eq!(setup.branch1.counts.num_tx_files, 1);
         assert_eq!(setup.branch1.counts.num_delete_files, 1);
-        assert_eq!(setup.branch1.counts.num_index_files, 8);
+        assert_eq!(setup.branch1.counts.num_index_files, 23);
         setup.assert_all_unchanged().await;
 
         // Now we clean the referenced files of branch1 by branch2 and branch3
@@ -2896,14 +3097,14 @@ mod tests {
         assert_eq!(setup.branch2.counts.num_data_files, 1);
         assert_eq!(setup.branch2.counts.num_tx_files, 1);
         assert_eq!(setup.branch2.counts.num_delete_files, 0);
-        assert_eq!(setup.branch2.counts.num_index_files, 4);
+        assert_eq!(setup.branch2.counts.num_index_files, 13);
         // Only the latest manifest is retained.
         // (1, 1, 1, 0, 4) is the counts for the latest version of compaction
         assert_eq!(setup.branch3.counts.num_manifest_files, 1);
         assert_eq!(setup.branch3.counts.num_data_files, 1);
         assert_eq!(setup.branch3.counts.num_tx_files, 1);
         assert_eq!(setup.branch3.counts.num_delete_files, 0);
-        assert_eq!(setup.branch3.counts.num_index_files, 4);
+        assert_eq!(setup.branch3.counts.num_index_files, 16);
         setup.branch1.run_cleanup().await.unwrap();
 
         // Only the latest manifest is retained.
@@ -2912,7 +3113,7 @@ mod tests {
         assert_eq!(setup.branch1.counts.num_data_files, 1);
         assert_eq!(setup.branch1.counts.num_tx_files, 1);
         assert_eq!(setup.branch1.counts.num_delete_files, 0);
-        assert_eq!(setup.branch1.counts.num_index_files, 4);
+        assert_eq!(setup.branch1.counts.num_index_files, 13);
         setup.assert_all_unchanged().await;
     }
 
@@ -2929,7 +3130,7 @@ mod tests {
         assert_eq!(setup.branch3.counts.num_data_files, 2);
         assert_eq!(setup.branch3.counts.num_tx_files, 1);
         assert_eq!(setup.branch3.counts.num_delete_files, 2);
-        assert_eq!(setup.branch3.counts.num_index_files, 4);
+        assert_eq!(setup.branch3.counts.num_index_files, 19);
         setup
             .assert_unchanged(&["branch1", "branch2", "branch4", "main"])
             .await;
@@ -2945,7 +3146,7 @@ mod tests {
         assert_eq!(setup.branch2.counts.num_data_files, 2);
         assert_eq!(setup.branch2.counts.num_tx_files, 1);
         assert_eq!(setup.branch2.counts.num_delete_files, 1);
-        assert_eq!(setup.branch2.counts.num_index_files, 4);
+        assert_eq!(setup.branch2.counts.num_index_files, 13);
 
         setup.branch3.compact().await.unwrap();
         setup.branch3.run_cleanup().await.unwrap();
@@ -2955,7 +3156,7 @@ mod tests {
         assert_eq!(setup.branch3.counts.num_data_files, 1);
         assert_eq!(setup.branch3.counts.num_tx_files, 1);
         assert_eq!(setup.branch3.counts.num_delete_files, 0);
-        assert_eq!(setup.branch3.counts.num_index_files, 4);
+        assert_eq!(setup.branch3.counts.num_index_files, 19);
         setup
             .assert_unchanged(&["branch1", "branch2", "branch4", "main"])
             .await;
@@ -2968,7 +3169,7 @@ mod tests {
         assert_eq!(setup.branch2.counts.num_data_files, 1);
         assert_eq!(setup.branch2.counts.num_tx_files, 1);
         assert_eq!(setup.branch2.counts.num_delete_files, 0);
-        assert_eq!(setup.branch2.counts.num_index_files, 4);
+        assert_eq!(setup.branch2.counts.num_index_files, 13);
     }
 
     #[tokio::test]
@@ -2985,7 +3186,7 @@ mod tests {
         assert_eq!(setup.branch4.counts.num_data_files, 2);
         assert_eq!(setup.branch4.counts.num_tx_files, 1);
         assert_eq!(setup.branch4.counts.num_delete_files, 2);
-        assert_eq!(setup.branch4.counts.num_index_files, 4);
+        assert_eq!(setup.branch4.counts.num_index_files, 16);
         setup.assert_all_unchanged().await;
 
         setup.main.compact().await.unwrap();
@@ -3005,7 +3206,7 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 4);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 2);
-        assert_eq!(setup.main.counts.num_index_files, 8);
+        assert_eq!(setup.main.counts.num_index_files, 17);
 
         setup.branch4.compact().await.unwrap();
         setup.branch4.run_cleanup().await.unwrap();
@@ -3015,7 +3216,7 @@ mod tests {
         assert_eq!(setup.branch4.counts.num_data_files, 1);
         assert_eq!(setup.branch4.counts.num_tx_files, 1);
         assert_eq!(setup.branch4.counts.num_delete_files, 0);
-        assert_eq!(setup.branch4.counts.num_index_files, 4);
+        assert_eq!(setup.branch4.counts.num_index_files, 16);
         setup.assert_all_unchanged().await;
 
         setup.main.run_cleanup().await.unwrap();
@@ -3029,7 +3230,7 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 3);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 1);
-        assert_eq!(setup.main.counts.num_index_files, 8);
+        assert_eq!(setup.main.counts.num_index_files, 17);
     }
 
     #[tokio::test]
@@ -3054,7 +3255,7 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 4);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 3);
-        assert_eq!(setup.main.counts.num_index_files, 12);
+        assert_eq!(setup.main.counts.num_index_files, 30);
         setup.assert_all_unchanged().await;
 
         setup.main.compact().await.unwrap();
@@ -3065,7 +3266,7 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 4);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 2);
-        assert_eq!(setup.main.counts.num_index_files, 12);
+        assert_eq!(setup.main.counts.num_index_files, 30);
         setup.assert_all_unchanged().await;
 
         setup.branch1.write_data().await.unwrap();
@@ -3086,14 +3287,14 @@ mod tests {
         assert_eq!(setup.branch2.counts.num_data_files, 2);
         assert_eq!(setup.branch2.counts.num_tx_files, 1);
         assert_eq!(setup.branch2.counts.num_delete_files, 1);
-        assert_eq!(setup.branch2.counts.num_index_files, 8);
+        assert_eq!(setup.branch2.counts.num_index_files, 29);
         setup.branch1.run_cleanup().await.unwrap();
         // Cleanup 4 index files referenced from branch2
         assert_eq!(setup.branch1.counts.num_manifest_files, 2);
         assert_eq!(setup.branch1.counts.num_data_files, 2);
         assert_eq!(setup.branch1.counts.num_tx_files, 1);
         assert_eq!(setup.branch1.counts.num_delete_files, 1);
-        assert_eq!(setup.branch1.counts.num_index_files, 4);
+        assert_eq!(setup.branch1.counts.num_index_files, 13);
 
         setup.main.run_cleanup().await.unwrap();
         // Branch3 holds references from main:
@@ -3109,7 +3310,7 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 4);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 2);
-        assert_eq!(setup.main.counts.num_index_files, 8);
+        assert_eq!(setup.main.counts.num_index_files, 23);
 
         setup.branch3.write_data().await.unwrap();
         setup.branch3.compact().await.unwrap();
@@ -3119,7 +3320,7 @@ mod tests {
         assert_eq!(setup.branch3.counts.num_data_files, 1);
         assert_eq!(setup.branch3.counts.num_tx_files, 1);
         assert_eq!(setup.branch3.counts.num_delete_files, 0);
-        assert_eq!(setup.branch3.counts.num_index_files, 4);
+        assert_eq!(setup.branch3.counts.num_index_files, 19);
 
         setup.main.run_cleanup().await.unwrap();
         // Cleanup doesn't take effects if we don't clean branch2 and branch1 first
@@ -3127,7 +3328,7 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 4);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 2);
-        assert_eq!(setup.main.counts.num_index_files, 8);
+        assert_eq!(setup.main.counts.num_index_files, 23);
 
         // Cleanup doesn't take effect if we don't clean branch2 first
         setup.branch1.run_cleanup().await.unwrap();
@@ -3135,7 +3336,7 @@ mod tests {
         assert_eq!(setup.branch1.counts.num_data_files, 2);
         assert_eq!(setup.branch1.counts.num_tx_files, 1);
         assert_eq!(setup.branch1.counts.num_delete_files, 1);
-        assert_eq!(setup.branch1.counts.num_index_files, 4);
+        assert_eq!(setup.branch1.counts.num_index_files, 13);
 
         setup.branch2.run_cleanup().await.unwrap();
         // Only the latest manifest is retained.
@@ -3144,7 +3345,7 @@ mod tests {
         assert_eq!(setup.branch2.counts.num_data_files, 1);
         assert_eq!(setup.branch2.counts.num_tx_files, 1);
         assert_eq!(setup.branch2.counts.num_delete_files, 0);
-        assert_eq!(setup.branch2.counts.num_index_files, 4);
+        assert_eq!(setup.branch2.counts.num_index_files, 16);
 
         setup.branch1.run_cleanup().await.unwrap();
         // Only the latest manifest is retained.
@@ -3153,7 +3354,7 @@ mod tests {
         assert_eq!(setup.branch1.counts.num_data_files, 1);
         assert_eq!(setup.branch1.counts.num_tx_files, 1);
         assert_eq!(setup.branch1.counts.num_delete_files, 0);
-        assert_eq!(setup.branch1.counts.num_index_files, 4);
+        assert_eq!(setup.branch1.counts.num_index_files, 13);
 
         setup.main.run_cleanup().await.unwrap();
         // Branch4 holds references from main:
@@ -3165,7 +3366,7 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 4);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 2);
-        assert_eq!(setup.main.counts.num_index_files, 8);
+        assert_eq!(setup.main.counts.num_index_files, 23);
 
         setup.branch4.write_data().await.unwrap();
         setup.branch4.compact().await.unwrap();
@@ -3176,7 +3377,7 @@ mod tests {
         assert_eq!(setup.branch4.counts.num_data_files, 1);
         assert_eq!(setup.branch4.counts.num_tx_files, 1);
         assert_eq!(setup.branch4.counts.num_delete_files, 0);
-        assert_eq!(setup.branch4.counts.num_index_files, 4);
+        assert_eq!(setup.branch4.counts.num_index_files, 16);
 
         setup.main.run_cleanup().await.unwrap();
         // Only the latest manifest is retained.
@@ -3185,7 +3386,7 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 1);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 0);
-        assert_eq!(setup.main.counts.num_index_files, 4);
+        assert_eq!(setup.main.counts.num_index_files, 13);
     }
 
     #[tokio::test]
@@ -3209,7 +3410,7 @@ mod tests {
         assert_eq!(setup.branch2.counts.num_data_files, 1);
         assert_eq!(setup.branch2.counts.num_tx_files, 1);
         assert_eq!(setup.branch2.counts.num_delete_files, 1);
-        assert_eq!(setup.branch2.counts.num_index_files, 4);
+        assert_eq!(setup.branch2.counts.num_index_files, 13);
         // After auto-clean: branch3
         // 2 appends produced 2 data files
         // 2 deletes produced 2 deletion files
@@ -3217,7 +3418,7 @@ mod tests {
         assert_eq!(setup.branch3.counts.num_data_files, 2);
         assert_eq!(setup.branch3.counts.num_tx_files, 1);
         assert_eq!(setup.branch3.counts.num_delete_files, 2);
-        assert_eq!(setup.branch3.counts.num_index_files, 4);
+        assert_eq!(setup.branch3.counts.num_index_files, 19);
         setup
             .assert_unchanged(&["branch1", "branch4", "main"])
             .await;
@@ -3239,14 +3440,14 @@ mod tests {
         assert_eq!(setup.branch2.counts.num_data_files, 1);
         assert_eq!(setup.branch2.counts.num_tx_files, 1);
         assert_eq!(setup.branch2.counts.num_delete_files, 0);
-        assert_eq!(setup.branch2.counts.num_index_files, 4);
+        assert_eq!(setup.branch2.counts.num_index_files, 16);
         // Only the latest manifest is retained.
         // (1, 1, 1, 0, 4) is the counts of one version
         assert_eq!(setup.branch3.counts.num_manifest_files, 1);
         assert_eq!(setup.branch3.counts.num_data_files, 1);
         assert_eq!(setup.branch3.counts.num_tx_files, 1);
         assert_eq!(setup.branch3.counts.num_delete_files, 0);
-        assert_eq!(setup.branch3.counts.num_index_files, 4);
+        assert_eq!(setup.branch3.counts.num_index_files, 19);
         setup
             .assert_unchanged(&["branch1", "branch4", "main"])
             .await;
@@ -3276,7 +3477,7 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 4);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 3);
-        assert_eq!(setup.main.counts.num_index_files, 4);
+        assert_eq!(setup.main.counts.num_index_files, 13);
 
         setup.main.compact().await.unwrap();
         setup
@@ -3296,7 +3497,7 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 4);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 2);
-        assert_eq!(setup.main.counts.num_index_files, 4);
+        assert_eq!(setup.main.counts.num_index_files, 13);
 
         setup.branch4.compact().await.unwrap();
         setup
@@ -3313,13 +3514,13 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 3);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 1);
-        assert_eq!(setup.main.counts.num_index_files, 4);
+        assert_eq!(setup.main.counts.num_index_files, 13);
         // (1, 1, 1, 0, 4) is the counts of one version
         assert_eq!(setup.branch4.counts.num_manifest_files, 1);
         assert_eq!(setup.branch4.counts.num_data_files, 1);
         assert_eq!(setup.branch4.counts.num_tx_files, 1);
         assert_eq!(setup.branch4.counts.num_delete_files, 0);
-        assert_eq!(setup.branch4.counts.num_index_files, 4);
+        assert_eq!(setup.branch4.counts.num_index_files, 13);
 
         setup.branch1.write_data().await.unwrap();
         setup.branch1.compact().await.unwrap();
@@ -3337,7 +3538,7 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 3);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 1);
-        assert_eq!(setup.main.counts.num_index_files, 4);
+        assert_eq!(setup.main.counts.num_index_files, 13);
         // Branch3 and branch2 still hold references from branch1:
         // - 1 manifest file
         // - 1 data files
@@ -3346,7 +3547,7 @@ mod tests {
         assert_eq!(setup.branch1.counts.num_data_files, 2);
         assert_eq!(setup.branch1.counts.num_tx_files, 1);
         assert_eq!(setup.branch1.counts.num_delete_files, 1);
-        assert_eq!(setup.branch1.counts.num_index_files, 4);
+        assert_eq!(setup.branch1.counts.num_index_files, 13);
 
         setup.branch2.write_data().await.unwrap();
         setup.branch2.compact().await.unwrap();
@@ -3364,7 +3565,7 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 3);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 1);
-        assert_eq!(setup.main.counts.num_index_files, 4);
+        assert_eq!(setup.main.counts.num_index_files, 13);
         // Branch3 still holds references from branch1:
         // - 1 manifest file
         // - 1 data files
@@ -3373,7 +3574,7 @@ mod tests {
         assert_eq!(setup.branch1.counts.num_data_files, 2);
         assert_eq!(setup.branch1.counts.num_tx_files, 1);
         assert_eq!(setup.branch1.counts.num_delete_files, 1);
-        assert_eq!(setup.branch1.counts.num_index_files, 4);
+        assert_eq!(setup.branch1.counts.num_index_files, 13);
         // Branch3 still holds references from branch2:
         // - 1 manifest file
         // - 1 data files
@@ -3382,7 +3583,7 @@ mod tests {
         assert_eq!(setup.branch2.counts.num_data_files, 2);
         assert_eq!(setup.branch2.counts.num_tx_files, 1);
         assert_eq!(setup.branch2.counts.num_delete_files, 1);
-        assert_eq!(setup.branch2.counts.num_index_files, 4);
+        assert_eq!(setup.branch2.counts.num_index_files, 16);
 
         setup.branch3.write_data().await.unwrap();
         setup.branch3.compact().await.unwrap();
@@ -3400,22 +3601,22 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 1);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 0);
-        assert_eq!(setup.main.counts.num_index_files, 4);
+        assert_eq!(setup.main.counts.num_index_files, 13);
         assert_eq!(setup.branch1.counts.num_manifest_files, 1);
         assert_eq!(setup.branch1.counts.num_data_files, 1);
         assert_eq!(setup.branch1.counts.num_tx_files, 1);
         assert_eq!(setup.branch1.counts.num_delete_files, 0);
-        assert_eq!(setup.branch1.counts.num_index_files, 4);
+        assert_eq!(setup.branch1.counts.num_index_files, 13);
         assert_eq!(setup.branch2.counts.num_manifest_files, 1);
         assert_eq!(setup.branch2.counts.num_data_files, 1);
         assert_eq!(setup.branch2.counts.num_tx_files, 1);
         assert_eq!(setup.branch2.counts.num_delete_files, 0);
-        assert_eq!(setup.branch2.counts.num_index_files, 4);
+        assert_eq!(setup.branch2.counts.num_index_files, 16);
         assert_eq!(setup.branch3.counts.num_manifest_files, 1);
         assert_eq!(setup.branch3.counts.num_data_files, 1);
         assert_eq!(setup.branch3.counts.num_tx_files, 1);
         assert_eq!(setup.branch3.counts.num_delete_files, 0);
-        assert_eq!(setup.branch3.counts.num_index_files, 4);
+        assert_eq!(setup.branch3.counts.num_index_files, 19);
         setup.assert_unchanged(&["branch4"]).await;
     }
 
@@ -3459,24 +3660,24 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 4);
         assert_eq!(setup.main.counts.num_tx_files, 2);
         assert_eq!(setup.main.counts.num_delete_files, 2);
-        assert_eq!(setup.main.counts.num_index_files, 8);
+        assert_eq!(setup.main.counts.num_index_files, 20);
         // Branch3 tag holds branch1 with 1 tx file, 1 data files, 1 deletion files and 4 index files
         assert_eq!(setup.branch2.counts.num_manifest_files, 2);
         assert_eq!(setup.branch2.counts.num_data_files, 2);
         assert_eq!(setup.branch2.counts.num_tx_files, 1);
         assert_eq!(setup.branch2.counts.num_delete_files, 1);
-        assert_eq!(setup.branch2.counts.num_index_files, 4);
+        assert_eq!(setup.branch2.counts.num_index_files, 13);
         // Branch3 tag holds branch2 with 1 tx file, 1 data files, 1 deletion files and 4 index files
         assert_eq!(setup.branch2.counts.num_manifest_files, 2);
         assert_eq!(setup.branch2.counts.num_data_files, 2);
         assert_eq!(setup.branch2.counts.num_tx_files, 1);
         assert_eq!(setup.branch2.counts.num_delete_files, 1);
-        assert_eq!(setup.branch2.counts.num_index_files, 4);
+        assert_eq!(setup.branch2.counts.num_index_files, 13);
         assert_eq!(setup.branch4.counts.num_manifest_files, 1);
         assert_eq!(setup.branch4.counts.num_data_files, 1);
         assert_eq!(setup.branch4.counts.num_tx_files, 1);
         assert_eq!(setup.branch4.counts.num_delete_files, 0);
-        assert_eq!(setup.branch4.counts.num_index_files, 4);
+        assert_eq!(setup.branch4.counts.num_index_files, 13);
 
         setup
             .branch3
@@ -3499,27 +3700,27 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 4);
         assert_eq!(setup.main.counts.num_tx_files, 2);
         assert_eq!(setup.main.counts.num_delete_files, 2);
-        assert_eq!(setup.main.counts.num_index_files, 8);
+        assert_eq!(setup.main.counts.num_index_files, 20);
         assert_eq!(setup.branch1.counts.num_manifest_files, 1);
         assert_eq!(setup.branch1.counts.num_data_files, 1);
         assert_eq!(setup.branch1.counts.num_tx_files, 1);
         assert_eq!(setup.branch1.counts.num_delete_files, 0);
-        assert_eq!(setup.branch1.counts.num_index_files, 4);
+        assert_eq!(setup.branch1.counts.num_index_files, 10);
         assert_eq!(setup.branch2.counts.num_manifest_files, 1);
         assert_eq!(setup.branch2.counts.num_data_files, 1);
         assert_eq!(setup.branch2.counts.num_tx_files, 1);
         assert_eq!(setup.branch2.counts.num_delete_files, 0);
-        assert_eq!(setup.branch2.counts.num_index_files, 4);
+        assert_eq!(setup.branch2.counts.num_index_files, 13);
         assert_eq!(setup.branch3.counts.num_manifest_files, 1);
         assert_eq!(setup.branch3.counts.num_data_files, 1);
         assert_eq!(setup.branch3.counts.num_tx_files, 1);
         assert_eq!(setup.branch3.counts.num_delete_files, 0);
-        assert_eq!(setup.branch3.counts.num_index_files, 4);
+        assert_eq!(setup.branch3.counts.num_index_files, 16);
         assert_eq!(setup.branch4.counts.num_manifest_files, 1);
         assert_eq!(setup.branch4.counts.num_data_files, 1);
         assert_eq!(setup.branch4.counts.num_tx_files, 1);
         assert_eq!(setup.branch4.counts.num_delete_files, 0);
-        assert_eq!(setup.branch4.counts.num_index_files, 4);
+        assert_eq!(setup.branch4.counts.num_index_files, 13);
 
         setup.main.dataset.tags().delete("main-tag").await.unwrap();
         setup
@@ -3535,22 +3736,22 @@ mod tests {
         assert_eq!(setup.main.counts.num_data_files, 1);
         assert_eq!(setup.main.counts.num_tx_files, 1);
         assert_eq!(setup.main.counts.num_delete_files, 0);
-        assert_eq!(setup.main.counts.num_index_files, 4);
+        assert_eq!(setup.main.counts.num_index_files, 10);
         assert_eq!(setup.branch2.counts.num_manifest_files, 1);
         assert_eq!(setup.branch2.counts.num_data_files, 1);
         assert_eq!(setup.branch2.counts.num_tx_files, 1);
         assert_eq!(setup.branch2.counts.num_delete_files, 0);
-        assert_eq!(setup.branch2.counts.num_index_files, 4);
+        assert_eq!(setup.branch2.counts.num_index_files, 13);
         assert_eq!(setup.branch3.counts.num_manifest_files, 1);
         assert_eq!(setup.branch3.counts.num_data_files, 1);
         assert_eq!(setup.branch3.counts.num_tx_files, 1);
         assert_eq!(setup.branch3.counts.num_delete_files, 0);
-        assert_eq!(setup.branch3.counts.num_index_files, 4);
+        assert_eq!(setup.branch3.counts.num_index_files, 16);
         assert_eq!(setup.branch4.counts.num_manifest_files, 1);
         assert_eq!(setup.branch4.counts.num_data_files, 1);
         assert_eq!(setup.branch4.counts.num_tx_files, 1);
         assert_eq!(setup.branch4.counts.num_delete_files, 0);
-        assert_eq!(setup.branch4.counts.num_index_files, 4);
+        assert_eq!(setup.branch4.counts.num_index_files, 13);
     }
 
     #[test]
diff --git a/rust/lance/src/dataset/fragment.rs b/rust/lance/src/dataset/fragment.rs
index b4d22d6511e..29b33a75018 100644
--- a/rust/lance/src/dataset/fragment.rs
+++ b/rust/lance/src/dataset/fragment.rs
@@ -1113,8 +1113,14 @@ impl FileFragment {
     /// Fails if the fragment does not have the physical row count in the metadata.  This method should
     /// only be called in new workflows which are not run on old versions of Lance.
     pub fn fast_physical_rows(&self) -> Result<usize> {
-        if self.dataset.manifest.writer_version.is_some() && self.metadata.physical_rows.is_some() {
-            Ok(self.metadata.physical_rows.unwrap())
+        if self.dataset.manifest.writer_version.is_some() {
+            let Some(physical_rows) = self.metadata.physical_rows else {
+                return Err(Error::internal(format!(
+                    "The method fast_physical_rows was called on a fragment that does not have the physical row count in the metadata. Fragment id: {}",
+                    self.id()
+                )));
+            };
+            Ok(physical_rows)
         } else {
             Err(Error::internal(format!(
                 "The method fast_physical_rows was called on a fragment that does not have the physical row count in the metadata. Fragment id: {}",
@@ -1168,8 +1174,10 @@ impl FileFragment {
         // we should not used the cached value. On write, we update the values
         // in the manifest, fixing the issue for future reads.
         // See: https://github.com/lance-format/lance/issues/1531
-        if self.dataset.manifest.writer_version.is_some() && self.metadata.physical_rows.is_some() {
-            return Ok(self.metadata.physical_rows.unwrap());
+        if self.dataset.manifest.writer_version.is_some()
+            && let Some(physical_rows) = self.metadata.physical_rows
+        {
+            return Ok(physical_rows);
         }
 
         // Just open any file. All of them should have same size.
@@ -1201,7 +1209,7 @@ impl FileFragment {
         let mut seen_fields = HashSet::new();
         for data_file in &self.metadata.files {
             let last = -1;
-            for field_id in &data_file.fields {
+            for field_id in data_file.fields.iter() {
                 if *field_id <= last {
                     return Err(Error::corrupt_file(
                         self.dataset
@@ -1658,12 +1666,19 @@ impl FileFragment {
         // Mark fields in updated data files as obsolete ("tombstone").
         let updated_fields = updated_fragment.files.last().unwrap().fields.clone();
         for data_file in &mut updated_fragment.files.iter_mut().rev().skip(1) {
-            for field in &mut data_file.fields {
-                if updated_fields.contains(field) {
-                    // Tombstone these fields
-                    *field = -2;
-                }
-            }
+            let new_fields: Arc<[i32]> = data_file
+                .fields
+                .iter()
+                .map(|field| {
+                    if updated_fields.contains(field) {
+                        -2 // Tombstone
+                    } else {
+                        *field
+                    }
+                })
+                .collect::<Vec<_>>()
+                .into();
+            data_file.fields = new_fields;
         }
         // Remove data files that have become entirely tombstoned.
         updated_fragment
@@ -1871,6 +1886,10 @@ impl CacheKey for FileMetadataCacheKey {
     fn key(&self) -> std::borrow::Cow<'_, str> {
         "".into()
     }
+
+    fn type_name() -> &'static str {
+        "FileMetadata"
+    }
 }
 
 impl From<FileFragment> for Fragment {
diff --git a/rust/lance/src/dataset/fragment/write.rs b/rust/lance/src/dataset/fragment/write.rs
index a1c02d20758..834d4dbd37b 100644
--- a/rust/lance/src/dataset/fragment/write.rs
+++ b/rust/lance/src/dataset/fragment/write.rs
@@ -15,6 +15,7 @@ use lance_io::object_store::ObjectStore;
 use lance_table::format::{DataFile, Fragment};
 use lance_table::io::manifest::ManifestDescribing;
 use std::borrow::Cow;
+use std::sync::Arc;
 use uuid::Uuid;
 
 use crate::Result;
@@ -170,16 +171,18 @@ impl<'a> FragmentCreateBuilder<'a> {
             return Err(Error::invalid_input("Input data was empty."));
         }
 
-        let field_ids = writer
+        let field_ids: Arc<[i32]> = writer
             .field_id_to_column_indices()
             .iter()
             .map(|(field_id, _)| *field_id as i32)
-            .collect::<Vec<_>>();
-        let column_indices = writer
+            .collect::<Vec<_>>()
+            .into();
+        let column_indices: Arc<[i32]> = writer
             .field_id_to_column_indices()
             .iter()
             .map(|(_, column_index)| *column_index as i32)
-            .collect::<Vec<_>>();
+            .collect::<Vec<_>>()
+            .into();
 
         fragment.files[0].fields = field_ids;
         fragment.files[0].column_indices = column_indices;
@@ -414,7 +417,7 @@ mod tests {
         assert_eq!(fragment.id, 0);
         assert_eq!(fragment.deletion_file, None);
         assert_eq!(fragment.files.len(), 1);
-        assert_eq!(fragment.files[0].fields, vec![0, 1]);
+        assert_eq!(fragment.files[0].fields.as_ref(), &[0, 1]);
     }
 
     #[tokio::test]
@@ -437,8 +440,8 @@ mod tests {
         assert_eq!(fragment.id, 42);
         assert_eq!(fragment.deletion_file, None);
         assert_eq!(fragment.files.len(), 1);
-        assert_eq!(fragment.files[0].fields, vec![3, 1]);
-        assert_eq!(fragment.files[0].column_indices, vec![0, 1]);
+        assert_eq!(fragment.files[0].fields.as_ref(), &[3, 1]);
+        assert_eq!(fragment.files[0].column_indices.as_ref(), &[0, 1]);
     }
 
     #[tokio::test]
@@ -500,7 +503,7 @@ mod tests {
         assert_eq!(fragments.len(), 1);
         assert_eq!(fragments[0].deletion_file, None);
         assert_eq!(fragments[0].files.len(), 1);
-        assert_eq!(fragments[0].files[0].fields, vec![0, 1]);
+        assert_eq!(fragments[0].files[0].fields.as_ref(), &[0, 1]);
     }
 
     #[tokio::test]
@@ -521,13 +524,13 @@ mod tests {
         assert_eq!(fragments.len(), 3);
         assert_eq!(fragments[0].deletion_file, None);
         assert_eq!(fragments[0].files.len(), 1);
-        assert_eq!(fragments[0].files[0].column_indices, vec![0, 1]);
+        assert_eq!(fragments[0].files[0].column_indices.as_ref(), &[0, 1]);
         assert_eq!(fragments[1].deletion_file, None);
         assert_eq!(fragments[1].files.len(), 1);
-        assert_eq!(fragments[1].files[0].column_indices, vec![0, 1]);
+        assert_eq!(fragments[1].files[0].column_indices.as_ref(), &[0, 1]);
         assert_eq!(fragments[2].deletion_file, None);
         assert_eq!(fragments[2].files.len(), 1);
-        assert_eq!(fragments[2].files[0].column_indices, vec![0, 1]);
+        assert_eq!(fragments[2].files[0].column_indices.as_ref(), &[0, 1]);
     }
 
     #[rstest]
diff --git a/rust/lance/src/dataset/index.rs b/rust/lance/src/dataset/index.rs
index 404e53dc0be..856f7361892 100644
--- a/rust/lance/src/dataset/index.rs
+++ b/rust/lance/src/dataset/index.rs
@@ -9,12 +9,13 @@ use std::sync::Arc;
 use crate::Dataset;
 use crate::dataset::optimize::RemappedIndex;
 use crate::dataset::optimize::remapping::RemapResult;
+use crate::index::DatasetIndexExt;
 use crate::index::remap_index;
 use crate::index::scalar::infer_scalar_index_details;
 use arrow_schema::DataType;
 use async_trait::async_trait;
 use lance_core::{Error, Result};
-use lance_index::DatasetIndexExt;
+use lance_encoding::version::LanceFileVersion;
 use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME;
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_table::format::IndexMetadata;
@@ -104,6 +105,7 @@ impl IndexRemapper for DatasetIndexRemapper {
                             new_id: id,
                             index_details,
                             index_version: index.index_version as u32,
+                            files: index.files.clone(),
                         });
                     }
                     RemapResult::Remapped(remapped_index) => {
@@ -128,14 +130,30 @@ pub trait LanceIndexStoreExt {
         Self: Sized;
 }
 
+/// Extract the lance file version from a dataset, floored at V2_0.
+///
+/// Index files should never use the legacy format. If the dataset uses legacy
+/// format or doesn't have a version set, V2_0 is used as the minimum.
+pub(crate) fn dataset_format_version(dataset: &Dataset) -> LanceFileVersion {
+    dataset
+        .manifest
+        .data_storage_format
+        .lance_file_version()
+        .ok()
+        .map(|v| v.resolve().max(LanceFileVersion::V2_0))
+        .unwrap_or(LanceFileVersion::V2_0)
+}
+
 impl LanceIndexStoreExt for LanceIndexStore {
     fn from_dataset_for_new(dataset: &Dataset, uuid: &str) -> Result<Self> {
         let index_dir = dataset.indices_dir().child(uuid);
         let cache = dataset.metadata_cache.file_metadata_cache(&index_dir);
-        Ok(Self::new(
+        let format_version = dataset_format_version(dataset);
+        Ok(Self::with_format_version(
             dataset.object_store.clone(),
             index_dir,
             Arc::new(cache),
+            format_version,
         ))
     }
 
@@ -144,10 +162,129 @@ impl LanceIndexStoreExt for LanceIndexStore {
             .indice_files_dir(index)?
             .child(index.uuid.to_string());
         let cache = dataset.metadata_cache.file_metadata_cache(&index_dir);
-        Ok(Self::new(
+        let format_version = dataset_format_version(dataset);
+        let store = Self::with_format_version(
             dataset.object_store.clone(),
             index_dir,
             Arc::new(cache),
-        ))
+            format_version,
+        );
+        Ok(store.with_file_sizes(index.file_size_map()))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use super::*;
+    use crate::dataset::WriteParams;
+    use crate::index::DatasetIndexExt;
+    use crate::index::vector::VectorIndexParams;
+    use lance_datagen::{BatchCount, RowCount, array};
+    use lance_index::IndexType;
+    use lance_linalg::distance::MetricType;
+    use uuid::Uuid;
+
+    #[tokio::test]
+    async fn test_remapper_only_touches_segments_with_affected_fragments() {
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(16.into()),
+            )
+            .into_reader_rows(RowCount::from(40), BatchCount::from(2));
+
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 20,
+                max_rows_per_group: 20,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let fragments = dataset.get_fragments();
+        assert!(
+            fragments.len() >= 2,
+            "expected at least two fragments for this test"
+        );
+        let target_fragments = fragments.iter().take(2).collect::<Vec<_>>();
+
+        let params = VectorIndexParams::ivf_flat(2, MetricType::L2);
+        let first_segment_uuid = Uuid::new_v4();
+        let second_segment_uuid = Uuid::new_v4();
+        let built_index = dataset
+            .create_index_builder(&["vector"], IndexType::Vector, &params)
+            .name("vector_idx".to_string())
+            .index_uuid(first_segment_uuid.to_string())
+            .execute_uncommitted()
+            .await
+            .unwrap();
+        let first_segment_dir = dataset.indices_dir().child(first_segment_uuid.to_string());
+        let second_segment_dir = dataset.indices_dir().child(second_segment_uuid.to_string());
+        for file_name in ["index.idx", "auxiliary.idx"] {
+            dataset
+                .object_store()
+                .copy(
+                    &first_segment_dir.child(file_name),
+                    &second_segment_dir.child(file_name),
+                )
+                .await
+                .unwrap();
+        }
+
+        let segments = vec![
+            IndexMetadata {
+                uuid: first_segment_uuid,
+                fragment_bitmap: Some(std::iter::once(target_fragments[0].id() as u32).collect()),
+                ..built_index.clone()
+            },
+            IndexMetadata {
+                uuid: second_segment_uuid,
+                fragment_bitmap: Some(std::iter::once(target_fragments[1].id() as u32).collect()),
+                ..built_index
+            },
+        ];
+
+        dataset
+            .commit_existing_index_segments("vector_idx", "vector", segments)
+            .await
+            .unwrap();
+        let committed = dataset.load_indices_by_name("vector_idx").await.unwrap();
+        let committed_ids = committed
+            .iter()
+            .map(|segment| segment.uuid)
+            .collect::<Vec<_>>();
+        let unaffected_segment_id = committed
+            .iter()
+            .find(|segment| {
+                segment
+                    .fragment_bitmap
+                    .as_ref()
+                    .is_some_and(|bitmap| bitmap.contains(target_fragments[1].id() as u32))
+            })
+            .map(|segment| segment.uuid)
+            .expect("expected one committed segment to cover the unaffected fragment");
+
+        let remapper = DatasetIndexRemapperOptions::default()
+            .create_remapper(&dataset)
+            .unwrap();
+        let remapped = remapper
+            .remap_indices(HashMap::new(), &[target_fragments[0].id() as u64])
+            .await
+            .unwrap();
+
+        assert_eq!(remapped.len(), 1);
+        assert!(committed_ids.contains(&remapped[0].old_id));
+        assert_ne!(remapped[0].old_id, unaffected_segment_id);
+        assert_ne!(remapped[0].new_id, unaffected_segment_id);
     }
 }
diff --git a/rust/lance/src/dataset/index/frag_reuse.rs b/rust/lance/src/dataset/index/frag_reuse.rs
index cc7b845313d..4fbefcd4725 100644
--- a/rust/lance/src/dataset/index/frag_reuse.rs
+++ b/rust/lance/src/dataset/index/frag_reuse.rs
@@ -149,12 +149,13 @@ fn is_index_remap_caught_up(
 mod tests {
     use super::*;
     use crate::dataset::optimize::{CompactionOptions, compact_files, remapping};
+    use crate::index::DatasetIndexExt;
     use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount};
     use all_asserts::{assert_false, assert_true};
     use arrow_array::types::{Float32Type, Int32Type};
     use lance_datagen::Dimension;
+    use lance_index::IndexType;
     use lance_index::scalar::ScalarIndexParams;
-    use lance_index::{DatasetIndexExt, IndexType};
 
     #[tokio::test]
     async fn test_cleanup_frag_reuse_index() {
diff --git a/rust/lance/src/dataset/mem_wal.rs b/rust/lance/src/dataset/mem_wal.rs
index 0092385edf7..c207eef7174 100644
--- a/rust/lance/src/dataset/mem_wal.rs
+++ b/rust/lance/src/dataset/mem_wal.rs
@@ -8,7 +8,7 @@
 //!
 //! ## Architecture
 //!
-//! Each region has:
+//! Each shard has:
 //! - A **MemTable** for in-memory data (immediately queryable)
 //! - A **WAL Buffer** for durability (persisted to object storage)
 //! - **In-memory indexes** (BTree, IVF-PQ, FTS) for indexed queries
@@ -29,8 +29,8 @@
 //!
 //! ## Epoch-Based Fencing
 //!
-//! Each region has exactly one active writer at any time, enforced via
-//! monotonically increasing writer epochs in the region manifest.
+//! Each shard has exactly one active writer at any time, enforced via
+//! monotonically increasing writer epochs in the shard manifest.
 
 mod api;
 mod index;
@@ -42,8 +42,8 @@ mod wal;
 pub mod write;
 
 pub use api::{DatasetMemWalExt, MemWalConfig};
-pub use manifest::RegionManifestStore;
+pub use manifest::ShardManifestStore;
 pub use memtable::scanner::MemTableScanner;
-pub use scanner::{LsmDataSource, LsmGeneration, LsmScanner, RegionSnapshot};
-pub use write::RegionWriter;
-pub use write::RegionWriterConfig;
+pub use scanner::{LsmDataSource, LsmGeneration, LsmScanner, ShardSnapshot};
+pub use write::ShardWriter;
+pub use write::ShardWriterConfig;
diff --git a/rust/lance/src/dataset/mem_wal/api.rs b/rust/lance/src/dataset/mem_wal/api.rs
index 99bb415d6a0..83bfa014771 100644
--- a/rust/lance/src/dataset/mem_wal/api.rs
+++ b/rust/lance/src/dataset/mem_wal/api.rs
@@ -8,10 +8,10 @@
 
 use std::sync::Arc;
 
+use crate::index::DatasetIndexExt;
 use async_trait::async_trait;
 use lance_core::{Error, Result};
-use lance_index::DatasetIndexExt;
-use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndexDetails, RegionSpec};
+use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndexDetails, ShardSpec};
 use lance_index::vector::ivf::storage::IvfModel;
 use lance_index::vector::pq::ProductQuantizer;
 use lance_io::object_store::ObjectStore;
@@ -24,19 +24,19 @@ use crate::dataset::transaction::{Operation, Transaction};
 use crate::index::DatasetIndexInternalExt;
 use crate::index::mem_wal::new_mem_wal_index_meta;
 
-use super::RegionWriterConfig;
+use super::ShardWriterConfig;
 use super::write::MemIndexConfig;
-use super::write::RegionWriter;
+use super::write::ShardWriter;
 
 /// Configuration for initializing MemWAL on a Dataset.
 #[derive(Debug, Clone, Default)]
 pub struct MemWalConfig {
-    /// Optional region specification for partitioning writes.
+    /// Optional shard specification for partitioning writes.
     ///
-    /// If None, MemWAL is initialized without any region spec (manual region management).
+    /// If None, MemWAL is initialized without any shard spec (manual shard management).
     ///
-    /// TODO: Add `add_region_spec()` API to add region specs after initialization.
-    pub region_spec: Option<RegionSpec>,
+    /// TODO: Add `add_shard_spec()` API to add shard specs after initialization.
+    pub shard_spec: Option<ShardSpec>,
     /// Index names to maintain in MemTables.
     /// These must reference indexes already defined on the base table.
     pub maintained_indexes: Vec<String>,
@@ -55,20 +55,20 @@ pub trait DatasetMemWalExt {
     /// ```ignore
     /// let mut dataset = Dataset::open("s3://bucket/dataset").await?;
     /// dataset.initialize_mem_wal(MemWalConfig {
-    ///     region_specs: vec![],
+    ///     shard_specs: vec![],
     ///     maintained_indexes: vec!["id_btree".to_string()],
     /// }).await?;
     /// ```
     async fn initialize_mem_wal(&mut self, config: MemWalConfig) -> Result<()>;
 
-    /// Get a RegionWriter for the specified region.
+    /// Get a ShardWriter for the specified shard.
     ///
     /// Automatically loads index configurations from the MemWalIndex
     /// and creates the appropriate in-memory indexes.
     ///
     /// # Arguments
     ///
-    /// * `region_id` - UUID identifying this region
+    /// * `shard_id` - UUID identifying this shard
     /// * `config` - Writer configuration (durability, buffer sizes, etc.)
     ///
     /// # Example
@@ -76,15 +76,15 @@ pub trait DatasetMemWalExt {
     /// ```ignore
     /// let writer = dataset.mem_wal_writer(
     ///     Uuid::new_v4(),
-    ///     RegionWriterConfig::default(),
+    ///     ShardWriterConfig::default(),
     /// ).await?;
     /// writer.put(vec![batch1, batch2]).await?;
     /// ```
     async fn mem_wal_writer(
         &self,
-        region_id: Uuid,
-        config: RegionWriterConfig,
-    ) -> Result<RegionWriter>;
+        shard_id: Uuid,
+        config: ShardWriterConfig,
+    ) -> Result<ShardWriter>;
 }
 
 #[async_trait]
@@ -119,7 +119,7 @@ impl DatasetMemWalExt for Dataset {
 
         // Create MemWalIndexDetails
         let details = MemWalIndexDetails {
-            region_specs: config.region_spec.into_iter().collect(),
+            shard_specs: config.shard_spec.into_iter().collect(),
             maintained_indexes: config.maintained_indexes,
             ..Default::default()
         };
@@ -149,9 +149,9 @@ impl DatasetMemWalExt for Dataset {
 
     async fn mem_wal_writer(
         &self,
-        region_id: Uuid,
-        mut config: RegionWriterConfig,
-    ) -> Result<RegionWriter> {
+        shard_id: Uuid,
+        mut config: ShardWriterConfig,
+    ) -> Result<ShardWriter> {
         use lance_index::metrics::NoOpMetricsCollector;
 
         // Load MemWalIndex to get maintained_indexes
@@ -214,15 +214,15 @@ impl DatasetMemWalExt for Dataset {
             };
         }
 
-        // Set region_id in config
-        config.region_id = region_id;
+        // Set shard_id in config
+        config.shard_id = shard_id;
 
         // Get object store and base path
         let base_uri = self.uri();
         let (store, base_path) = ObjectStore::from_uri(base_uri).await?;
 
-        // Create RegionWriter
-        RegionWriter::open(
+        // Create ShardWriter
+        ShardWriter::open(
             store,
             base_path,
             base_uri,
diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs
index e7eb7394c45..a267a9d2e14 100644
--- a/rust/lance/src/dataset/mem_wal/index.rs
+++ b/rust/lance/src/dataset/mem_wal/index.rs
@@ -31,6 +31,7 @@ use lance_index::vector::pq::ProductQuantizer;
 use lance_linalg::distance::DistanceType;
 use lance_table::format::IndexMetadata;
 use prost::Message as _;
+use tracing::instrument;
 
 /// Row position in MemTable.
 ///
@@ -331,6 +332,7 @@ impl IndexStore {
     }
 
     /// Insert a batch into all indexes with batch position tracking.
+    #[instrument(name = "idx_insert_batch", level = "debug", skip_all, fields(num_rows = batch.num_rows(), row_offset, batch_position))]
     pub fn insert_with_batch_position(
         &self,
         batch: &RecordBatch,
@@ -378,6 +380,7 @@ impl IndexStore {
     /// For IVF-PQ indexes, this enables vectorized partition assignment and
     /// PQ encoding across all batches, improving performance through better
     /// SIMD utilization.
+    #[instrument(name = "idx_insert_batches", level = "debug", skip_all, fields(batch_count = batches.len()))]
     pub fn insert_batches(&self, batches: &[StoredBatch]) -> Result<()> {
         if batches.is_empty() {
             return Ok(());
@@ -419,6 +422,7 @@ impl IndexStore {
     ///
     /// Returns a map of index names to their update durations for performance tracking.
     #[allow(clippy::print_stderr)]
+    #[instrument(name = "idx_insert_batches_parallel", level = "debug", skip_all, fields(batch_count = batches.len()))]
     pub fn insert_batches_parallel(
         &self,
         batches: &[StoredBatch],
diff --git a/rust/lance/src/dataset/mem_wal/index/fts.rs b/rust/lance/src/dataset/mem_wal/index/fts.rs
index 3130f6474ce..59200654b32 100644
--- a/rust/lance/src/dataset/mem_wal/index/fts.rs
+++ b/rust/lance/src/dataset/mem_wal/index/fts.rs
@@ -33,8 +33,8 @@ use crossbeam_skiplist::SkipMap;
 use datafusion::common::ScalarValue;
 use lance_core::{Error, Result};
 use lance_index::scalar::InvertedIndexParams;
-use lance_index::scalar::inverted::tokenizer::lance_tokenizer::LanceTokenizer;
-use tantivy::tokenizer::TokenStream;
+use lance_index::scalar::inverted::tokenizer::document_tokenizer::LanceTokenizer;
+use lance_tokenizer::TokenStream;
 
 use super::RowPosition;
 
@@ -96,11 +96,11 @@ pub enum FtsQueryExpr {
     /// Boolean combination of queries.
     Boolean {
         /// All MUST clauses must match for a document to be included.
-        must: Vec<FtsQueryExpr>,
+        must: Vec<Self>,
         /// At least one SHOULD clause should match (adds to score).
-        should: Vec<FtsQueryExpr>,
+        should: Vec<Self>,
         /// No MUST_NOT clause may match (excludes documents).
-        must_not: Vec<FtsQueryExpr>,
+        must_not: Vec<Self>,
     },
     /// Boosting query with positive and optional negative components.
     ///
@@ -109,9 +109,9 @@ pub enum FtsQueryExpr {
     /// and negative have their scores reduced by `negative_boost`.
     Boost {
         /// The primary query (documents must match this).
-        positive: Box<FtsQueryExpr>,
+        positive: Box<Self>,
         /// Optional query to demote matching documents.
-        negative: Option<Box<FtsQueryExpr>>,
+        negative: Option<Box<Self>>,
         /// Boost factor for documents matching negative query (typically < 1.0).
         /// Score becomes: original_score * negative_boost for docs matching negative.
         negative_boost: f32,
diff --git a/rust/lance/src/dataset/mem_wal/manifest.rs b/rust/lance/src/dataset/mem_wal/manifest.rs
index 6bca29568c2..744dcb2def2 100644
--- a/rust/lance/src/dataset/mem_wal/manifest.rs
+++ b/rust/lance/src/dataset/mem_wal/manifest.rs
@@ -1,15 +1,15 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-//! Region manifest storage with bit-reversed versioned naming.
+//! Shard manifest storage with bit-reversed versioned naming.
 //!
-//! Region manifests are stored as versioned protobuf files using bit-reversed
+//! Shard manifests are stored as versioned protobuf files using bit-reversed
 //! naming scheme to distribute files across object store keyspace.
 //!
 //! ## File Layout
 //!
 //! ```text
-//! _mem_wal/{region_id}/manifest/
+//! _mem_wal/{shard_id}/manifest/
 //!   ├── {bit_reversed_version}.binpb  # Versioned manifest files
 //!   └── version_hint.json             # Best-effort version hint
 //! ```
@@ -33,7 +33,7 @@ use bytes::Bytes;
 use futures::StreamExt;
 use futures::stream::FuturesUnordered;
 use lance_core::{Error, Result};
-use lance_index::mem_wal::RegionManifest;
+use lance_index::mem_wal::ShardManifest;
 use lance_io::object_store::ObjectStore;
 use lance_table::format::pb;
 use log::{info, warn};
@@ -42,9 +42,10 @@ use object_store::PutOptions;
 use object_store::path::Path;
 use prost::Message;
 use serde::{Deserialize, Serialize};
+use tracing::instrument;
 use uuid::Uuid;
 
-use super::util::{manifest_filename, parse_bit_reversed_filename, region_manifest_path};
+use super::util::{manifest_filename, parse_bit_reversed_filename, shard_manifest_path};
 
 /// Version hint file structure.
 #[derive(Debug, Serialize, Deserialize)]
@@ -52,37 +53,37 @@ struct VersionHint {
     version: u64,
 }
 
-/// Store for reading and writing region manifests.
+/// Store for reading and writing shard manifests.
 ///
 /// Handles versioned manifest files with bit-reversed naming scheme
 /// and PUT-IF-NOT-EXISTS atomicity.
 #[derive(Debug)]
-pub struct RegionManifestStore {
+pub struct ShardManifestStore {
     object_store: Arc<ObjectStore>,
-    region_id: Uuid,
+    shard_id: Uuid,
     manifest_dir: Path,
     manifest_scan_batch_size: usize,
 }
 
-impl RegionManifestStore {
-    /// Create a new manifest store for the given region.
+impl ShardManifestStore {
+    /// Create a new manifest store for the given shard.
     ///
     /// # Arguments
     ///
     /// * `object_store` - Object store for reading/writing manifests
     /// * `base_path` - Base path within the object store (from ObjectStore::from_uri)
-    /// * `region_id` - Region UUID
+    /// * `shard_id` - Shard UUID
     /// * `manifest_scan_batch_size` - Batch size for parallel HEAD requests when scanning versions
     pub fn new(
         object_store: Arc<ObjectStore>,
         base_path: &Path,
-        region_id: Uuid,
+        shard_id: Uuid,
         manifest_scan_batch_size: usize,
     ) -> Self {
-        let manifest_dir = region_manifest_path(base_path, &region_id);
+        let manifest_dir = shard_manifest_path(base_path, &shard_id);
         Self {
             object_store,
-            region_id,
+            shard_id,
             manifest_dir,
             manifest_scan_batch_size,
         }
@@ -90,8 +91,9 @@ impl RegionManifestStore {
 
     /// Read the latest manifest version.
     ///
-    /// Returns `None` if no manifest exists (new region).
-    pub async fn read_latest(&self) -> Result<Option<RegionManifest>> {
+    /// Returns `None` if no manifest exists (new shard).
+    #[instrument(name = "manifest_read_latest", level = "debug", skip_all, fields(shard_id = %self.shard_id))]
+    pub async fn read_latest(&self) -> Result<Option<ShardManifest>> {
         let version = self.find_latest_version().await?;
         if version == 0 {
             return Ok(None);
@@ -101,14 +103,14 @@ impl RegionManifestStore {
     }
 
     /// Read a specific manifest version.
-    pub async fn read_version(&self, version: u64) -> Result<RegionManifest> {
+    pub async fn read_version(&self, version: u64) -> Result<ShardManifest> {
         let filename = manifest_filename(version);
         let path = self.manifest_dir.child(filename.as_str());
 
         let data = self.object_store.inner.get(&path).await.map_err(|e| {
             Error::io(format!(
-                "Failed to read manifest version {} for region {}: {}",
-                version, self.region_id, e
+                "Failed to read manifest version {} for shard {}: {}",
+                version, self.shard_id, e
             ))
         })?;
 
@@ -117,10 +119,10 @@ impl RegionManifestStore {
             .await
             .map_err(|e| Error::io(format!("Failed to read manifest bytes: {}", e)))?;
 
-        let pb_manifest = pb::RegionManifest::decode(bytes)
+        let pb_manifest = pb::ShardManifest::decode(bytes)
             .map_err(|e| Error::io(format!("Failed to decode manifest protobuf: {}", e)))?;
 
-        RegionManifest::try_from(pb_manifest)
+        ShardManifest::try_from(pb_manifest)
     }
 
     /// Write a new manifest version atomically.
@@ -134,12 +136,13 @@ impl RegionManifestStore {
     /// # Errors
     ///
     /// Returns `Error::AlreadyExists` if another writer already wrote this version.
-    pub async fn write(&self, manifest: &RegionManifest) -> Result<u64> {
+    #[instrument(name = "manifest_write", level = "debug", skip_all, fields(shard_id = %self.shard_id, version = manifest.version, epoch = manifest.writer_epoch))]
+    pub async fn write(&self, manifest: &ShardManifest) -> Result<u64> {
         let version = manifest.version;
         let filename = manifest_filename(version);
         let path = self.manifest_dir.child(filename.as_str());
 
-        let pb_manifest = pb::RegionManifest::from(manifest);
+        let pb_manifest = pb::ShardManifest::from(manifest);
         let bytes = pb_manifest.encode_to_vec();
 
         if self.object_store.is_local() {
@@ -166,16 +169,16 @@ impl RegionManifestStore {
                     // Clean up temp file
                     let _ = self.object_store.delete(&temp_path).await;
                     return Err(Error::io(format!(
-                        "Manifest version {} already exists for region {}",
-                        version, self.region_id
+                        "Manifest version {} already exists for shard {}",
+                        version, self.shard_id
                     )));
                 }
                 Err(e) => {
                     // Clean up temp file
                     let _ = self.object_store.delete(&temp_path).await;
                     return Err(Error::io(format!(
-                        "Failed to write manifest version {} for region {}: {}",
-                        version, self.region_id, e
+                        "Failed to write manifest version {} for shard {}: {}",
+                        version, self.shard_id, e
                     )));
                 }
             }
@@ -193,13 +196,13 @@ impl RegionManifestStore {
                 .map_err(|e| {
                     if matches!(e, object_store::Error::AlreadyExists { .. }) {
                         Error::io(format!(
-                            "Manifest version {} already exists for region {}",
-                            version, self.region_id
+                            "Manifest version {} already exists for shard {}",
+                            version, self.shard_id
                         ))
                     } else {
                         Error::io(format!(
-                            "Failed to write manifest version {} for region {}: {}",
-                            version, self.region_id, e
+                            "Failed to write manifest version {} for shard {}: {}",
+                            version, self.shard_id, e
                         ))
                     }
                 })?;
@@ -299,8 +302,8 @@ impl RegionManifestStore {
                     .await
                 {
                     warn!(
-                        "Failed to write version hint for region {}: {}",
-                        self.region_id, e
+                        "Failed to write version hint for shard {}: {}",
+                        self.shard_id, e
                     );
                 }
             }
@@ -341,35 +344,36 @@ impl RegionManifestStore {
         Ok(versions)
     }
 
-    /// Get the region ID.
-    pub fn region_id(&self) -> Uuid {
-        self.region_id
+    /// Get the shard ID.
+    pub fn shard_id(&self) -> Uuid {
+        self.shard_id
     }
 
     // ========================================================================
     // Epoch-based Writer Fencing
     // ========================================================================
 
-    /// Claim a region by incrementing its writer epoch.
+    /// Claim a shard by incrementing its writer epoch.
     ///
     /// This establishes single-writer semantics by:
     /// 1. Loading the current manifest (or creating initial state)
     /// 2. Incrementing the writer epoch
     /// 3. Atomically writing the new manifest
     ///
-    /// If another writer has already claimed the region (version conflict),
+    /// If another writer has already claimed the shard (version conflict),
     /// this fails immediately rather than retrying. This prevents "epoch wars"
     /// where multiple writers keep fencing each other.
     ///
     /// # Returns
     ///
-    /// A tuple of `(epoch, RegionManifest)` where the manifest is the
+    /// A tuple of `(epoch, ShardManifest)` where the manifest is the
     /// claimed state (may be freshly created or loaded and epoch-bumped).
     ///
     /// # Errors
     ///
-    /// Returns an error if another writer already claimed the region.
-    pub async fn claim_epoch(&self, region_spec_id: u32) -> Result<(u64, RegionManifest)> {
+    /// Returns an error if another writer already claimed the shard.
+    #[instrument(name = "manifest_claim_epoch", level = "info", skip_all, fields(shard_id = %self.shard_id, shard_spec_id))]
+    pub async fn claim_epoch(&self, shard_spec_id: u32) -> Result<(u64, ShardManifest)> {
         let current = self.read_latest().await?;
 
         let (next_version, next_epoch, base_manifest) = match current {
@@ -378,16 +382,16 @@ impl RegionManifestStore {
         };
 
         let new_manifest = if let Some(base) = base_manifest {
-            RegionManifest {
+            ShardManifest {
                 version: next_version,
                 writer_epoch: next_epoch,
                 ..base
             }
         } else {
-            RegionManifest {
-                region_id: self.region_id,
+            ShardManifest {
+                shard_id: self.shard_id,
                 version: next_version,
-                region_spec_id,
+                shard_spec_id,
                 writer_epoch: next_epoch,
                 replay_after_wal_entry_position: 0,
                 wal_entry_position_last_seen: 0,
@@ -398,14 +402,14 @@ impl RegionManifestStore {
 
         self.write(&new_manifest).await.map_err(|e| {
             Error::io(format!(
-                "Failed to claim region {} (version {}): another writer may have claimed it: {}",
-                self.region_id, next_version, e
+                "Failed to claim shard {} (version {}): another writer may have claimed it: {}",
+                self.shard_id, next_version, e
             ))
         })?;
 
         info!(
-            "Claimed region {} with epoch {} (version {})",
-            self.region_id, next_epoch, next_version
+            "Claimed shard {} with epoch {} (version {})",
+            self.shard_id, next_epoch, next_version
         );
 
         Ok((next_epoch, new_manifest))
@@ -415,21 +419,22 @@ impl RegionManifestStore {
     ///
     /// Loads the current manifest and compares epochs. If the stored epoch
     /// is higher than the local epoch, the writer has been fenced.
+    #[instrument(name = "manifest_check_fenced", level = "debug", skip_all, fields(shard_id = %self.shard_id, local_epoch))]
     pub async fn check_fenced(&self, local_epoch: u64) -> Result<()> {
         let current = self.read_latest().await?;
-        Self::check_fenced_against(&current, local_epoch, self.region_id)
+        Self::check_fenced_against(&current, local_epoch, self.shard_id)
     }
 
     /// Check fencing against a pre-read manifest (avoids redundant read).
     fn check_fenced_against(
-        manifest: &Option<RegionManifest>,
+        manifest: &Option<ShardManifest>,
         local_epoch: u64,
-        region_id: Uuid,
+        shard_id: Uuid,
     ) -> Result<()> {
         match manifest {
             Some(m) if m.writer_epoch > local_epoch => Err(Error::io(format!(
-                "Writer fenced: local epoch {} < stored epoch {} for region {}",
-                local_epoch, m.writer_epoch, region_id
+                "Writer fenced: local epoch {} < stored epoch {} for shard {}",
+                local_epoch, m.writer_epoch, shard_id
             ))),
             _ => Ok(()),
         }
@@ -452,9 +457,10 @@ impl RegionManifestStore {
     /// # Returns
     ///
     /// The successfully written manifest.
-    pub async fn commit_update<F>(&self, local_epoch: u64, prepare_fn: F) -> Result<RegionManifest>
+    #[instrument(name = "manifest_commit_update", level = "debug", skip_all, fields(shard_id = %self.shard_id, local_epoch))]
+    pub async fn commit_update<F>(&self, local_epoch: u64, prepare_fn: F) -> Result<ShardManifest>
     where
-        F: Fn(&RegionManifest) -> RegionManifest,
+        F: Fn(&ShardManifest) -> ShardManifest,
     {
         const MAX_RETRIES: usize = 10;
 
@@ -463,10 +469,10 @@ impl RegionManifestStore {
             let current = self
                 .read_latest()
                 .await?
-                .ok_or_else(|| Error::io("Region manifest not found"))?;
+                .ok_or_else(|| Error::io("Shard manifest not found"))?;
 
             // Step 2: Check fencing
-            Self::check_fenced_against(&Some(current.clone()), local_epoch, self.region_id)?;
+            Self::check_fenced_against(&Some(current.clone()), local_epoch, self.shard_id)?;
 
             // Step 3: Prepare new manifest
             let new_manifest = prepare_fn(&current);
@@ -498,8 +504,8 @@ impl RegionManifestStore {
         }
 
         Err(Error::io(format!(
-            "Failed to update manifest for region {} after {} attempts",
-            self.region_id, MAX_RETRIES
+            "Failed to update manifest for shard {} after {} attempts",
+            self.shard_id, MAX_RETRIES
         )))
     }
 }
@@ -516,11 +522,11 @@ mod tests {
         (store, path, temp_dir)
     }
 
-    fn create_test_manifest(region_id: Uuid, version: u64, epoch: u64) -> RegionManifest {
-        RegionManifest {
-            region_id,
+    fn create_test_manifest(shard_id: Uuid, version: u64, epoch: u64) -> ShardManifest {
+        ShardManifest {
+            shard_id,
             version,
-            region_spec_id: 0,
+            shard_spec_id: 0,
             writer_epoch: epoch,
             replay_after_wal_entry_position: 0,
             wal_entry_position_last_seen: 0,
@@ -532,8 +538,8 @@ mod tests {
     #[tokio::test]
     async fn test_read_latest_empty() {
         let (store, base_path, _temp_dir) = create_local_store().await;
-        let region_id = Uuid::new_v4();
-        let manifest_store = RegionManifestStore::new(store, &base_path, region_id, 2);
+        let shard_id = Uuid::new_v4();
+        let manifest_store = ShardManifestStore::new(store, &base_path, shard_id, 2);
 
         let result = manifest_store.read_latest().await.unwrap();
         assert!(result.is_none());
@@ -542,27 +548,27 @@ mod tests {
     #[tokio::test]
     async fn test_write_and_read_manifest() {
         let (store, base_path, _temp_dir) = create_local_store().await;
-        let region_id = Uuid::new_v4();
-        let manifest_store = RegionManifestStore::new(store, &base_path, region_id, 2);
+        let shard_id = Uuid::new_v4();
+        let manifest_store = ShardManifestStore::new(store, &base_path, shard_id, 2);
 
-        let manifest = create_test_manifest(region_id, 1, 1);
+        let manifest = create_test_manifest(shard_id, 1, 1);
         manifest_store.write(&manifest).await.unwrap();
 
         let loaded = manifest_store.read_latest().await.unwrap().unwrap();
         assert_eq!(loaded.version, 1);
         assert_eq!(loaded.writer_epoch, 1);
-        assert_eq!(loaded.region_id, region_id);
+        assert_eq!(loaded.shard_id, shard_id);
     }
 
     #[tokio::test]
     async fn test_multiple_versions() {
         let (store, base_path, _temp_dir) = create_local_store().await;
-        let region_id = Uuid::new_v4();
-        let manifest_store = RegionManifestStore::new(store, &base_path, region_id, 2);
+        let shard_id = Uuid::new_v4();
+        let manifest_store = ShardManifestStore::new(store, &base_path, shard_id, 2);
 
         // Write multiple versions
         for version in 1..=5 {
-            let manifest = create_test_manifest(region_id, version, version);
+            let manifest = create_test_manifest(shard_id, version, version);
             manifest_store.write(&manifest).await.unwrap();
         }
 
@@ -579,11 +585,11 @@ mod tests {
     #[tokio::test]
     async fn test_read_specific_version() {
         let (store, base_path, _temp_dir) = create_local_store().await;
-        let region_id = Uuid::new_v4();
-        let manifest_store = RegionManifestStore::new(store, &base_path, region_id, 2);
+        let shard_id = Uuid::new_v4();
+        let manifest_store = ShardManifestStore::new(store, &base_path, shard_id, 2);
 
         for version in 1..=3 {
-            let manifest = create_test_manifest(region_id, version, version * 10);
+            let manifest = create_test_manifest(shard_id, version, version * 10);
             manifest_store.write(&manifest).await.unwrap();
         }
 
@@ -595,14 +601,14 @@ mod tests {
     #[tokio::test]
     async fn test_put_if_not_exists() {
         let (store, base_path, _temp_dir) = create_local_store().await;
-        let region_id = Uuid::new_v4();
-        let manifest_store = RegionManifestStore::new(store, &base_path, region_id, 2);
+        let shard_id = Uuid::new_v4();
+        let manifest_store = ShardManifestStore::new(store, &base_path, shard_id, 2);
 
-        let manifest1 = create_test_manifest(region_id, 1, 1);
+        let manifest1 = create_test_manifest(shard_id, 1, 1);
         manifest_store.write(&manifest1).await.unwrap();
 
         // Second write to same version should fail
-        let manifest2 = create_test_manifest(region_id, 1, 2);
+        let manifest2 = create_test_manifest(shard_id, 1, 2);
         let result = manifest_store.write(&manifest2).await;
         assert!(result.is_err());
     }
diff --git a/rust/lance/src/dataset/mem_wal/memtable.rs b/rust/lance/src/dataset/mem_wal/memtable.rs
index a93b5627f2d..7f40ff35df4 100644
--- a/rust/lance/src/dataset/mem_wal/memtable.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable.rs
@@ -17,6 +17,7 @@ use lance_core::datatypes::Schema;
 use lance_core::{Error, Result};
 use lance_index::scalar::bloomfilter::sbbf::Sbbf;
 use tokio::sync::RwLock;
+use tracing::instrument;
 use uuid::Uuid;
 
 use super::index::IndexStore;
@@ -341,6 +342,7 @@ impl MemTable {
     /// # Single Writer Requirement
     ///
     /// This method MUST only be called from the single writer task.
+    #[instrument(name = "mt_insert", level = "debug", skip_all, fields(num_rows = batch.num_rows(), generation = self.generation))]
     pub async fn insert(&mut self, batch: RecordBatch) -> Result<usize> {
         // Validate schema compatibility
         if batch.schema() != self.schema {
@@ -423,6 +425,7 @@ impl MemTable {
     /// # Single Writer Requirement
     ///
     /// This method MUST only be called from the single writer task.
+    #[instrument(name = "mt_insert_batches", level = "debug", skip_all, fields(batch_count = batches.len(), generation = self.generation))]
     pub async fn insert_batches_only(
         &mut self,
         batches: Vec<RecordBatch>,
diff --git a/rust/lance/src/dataset/mem_wal/memtable/flush.rs b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
index eb91e856673..15c9da44f7a 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/flush.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
@@ -9,18 +9,19 @@ use bytes::Bytes;
 use lance_core::cache::LanceCache;
 use lance_core::{Error, Result};
 use lance_index::IndexType;
-use lance_index::mem_wal::{FlushedGeneration, RegionManifest};
+use lance_index::mem_wal::{FlushedGeneration, ShardManifest};
 use lance_index::scalar::{IndexStore, ScalarIndexParams};
 use lance_io::object_store::ObjectStore;
 use lance_table::format::IndexMetadata;
 use log::info;
 use object_store::path::Path;
+use tracing::instrument;
 use uuid::Uuid;
 
 use super::super::index::MemIndexConfig;
 use super::super::memtable::MemTable;
 use crate::Dataset;
-use crate::dataset::mem_wal::manifest::RegionManifestStore;
+use crate::dataset::mem_wal::manifest::ShardManifestStore;
 use crate::dataset::mem_wal::util::{flushed_memtable_path, generate_random_hash};
 
 #[derive(Debug, Clone)]
@@ -34,8 +35,8 @@ pub struct MemTableFlusher {
     object_store: Arc<ObjectStore>,
     base_path: Path,
     base_uri: String,
-    region_id: Uuid,
-    manifest_store: Arc<RegionManifestStore>,
+    shard_id: Uuid,
+    manifest_store: Arc<ShardManifestStore>,
 }
 
 impl MemTableFlusher {
@@ -43,14 +44,14 @@ impl MemTableFlusher {
         object_store: Arc<ObjectStore>,
         base_path: Path,
         base_uri: impl Into<String>,
-        region_id: Uuid,
-        manifest_store: Arc<RegionManifestStore>,
+        shard_id: Uuid,
+        manifest_store: Arc<ShardManifestStore>,
     ) -> Self {
         Self {
             object_store,
             base_path,
             base_uri: base_uri.into(),
-            region_id,
+            shard_id,
             manifest_store,
         }
     }
@@ -77,6 +78,7 @@ impl MemTableFlusher {
     }
 
     /// Flush the MemTable to storage (data files, indexes, bloom filter).
+    #[instrument(name = "mt_flush_storage", level = "info", skip_all, fields(shard_id = %self.shard_id, epoch, generation = memtable.generation(), row_count = memtable.row_count()))]
     pub async fn flush(&self, memtable: &MemTable, epoch: u64) -> Result<FlushResult> {
         self.manifest_store.check_fenced(epoch).await?;
 
@@ -94,7 +96,7 @@ impl MemTableFlusher {
         let generation = memtable.generation();
         let gen_folder_name = format!("{}_gen_{}", random_hash, generation);
         let gen_path =
-            flushed_memtable_path(&self.base_path, &self.region_id, &random_hash, generation);
+            flushed_memtable_path(&self.base_path, &self.shard_id, &random_hash, generation);
 
         info!(
             "Flushing MemTable generation {} to {} ({} rows, {} batches)",
@@ -116,8 +118,8 @@ impl MemTableFlusher {
             .await?;
 
         info!(
-            "Flushed generation {} for region {} (manifest version {})",
-            generation, self.region_id, new_manifest.version
+            "Flushed generation {} for shard {} (manifest version {})",
+            generation, self.shard_id, new_manifest.version
         );
 
         Ok(FlushResult {
@@ -134,6 +136,7 @@ impl MemTableFlusher {
     ///
     /// Returns the total number of rows written, which is needed for
     /// reversing row positions in indexes.
+    #[instrument(name = "mt_write_data_file", level = "debug", skip_all, fields(path = %path))]
     async fn write_data_file(&self, path: &Path, memtable: &MemTable) -> Result<usize> {
         use arrow_array::RecordBatchIterator;
 
@@ -180,6 +183,7 @@ impl MemTableFlusher {
     }
 
     /// Flush the MemTable to storage with indexes.
+    #[instrument(name = "mt_flush_with_indexes", level = "info", skip_all, fields(shard_id = %self.shard_id, epoch, generation = memtable.generation(), row_count = memtable.row_count(), index_count = index_configs.len()))]
     pub async fn flush_with_indexes(
         &self,
         memtable: &MemTable,
@@ -202,7 +206,7 @@ impl MemTableFlusher {
         let generation = memtable.generation();
         let gen_folder_name = format!("{}_gen_{}", random_hash, generation);
         let gen_path =
-            flushed_memtable_path(&self.base_path, &self.region_id, &random_hash, generation);
+            flushed_memtable_path(&self.base_path, &self.shard_id, &random_hash, generation);
 
         info!(
             "Flushing MemTable generation {} with indexes to {} ({} rows, {} batches)",
@@ -247,11 +251,8 @@ impl MemTableFlusher {
                     index_meta.fields = vec![field_idx];
                     index_meta.dataset_version = dataset.version().version;
                     // Calculate fragment_bitmap from dataset fragments
-                    let fragment_ids: roaring::RoaringBitmap = dataset
-                        .get_fragments()
-                        .iter()
-                        .map(|f| f.id() as u32)
-                        .collect();
+                    let fragment_ids: roaring::RoaringBitmap =
+                        dataset.fragment_bitmap.as_ref().clone();
                     index_meta.fragment_bitmap = Some(fragment_ids);
 
                     // Commit the index to the dataset
@@ -290,8 +291,8 @@ impl MemTableFlusher {
             .await?;
 
         info!(
-            "Flushed generation {} for region {} (manifest version {})",
-            generation, self.region_id, new_manifest.version
+            "Flushed generation {} for shard {} (manifest version {})",
+            generation, self.shard_id, new_manifest.version
         );
 
         Ok(FlushResult {
@@ -403,7 +404,7 @@ impl MemTableFlusher {
         total_rows: usize,
     ) -> Result<()> {
         use lance_index::pbold;
-        use lance_index::scalar::inverted::INVERTED_INDEX_VERSION;
+        use lance_index::scalar::inverted::current_fts_format_version;
         use lance_index::scalar::lance_format::LanceIndexStore;
 
         let fts_configs: Vec<_> = index_configs
@@ -467,11 +468,7 @@ impl MemTableFlusher {
             let schema = dataset.schema();
             let field_idx = schema.field(&fts_cfg.column).map(|f| f.id).unwrap_or(0);
 
-            let fragment_ids: roaring::RoaringBitmap = dataset
-                .get_fragments()
-                .iter()
-                .map(|f| f.id() as u32)
-                .collect();
+            let fragment_ids: roaring::RoaringBitmap = dataset.fragment_bitmap.as_ref().clone();
 
             let index_meta = IndexMetadata {
                 uuid: index_uuid,
@@ -480,9 +477,10 @@ impl MemTableFlusher {
                 dataset_version: dataset.version().version,
                 fragment_bitmap: Some(fragment_ids),
                 index_details: Some(Arc::new(index_details)),
-                index_version: INVERTED_INDEX_VERSION as i32,
+                index_version: current_fts_format_version().index_version() as i32,
                 created_at: None,
                 base_id: None,
+                files: None,
             };
 
             // Commit the index to the dataset
@@ -726,19 +724,20 @@ impl MemTableFlusher {
             base_id: None,
             created_at: Some(chrono::Utc::now()),
             index_version: 1,
+            files: None,
         };
 
         Ok(index_meta)
     }
 
-    /// Update the region manifest with the new flushed generation.
+    /// Update the shard manifest with the new flushed generation.
     async fn update_manifest(
         &self,
         epoch: u64,
         generation: u64,
         gen_path: &str,
         covered_wal_entry_position: u64,
-    ) -> Result<RegionManifest> {
+    ) -> Result<ShardManifest> {
         let gen_path = gen_path.to_string();
 
         self.manifest_store
@@ -749,7 +748,7 @@ impl MemTableFlusher {
                     path: gen_path.clone(),
                 });
 
-                RegionManifest {
+                ShardManifest {
                     version: current.version + 1,
                     replay_after_wal_entry_position: covered_wal_entry_position,
                     wal_entry_position_last_seen: current
@@ -861,15 +860,15 @@ mod tests {
     #[tokio::test]
     async fn test_flusher_requires_wal_flush() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
-        let region_id = Uuid::new_v4();
-        let manifest_store = Arc::new(RegionManifestStore::new(
+        let shard_id = Uuid::new_v4();
+        let manifest_store = Arc::new(ShardManifestStore::new(
             store.clone(),
             &base_path,
-            region_id,
+            shard_id,
             2,
         ));
 
-        // Claim region
+        // Claim shard
         let (epoch, _manifest) = manifest_store.claim_epoch(0).await.unwrap();
 
         let schema = create_test_schema();
@@ -882,7 +881,7 @@ mod tests {
         // Not flushed to WAL yet
         assert!(!memtable.all_flushed_to_wal());
 
-        let flusher = MemTableFlusher::new(store, base_path, base_uri, region_id, manifest_store);
+        let flusher = MemTableFlusher::new(store, base_path, base_uri, shard_id, manifest_store);
         let result = flusher.flush(&memtable, epoch).await;
 
         assert!(result.is_err());
@@ -897,21 +896,21 @@ mod tests {
     #[tokio::test]
     async fn test_flusher_empty_memtable() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
-        let region_id = Uuid::new_v4();
-        let manifest_store = Arc::new(RegionManifestStore::new(
+        let shard_id = Uuid::new_v4();
+        let manifest_store = Arc::new(ShardManifestStore::new(
             store.clone(),
             &base_path,
-            region_id,
+            shard_id,
             2,
         ));
 
-        // Claim region
+        // Claim shard
         let (epoch, _manifest) = manifest_store.claim_epoch(0).await.unwrap();
 
         let schema = create_test_schema();
         let memtable = MemTable::new(schema, 1, vec![]).unwrap();
 
-        let flusher = MemTableFlusher::new(store, base_path, base_uri, region_id, manifest_store);
+        let flusher = MemTableFlusher::new(store, base_path, base_uri, shard_id, manifest_store);
         let result = flusher.flush(&memtable, epoch).await;
 
         assert!(result.is_err());
@@ -921,15 +920,15 @@ mod tests {
     #[tokio::test]
     async fn test_flusher_success() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
-        let region_id = Uuid::new_v4();
-        let manifest_store = Arc::new(RegionManifestStore::new(
+        let shard_id = Uuid::new_v4();
+        let manifest_store = Arc::new(ShardManifestStore::new(
             store.clone(),
             &base_path,
-            region_id,
+            shard_id,
             2,
         ));
 
-        // Claim region
+        // Claim shard
         let (epoch, _manifest) = manifest_store.claim_epoch(0).await.unwrap();
 
         let schema = create_test_schema();
@@ -947,7 +946,7 @@ mod tests {
             store.clone(),
             base_path,
             base_uri,
-            region_id,
+            shard_id,
             manifest_store.clone(),
         );
         let result = flusher.flush(&memtable, epoch).await.unwrap();
@@ -967,18 +966,18 @@ mod tests {
     #[tokio::test]
     async fn test_flusher_with_btree_index() {
         use super::super::super::index::{BTreeIndexConfig, IndexStore};
-        use lance_index::DatasetIndexExt;
+        use crate::index::DatasetIndexExt;
 
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
-        let region_id = Uuid::new_v4();
-        let manifest_store = Arc::new(RegionManifestStore::new(
+        let shard_id = Uuid::new_v4();
+        let manifest_store = Arc::new(ShardManifestStore::new(
             store.clone(),
             &base_path,
-            region_id,
+            shard_id,
             2,
         ));
 
-        // Claim region
+        // Claim shard
         let (epoch, _manifest) = manifest_store.claim_epoch(0).await.unwrap();
 
         // Create index config for the 'id' column (field_id = 0)
@@ -1007,7 +1006,7 @@ mod tests {
             store.clone(),
             base_path.clone(),
             base_uri.clone(),
-            region_id,
+            shard_id,
             manifest_store.clone(),
         );
         let result = flusher
@@ -1022,7 +1021,7 @@ mod tests {
         // result.generation.path is just the folder name, construct full URI
         let gen_uri = format!(
             "{}/_mem_wal/{}/{}",
-            base_uri, region_id, result.generation.path
+            base_uri, shard_id, result.generation.path
         );
         let dataset = Dataset::open(&gen_uri).await.unwrap();
         let indices = dataset.load_indices().await.unwrap();
@@ -1065,24 +1064,24 @@ mod tests {
     #[tokio::test]
     async fn test_flusher_with_ivf_pq_index() {
         use super::super::super::index::{IndexStore, IvfPqIndexConfig};
+        use crate::index::DatasetIndexExt;
         use arrow_array::{FixedSizeListArray, Float32Array};
         use lance_arrow::FixedSizeListArrayExt;
-        use lance_index::DatasetIndexExt;
         use lance_index::vector::ivf::storage::IvfModel;
         use lance_index::vector::kmeans::{KMeansParams, train_kmeans};
         use lance_index::vector::pq::PQBuildParams;
         use lance_linalg::distance::DistanceType;
 
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
-        let region_id = Uuid::new_v4();
-        let manifest_store = Arc::new(RegionManifestStore::new(
+        let shard_id = Uuid::new_v4();
+        let manifest_store = Arc::new(ShardManifestStore::new(
             store.clone(),
             &base_path,
-            region_id,
+            shard_id,
             2,
         ));
 
-        // Claim region
+        // Claim shard
         let (epoch, _manifest) = manifest_store.claim_epoch(0).await.unwrap();
 
         // Create schema with vector column
@@ -1193,7 +1192,7 @@ mod tests {
             store.clone(),
             base_path.clone(),
             base_uri.clone(),
-            region_id,
+            shard_id,
             manifest_store.clone(),
         );
         let result = flusher
@@ -1207,7 +1206,7 @@ mod tests {
         // Verify the flushed dataset has the IVF-PQ index
         let gen_uri = format!(
             "{}/_mem_wal/{}/{}",
-            base_uri, region_id, result.generation.path
+            base_uri, shard_id, result.generation.path
         );
         let dataset = Dataset::open(&gen_uri).await.unwrap();
         let indices = dataset.load_indices().await.unwrap();
@@ -1289,21 +1288,21 @@ mod tests {
     #[tokio::test]
     async fn test_flusher_with_fts_index() {
         use super::super::super::index::{FtsIndexConfig, IndexStore};
+        use crate::index::DatasetIndexExt;
         use arrow_array::StringArray;
         use arrow_schema::{DataType, Field, Schema as ArrowSchema};
-        use lance_index::DatasetIndexExt;
         use std::sync::Arc;
 
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
-        let region_id = Uuid::new_v4();
-        let manifest_store = Arc::new(RegionManifestStore::new(
+        let shard_id = Uuid::new_v4();
+        let manifest_store = Arc::new(ShardManifestStore::new(
             store.clone(),
             &base_path,
-            region_id,
+            shard_id,
             2,
         ));
 
-        // Claim region
+        // Claim shard
         let (epoch, _manifest) = manifest_store.claim_epoch(0).await.unwrap();
 
         // Create schema with text column
@@ -1348,7 +1347,7 @@ mod tests {
             store.clone(),
             base_path.clone(),
             base_uri.clone(),
-            region_id,
+            shard_id,
             manifest_store.clone(),
         );
         let result = flusher
@@ -1362,7 +1361,7 @@ mod tests {
         // Verify the flushed dataset has the FTS index
         let gen_uri = format!(
             "{}/_mem_wal/{}/{}",
-            base_uri, region_id, result.generation.path
+            base_uri, shard_id, result.generation.path
         );
         let dataset = Dataset::open(&gen_uri).await.unwrap();
         let indices = dataset.load_indices().await.unwrap();
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs
index 6a848da0e7b..4451b1a1bc9 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs
@@ -231,73 +231,6 @@ impl FtsIndexExec {
             .collect()
     }
 
-    /// Materialize rows from batch store with score column (for unsorted results).
-    #[allow(dead_code)]
-    fn materialize_rows(&self, results: &[(u64, f32)]) -> DataFusionResult<Vec<RecordBatch>> {
-        if results.is_empty() {
-            return Ok(vec![]);
-        }
-
-        // Group rows by batch using binary search on pre-computed ranges
-        // Track (row_in_batch, score, original_row_position)
-        let mut batches_data: std::collections::HashMap<usize, Vec<(usize, f32, u64)>> =
-            std::collections::HashMap::new();
-
-        for &(pos, score) in results {
-            if let Some(batch) = self.find_batch(pos as usize) {
-                batches_data.entry(batch.batch_id).or_default().push((
-                    pos as usize - batch.start,
-                    score,
-                    pos,
-                ));
-            }
-        }
-
-        let mut all_batches = Vec::new();
-
-        for (batch_id, rows_with_score) in batches_data {
-            if let Some(stored) = self.batch_store.get(batch_id) {
-                let rows: Vec<u32> = rows_with_score.iter().map(|&(r, _, _)| r as u32).collect();
-                let scores: Vec<f32> = rows_with_score.iter().map(|&(_, s, _)| s).collect();
-                let row_positions: Vec<u64> =
-                    rows_with_score.iter().map(|&(_, _, pos)| pos).collect();
-
-                let indices = UInt32Array::from(rows);
-
-                let mut columns: Vec<Arc<dyn arrow_array::Array>> = stored
-                    .data
-                    .columns()
-                    .iter()
-                    .map(|col| arrow_select::take::take(col.as_ref(), &indices, None).unwrap())
-                    .collect();
-
-                // Add score column
-                columns.push(Arc::new(Float32Array::from(scores)));
-
-                // Apply projection if needed (excluding score column which is always included)
-                let mut final_columns = if let Some(ref proj_indices) = self.projection {
-                    let mut projected: Vec<_> =
-                        proj_indices.iter().map(|&i| columns[i].clone()).collect();
-                    // Always include score as last column
-                    projected.push(columns.last().unwrap().clone());
-                    projected
-                } else {
-                    columns
-                };
-
-                // Add _rowid column if requested
-                if self.with_row_id {
-                    final_columns.push(Arc::new(UInt64Array::from(row_positions)));
-                }
-
-                let batch = RecordBatch::try_new(self.output_schema.clone(), final_columns)?;
-                all_batches.push(batch);
-            }
-        }
-
-        Ok(all_batches)
-    }
-
     /// Materialize rows from batch store preserving input order (for sorted results).
     ///
     /// This method processes results one at a time to preserve the score-sorted order,
diff --git a/rust/lance/src/dataset/mem_wal/scanner.rs b/rust/lance/src/dataset/mem_wal/scanner.rs
index a6adb9f75e1..2d7daf44e5a 100644
--- a/rust/lance/src/dataset/mem_wal/scanner.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner.rs
@@ -23,7 +23,7 @@
 //! ```ignore
 //! use lance::dataset::mem_wal::scanner::LsmScanner;
 //!
-//! let scanner = LsmScanner::new(base_table, region_snapshots, vec!["pk".to_string()])
+//! let scanner = LsmScanner::new(base_table, shard_snapshots, vec!["pk".to_string()])
 //!     .project(&["id", "name"])
 //!     .filter("id > 10")?
 //!     .limit(100, None);
@@ -41,6 +41,6 @@ mod vector_search;
 
 pub use builder::LsmScanner;
 pub use collector::{ActiveMemTableRef, LsmDataSourceCollector};
-pub use data_source::{FlushedGeneration, LsmDataSource, LsmGeneration, RegionSnapshot};
+pub use data_source::{FlushedGeneration, LsmDataSource, LsmGeneration, ShardSnapshot};
 pub use point_lookup::LsmPointLookupPlanner;
 pub use vector_search::{DISTANCE_COLUMN, LsmVectorSearchPlanner};
diff --git a/rust/lance/src/dataset/mem_wal/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/scanner/builder.rs
index ddc14c08b33..3168f8f5feb 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/builder.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/builder.rs
@@ -16,7 +16,7 @@ use lance_core::{Error, Result};
 use uuid::Uuid;
 
 use super::collector::{ActiveMemTableRef, LsmDataSourceCollector};
-use super::data_source::RegionSnapshot;
+use super::data_source::ShardSnapshot;
 use super::planner::LsmScanPlanner;
 use crate::dataset::Dataset;
 
@@ -34,7 +34,7 @@ use crate::dataset::Dataset;
 /// # Example
 ///
 /// ```ignore
-/// let scanner = LsmScanner::new(base_table, region_snapshots, vec!["pk".to_string()])
+/// let scanner = LsmScanner::new(base_table, shard_snapshots, vec!["pk".to_string()])
 ///     .project(&["id", "name"])
 ///     .filter("id > 10")?
 ///     .limit(100, None);
@@ -44,7 +44,7 @@ use crate::dataset::Dataset;
 pub struct LsmScanner {
     // Data sources
     base_table: Arc<Dataset>,
-    region_snapshots: Vec<RegionSnapshot>,
+    shard_snapshots: Vec<ShardSnapshot>,
     active_memtables: HashMap<Uuid, ActiveMemTableRef>,
 
     // Query configuration
@@ -67,16 +67,16 @@ impl LsmScanner {
     /// # Arguments
     ///
     /// * `base_table` - The base Lance table (merged data)
-    /// * `region_snapshots` - Snapshots of region states from MemWAL index
+    /// * `shard_snapshots` - Snapshots of shard states from MemWAL index
     /// * `pk_columns` - Primary key column names for deduplication
     pub fn new(
         base_table: Arc<Dataset>,
-        region_snapshots: Vec<RegionSnapshot>,
+        shard_snapshots: Vec<ShardSnapshot>,
         pk_columns: Vec<String>,
     ) -> Self {
         Self {
             base_table,
-            region_snapshots,
+            shard_snapshots,
             active_memtables: HashMap::new(),
             projection: None,
             filter: None,
@@ -93,8 +93,8 @@ impl LsmScanner {
     /// Active MemTables contain data that may not be persisted yet.
     /// Including them provides strong consistency at the cost of
     /// requiring coordination with the writer.
-    pub fn with_active_memtable(mut self, region_id: Uuid, memtable: ActiveMemTableRef) -> Self {
-        self.active_memtables.insert(region_id, memtable);
+    pub fn with_active_memtable(mut self, shard_id: Uuid, memtable: ActiveMemTableRef) -> Self {
+        self.active_memtables.insert(shard_id, memtable);
         self
     }
 
@@ -223,10 +223,10 @@ impl LsmScanner {
     /// Build the data source collector.
     fn build_collector(&self) -> LsmDataSourceCollector {
         let mut collector =
-            LsmDataSourceCollector::new(self.base_table.clone(), self.region_snapshots.clone());
+            LsmDataSourceCollector::new(self.base_table.clone(), self.shard_snapshots.clone());
 
-        for (region_id, memtable) in &self.active_memtables {
-            collector = collector.with_active_memtable(*region_id, memtable.clone());
+        for (shard_id, memtable) in &self.active_memtables {
+            collector = collector.with_active_memtable(*shard_id, memtable.clone());
         }
 
         collector
@@ -237,7 +237,7 @@ impl std::fmt::Debug for LsmScanner {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("LsmScanner")
             .field("base_table", &self.base_table.uri())
-            .field("num_regions", &self.region_snapshots.len())
+            .field("num_shards", &self.shard_snapshots.len())
             .field("num_active_memtables", &self.active_memtables.len())
             .field("projection", &self.projection)
             .field("limit", &self.limit)
@@ -257,26 +257,26 @@ mod tests {
         // Full integration tests would require a real dataset
 
         let pk_columns = ["id".to_string()];
-        let region_snapshots: Vec<RegionSnapshot> = vec![];
+        let shard_snapshots: Vec<ShardSnapshot> = vec![];
 
         // We can't easily create an Arc<Dataset> without I/O,
         // so just test the type construction
         assert_eq!(pk_columns.len(), 1);
-        assert!(region_snapshots.is_empty());
+        assert!(shard_snapshots.is_empty());
     }
 
     #[test]
-    fn test_region_snapshot_construction() {
-        use super::super::data_source::RegionSnapshot;
+    fn test_shard_snapshot_construction() {
+        use super::super::data_source::ShardSnapshot;
 
-        let region_id = Uuid::new_v4();
-        let snapshot = RegionSnapshot::new(region_id)
+        let shard_id = Uuid::new_v4();
+        let snapshot = ShardSnapshot::new(shard_id)
             .with_spec_id(1)
             .with_current_generation(5)
             .with_flushed_generation(1, "path/gen_1".to_string())
             .with_flushed_generation(2, "path/gen_2".to_string());
 
-        assert_eq!(snapshot.region_id, region_id);
+        assert_eq!(snapshot.shard_id, shard_id);
         assert_eq!(snapshot.spec_id, 1);
         assert_eq!(snapshot.current_generation, 5);
         assert_eq!(snapshot.flushed_generations.len(), 2);
diff --git a/rust/lance/src/dataset/mem_wal/scanner/collector.rs b/rust/lance/src/dataset/mem_wal/scanner/collector.rs
index 90f38b3ea25..80c9bd26aec 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/collector.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/collector.rs
@@ -10,7 +10,7 @@ use arrow_schema::SchemaRef;
 use lance_core::Result;
 use uuid::Uuid;
 
-use super::data_source::{LsmDataSource, LsmGeneration, RegionSnapshot};
+use super::data_source::{LsmDataSource, LsmGeneration, ShardSnapshot};
 use crate::dataset::Dataset;
 use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
 
@@ -27,39 +27,39 @@ pub struct ActiveMemTableRef {
     pub generation: u64,
 }
 
-/// Collects data sources from base table and MemWAL regions.
+/// Collects data sources from base table and MemWAL shards.
 ///
 /// This collector gathers all data sources that need to be scanned
 /// for a query, including:
 /// - The base table (merged data)
-/// - Flushed MemTables from each region
+/// - Flushed MemTables from each shard
 /// - Active MemTables (optional, for strong consistency)
 pub struct LsmDataSourceCollector {
     /// Base Lance table.
     base_table: Arc<Dataset>,
     /// Base path for resolving relative paths.
     base_path: String,
-    /// Region snapshots from MemWAL index.
-    region_snapshots: Vec<RegionSnapshot>,
-    /// Active MemTables by region (for strong consistency).
+    /// Shard snapshots from MemWAL index.
+    shard_snapshots: Vec<ShardSnapshot>,
+    /// Active MemTables by shard (for strong consistency).
     active_memtables: HashMap<Uuid, ActiveMemTableRef>,
 }
 
 impl LsmDataSourceCollector {
-    /// Create a new collector from base table and region snapshots.
+    /// Create a new collector from base table and shard snapshots.
     ///
     /// # Arguments
     ///
     /// * `base_table` - The base Lance table (merged data)
-    /// * `region_snapshots` - Snapshots of region states from MemWAL index
-    pub fn new(base_table: Arc<Dataset>, region_snapshots: Vec<RegionSnapshot>) -> Self {
+    /// * `shard_snapshots` - Snapshots of shard states from MemWAL index
+    pub fn new(base_table: Arc<Dataset>, shard_snapshots: Vec<ShardSnapshot>) -> Self {
         // Use the dataset's URI as base path for resolving relative paths.
         // This ensures memory:// and other scheme-based URIs work correctly.
         let base_path = base_table.uri().trim_end_matches('/').to_string();
         Self {
             base_table,
             base_path,
-            region_snapshots,
+            shard_snapshots,
             active_memtables: HashMap::new(),
         }
     }
@@ -69,8 +69,8 @@ impl LsmDataSourceCollector {
     /// Active MemTables contain data that may not be persisted yet.
     /// Including them provides strong consistency at the cost of
     /// requiring coordination with the writer.
-    pub fn with_active_memtable(mut self, region_id: Uuid, memtable: ActiveMemTableRef) -> Self {
-        self.active_memtables.insert(region_id, memtable);
+    pub fn with_active_memtable(mut self, shard_id: Uuid, memtable: ActiveMemTableRef) -> Self {
+        self.active_memtables.insert(shard_id, memtable);
         self
     }
 
@@ -79,9 +79,9 @@ impl LsmDataSourceCollector {
         &self.base_table
     }
 
-    /// Get all region snapshots.
-    pub fn region_snapshots(&self) -> &[RegionSnapshot] {
-        &self.region_snapshots
+    /// Get all shard snapshots.
+    pub fn shard_snapshots(&self) -> &[ShardSnapshot] {
+        &self.shard_snapshots
     }
 
     /// Get active MemTables.
@@ -93,8 +93,8 @@ impl LsmDataSourceCollector {
     ///
     /// Returns sources in a consistent order:
     /// 1. Base table (gen=0)
-    /// 2. Flushed MemTables per region, ordered by generation
-    /// 3. Active MemTables per region
+    /// 2. Flushed MemTables per shard, ordered by generation
+    /// 3. Active MemTables per shard
     pub fn collect(&self) -> Result<Vec<LsmDataSource>> {
         let mut sources = Vec::new();
 
@@ -103,25 +103,25 @@ impl LsmDataSourceCollector {
             dataset: self.base_table.clone(),
         });
 
-        // 2. Add flushed MemTables from each region
-        for snapshot in &self.region_snapshots {
+        // 2. Add flushed MemTables from each shard
+        for snapshot in &self.shard_snapshots {
             for flushed in &snapshot.flushed_generations {
-                let path = self.resolve_flushed_path(&snapshot.region_id, &flushed.path);
+                let path = self.resolve_flushed_path(&snapshot.shard_id, &flushed.path);
                 sources.push(LsmDataSource::FlushedMemTable {
                     path,
-                    region_id: snapshot.region_id,
+                    shard_id: snapshot.shard_id,
                     generation: LsmGeneration::memtable(flushed.generation),
                 });
             }
         }
 
         // 3. Add active MemTables
-        for (region_id, memtable) in &self.active_memtables {
+        for (shard_id, memtable) in &self.active_memtables {
             sources.push(LsmDataSource::ActiveMemTable {
                 batch_store: memtable.batch_store.clone(),
                 index_store: memtable.index_store.clone(),
                 schema: memtable.schema.clone(),
-                region_id: *region_id,
+                shard_id: *shard_id,
                 generation: LsmGeneration::memtable(memtable.generation),
             });
         }
@@ -129,40 +129,40 @@ impl LsmDataSourceCollector {
         Ok(sources)
     }
 
-    /// Collect data sources for specific regions only.
+    /// Collect data sources for specific shards only.
     ///
-    /// This is used after region pruning to avoid loading data from
-    /// regions that cannot contain matching rows.
+    /// This is used after shard pruning to avoid loading data from
+    /// shards that cannot contain matching rows.
     ///
     /// The base table is always included since it may contain data
-    /// from any region (after merging).
-    pub fn collect_for_regions(&self, region_ids: &HashSet<Uuid>) -> Result<Vec<LsmDataSource>> {
+    /// from any shard (after merging).
+    pub fn collect_for_shards(&self, shard_ids: &HashSet<Uuid>) -> Result<Vec<LsmDataSource>> {
         let mut sources = Vec::new();
 
-        // Base table is always included (contains merged data from all regions)
+        // Base table is always included (contains merged data from all shards)
         sources.push(LsmDataSource::BaseTable {
             dataset: self.base_table.clone(),
         });
 
-        // Filter flushed MemTables by region
-        for snapshot in &self.region_snapshots {
-            if !region_ids.contains(&snapshot.region_id) {
+        // Filter flushed MemTables by shard
+        for snapshot in &self.shard_snapshots {
+            if !shard_ids.contains(&snapshot.shard_id) {
                 continue;
             }
 
             for flushed in &snapshot.flushed_generations {
-                let path = self.resolve_flushed_path(&snapshot.region_id, &flushed.path);
+                let path = self.resolve_flushed_path(&snapshot.shard_id, &flushed.path);
                 sources.push(LsmDataSource::FlushedMemTable {
                     path,
-                    region_id: snapshot.region_id,
+                    shard_id: snapshot.shard_id,
                     generation: LsmGeneration::memtable(flushed.generation),
                 });
             }
         }
 
-        // Filter active MemTables by region
-        for (region_id, memtable) in &self.active_memtables {
-            if !region_ids.contains(region_id) {
+        // Filter active MemTables by shard
+        for (shard_id, memtable) in &self.active_memtables {
+            if !shard_ids.contains(shard_id) {
                 continue;
             }
 
@@ -170,7 +170,7 @@ impl LsmDataSourceCollector {
                 batch_store: memtable.batch_store.clone(),
                 index_store: memtable.index_store.clone(),
                 schema: memtable.schema.clone(),
-                region_id: *region_id,
+                shard_id: *shard_id,
                 generation: LsmGeneration::memtable(memtable.generation),
             });
         }
@@ -181,7 +181,7 @@ impl LsmDataSourceCollector {
     /// Get the total number of data sources.
     pub fn num_sources(&self) -> usize {
         let flushed_count: usize = self
-            .region_snapshots
+            .shard_snapshots
             .iter()
             .map(|s| s.flushed_generations.len())
             .sum();
@@ -190,10 +190,10 @@ impl LsmDataSourceCollector {
 
     /// Resolve a flushed MemTable path to an absolute path.
     ///
-    /// Flushed MemTables are stored at: `{base_path}/_mem_wal/{region_id}/{folder_name}`
+    /// Flushed MemTables are stored at: `{base_path}/_mem_wal/{shard_id}/{folder_name}`
     /// The `folder_name` is what's stored in `FlushedGeneration.path`.
-    fn resolve_flushed_path(&self, region_id: &Uuid, folder_name: &str) -> String {
-        format!("{}/_mem_wal/{}/{}", self.base_path, region_id, folder_name)
+    fn resolve_flushed_path(&self, shard_id: &Uuid, folder_name: &str) -> String {
+        format!("{}/_mem_wal/{}/{}", self.base_path, shard_id, folder_name)
     }
 }
 
@@ -202,13 +202,13 @@ mod tests {
     use super::*;
     use crate::dataset::mem_wal::scanner::data_source::FlushedGeneration;
 
-    fn create_test_snapshots() -> Vec<RegionSnapshot> {
-        let region_a = Uuid::new_v4();
-        let region_b = Uuid::new_v4();
+    fn create_test_snapshots() -> Vec<ShardSnapshot> {
+        let shard_a = Uuid::new_v4();
+        let shard_b = Uuid::new_v4();
 
         vec![
-            RegionSnapshot {
-                region_id: region_a,
+            ShardSnapshot {
+                shard_id: shard_a,
                 spec_id: 1,
                 current_generation: 3,
                 flushed_generations: vec![
@@ -222,8 +222,8 @@ mod tests {
                     },
                 ],
             },
-            RegionSnapshot {
-                region_id: region_b,
+            ShardSnapshot {
+                shard_id: shard_b,
                 spec_id: 1,
                 current_generation: 2,
                 flushed_generations: vec![FlushedGeneration {
@@ -237,7 +237,7 @@ mod tests {
     #[test]
     fn test_collector_num_sources() {
         let snapshots = create_test_snapshots();
-        // 1 base table + 2 flushed from region_a + 1 flushed from region_b = 4
+        // 1 base table + 2 flushed from shard_a + 1 flushed from shard_b = 4
         // Using a mock dataset is complex, so we just test the counting logic
         assert_eq!(snapshots[0].flushed_generations.len(), 2);
         assert_eq!(snapshots[1].flushed_generations.len(), 1);
diff --git a/rust/lance/src/dataset/mem_wal/scanner/data_source.rs b/rust/lance/src/dataset/mem_wal/scanner/data_source.rs
index f94b5199dc5..1a6207f27e3 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/data_source.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/data_source.rs
@@ -79,15 +79,15 @@ pub struct FlushedGeneration {
     pub path: String,
 }
 
-/// Snapshot of a region's state at a point in time.
+/// Snapshot of a shard's state at a point in time.
 ///
 /// This is read from the MemWAL index for eventual consistency,
-/// or from region manifests directly for strong consistency.
+/// or from shard manifests directly for strong consistency.
 #[derive(Debug, Clone)]
-pub struct RegionSnapshot {
-    /// Region UUID.
-    pub region_id: Uuid,
-    /// Region spec ID (0 if manual region).
+pub struct ShardSnapshot {
+    /// Shard UUID.
+    pub shard_id: Uuid,
+    /// Shard spec ID (0 if manual shard).
     pub spec_id: u32,
     /// Current generation being written (next flush will be this generation).
     pub current_generation: u64,
@@ -95,11 +95,11 @@ pub struct RegionSnapshot {
     pub flushed_generations: Vec<FlushedGeneration>,
 }
 
-impl RegionSnapshot {
-    /// Create a new region snapshot.
-    pub fn new(region_id: Uuid) -> Self {
+impl ShardSnapshot {
+    /// Create a new shard snapshot.
+    pub fn new(shard_id: Uuid) -> Self {
         Self {
-            region_id,
+            shard_id,
             spec_id: 0,
             current_generation: 1,
             flushed_generations: Vec::new(),
@@ -137,8 +137,8 @@ pub enum LsmDataSource {
     FlushedMemTable {
         /// Absolute path to the flushed MemTable directory.
         path: String,
-        /// Region this MemTable belongs to.
-        region_id: Uuid,
+        /// Shard this MemTable belongs to.
+        shard_id: Uuid,
         /// Generation number (1, 2, 3, ...).
         generation: LsmGeneration,
     },
@@ -150,8 +150,8 @@ pub enum LsmDataSource {
         index_store: Arc<IndexStore>,
         /// Schema of the data.
         schema: SchemaRef,
-        /// Region this MemTable belongs to.
-        region_id: Uuid,
+        /// Shard this MemTable belongs to.
+        shard_id: Uuid,
         /// Generation number.
         generation: LsmGeneration,
     },
@@ -167,12 +167,12 @@ impl LsmDataSource {
         }
     }
 
-    /// Get the region ID if this is a regional source.
-    pub fn region_id(&self) -> Option<Uuid> {
+    /// Get the shard ID if this is a shard source.
+    pub fn shard_id(&self) -> Option<Uuid> {
         match self {
             Self::BaseTable { .. } => None,
-            Self::FlushedMemTable { region_id, .. } => Some(*region_id),
-            Self::ActiveMemTable { region_id, .. } => Some(*region_id),
+            Self::FlushedMemTable { shard_id, .. } => Some(*shard_id),
+            Self::ActiveMemTable { shard_id, .. } => Some(*shard_id),
         }
     }
 
@@ -191,15 +191,15 @@ impl LsmDataSource {
         match self {
             Self::BaseTable { .. } => "base_table".to_string(),
             Self::FlushedMemTable {
-                region_id,
+                shard_id,
                 generation,
                 ..
-            } => format!("flushed[{}:{}]", &region_id.to_string()[..8], generation),
+            } => format!("flushed[{}:{}]", &shard_id.to_string()[..8], generation),
             Self::ActiveMemTable {
-                region_id,
+                shard_id,
                 generation,
                 ..
-            } => format!("memtable[{}:{}]", &region_id.to_string()[..8], generation),
+            } => format!("memtable[{}:{}]", &shard_id.to_string()[..8], generation),
         }
     }
 }
@@ -251,15 +251,15 @@ mod tests {
     }
 
     #[test]
-    fn test_region_snapshot_builder() {
-        let region_id = Uuid::new_v4();
-        let snapshot = RegionSnapshot::new(region_id)
+    fn test_shard_snapshot_builder() {
+        let shard_id = Uuid::new_v4();
+        let snapshot = ShardSnapshot::new(shard_id)
             .with_spec_id(1)
             .with_current_generation(5)
             .with_flushed_generation(1, "abc123_gen_1".to_string())
             .with_flushed_generation(2, "def456_gen_2".to_string());
 
-        assert_eq!(snapshot.region_id, region_id);
+        assert_eq!(snapshot.shard_id, shard_id);
         assert_eq!(snapshot.spec_id, 1);
         assert_eq!(snapshot.current_generation, 5);
         assert_eq!(snapshot.flushed_generations.len(), 2);
diff --git a/rust/lance/src/dataset/mem_wal/scanner/planner.rs b/rust/lance/src/dataset/mem_wal/scanner/planner.rs
index f86fc6894d1..c70c127f747 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/planner.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/planner.rs
@@ -14,6 +14,7 @@ use datafusion::physical_plan::union::UnionExec;
 use datafusion::physical_plan::{ExecutionPlan, limit::GlobalLimitExec};
 use datafusion::prelude::Expr;
 use lance_core::Result;
+use tracing::instrument;
 
 use super::collector::LsmDataSourceCollector;
 use super::data_source::LsmDataSource;
@@ -75,6 +76,7 @@ impl LsmScanPlanner {
     /// - SortPreservingMergeExec is O(N log K) where K is the number of sources
     /// - Memory usage is bounded by the sum of K sort buffers rather than all data
     /// - No extra column for _memtable_gen in the common case
+    #[instrument(name = "lsm_plan_scan", level = "debug", skip_all, fields(has_filter = filter.is_some(), limit, offset))]
     pub async fn plan_scan(
         &self,
         projection: Option<&[String]>,
@@ -387,7 +389,7 @@ impl LsmScanPlanner {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::dataset::mem_wal::scanner::data_source::RegionSnapshot;
+    use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot;
 
     fn create_test_schema() -> SchemaRef {
         Arc::new(Schema::new(vec![
@@ -422,9 +424,9 @@ mod tests {
     }
 
     #[test]
-    fn test_region_snapshot() {
-        let region_id = uuid::Uuid::new_v4();
-        let snapshot = RegionSnapshot::new(region_id)
+    fn test_shard_snapshot() {
+        let shard_id = uuid::Uuid::new_v4();
+        let snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(5)
             .with_flushed_generation(1, "gen_1".to_string())
             .with_flushed_generation(2, "gen_2".to_string());
@@ -452,7 +454,7 @@ mod integration_tests {
 
     use crate::dataset::mem_wal::scanner::LsmScanner;
     use crate::dataset::mem_wal::scanner::collector::ActiveMemTableRef;
-    use crate::dataset::mem_wal::scanner::data_source::RegionSnapshot;
+    use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot;
     use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
     use crate::dataset::{Dataset, WriteParams};
     use crate::utils::test::assert_plan_node_equals;
@@ -513,7 +515,7 @@ mod integration_tests {
     /// - id=7: "active_7" (added in active)
     async fn setup_multi_level_lsm() -> (
         Arc<Dataset>,
-        Vec<RegionSnapshot>,
+        Vec<ShardSnapshot>,
         Option<(Uuid, ActiveMemTableRef)>,
         Vec<String>,
         String, // temp_dir path for cleanup
@@ -528,18 +530,18 @@ mod integration_tests {
         let base_dataset = Arc::new(create_dataset(&base_uri, vec![base_batch]).await);
 
         // Create flushed gen1 as a separate dataset
-        let region_id = Uuid::new_v4();
-        let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, region_id);
+        let shard_id = Uuid::new_v4();
+        let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
         let gen1_batch = create_test_batch(&schema, &[3, 4], "gen1");
         create_dataset(&gen1_uri, vec![gen1_batch]).await;
 
         // Create flushed gen2 as a separate dataset
-        let gen2_uri = format!("{}/_mem_wal/{}/gen_2", base_uri, region_id);
+        let gen2_uri = format!("{}/_mem_wal/{}/gen_2", base_uri, shard_id);
         let gen2_batch = create_test_batch(&schema, &[4, 5, 6], "gen2");
         create_dataset(&gen2_uri, vec![gen2_batch]).await;
 
-        // Build region snapshot
-        let region_snapshot = RegionSnapshot::new(region_id)
+        // Build shard snapshot
+        let shard_snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(3)
             .with_flushed_generation(1, "gen_1".to_string())
             .with_flushed_generation(2, "gen_2".to_string());
@@ -564,8 +566,8 @@ mod integration_tests {
 
         (
             base_dataset,
-            vec![region_snapshot],
-            Some((region_id, active_memtable)),
+            vec![shard_snapshot],
+            Some((shard_id, active_memtable)),
             pk_columns,
             temp_path,
         )
@@ -573,13 +575,13 @@ mod integration_tests {
 
     #[tokio::test]
     async fn test_lsm_scan_query_plan_without_memtable_gen() {
-        let (base_dataset, region_snapshots, active_memtable, pk_columns, _temp_path) =
+        let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) =
             setup_multi_level_lsm().await;
 
         // Create scanner without requesting _memtable_gen
-        let mut scanner = LsmScanner::new(base_dataset, region_snapshots, pk_columns);
-        if let Some((region_id, memtable)) = active_memtable {
-            scanner = scanner.with_active_memtable(region_id, memtable);
+        let mut scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns);
+        if let Some((shard_id, memtable)) = active_memtable {
+            scanner = scanner.with_active_memtable(shard_id, memtable);
         }
 
         let plan = scanner.create_plan().await.unwrap();
@@ -609,14 +611,14 @@ mod integration_tests {
 
     #[tokio::test]
     async fn test_lsm_scan_query_plan_with_memtable_gen() {
-        let (base_dataset, region_snapshots, active_memtable, pk_columns, _temp_path) =
+        let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) =
             setup_multi_level_lsm().await;
 
         // Create scanner requesting _memtable_gen
         let mut scanner =
-            LsmScanner::new(base_dataset, region_snapshots, pk_columns).with_memtable_gen();
-        if let Some((region_id, memtable)) = active_memtable {
-            scanner = scanner.with_active_memtable(region_id, memtable);
+            LsmScanner::new(base_dataset, shard_snapshots, pk_columns).with_memtable_gen();
+        if let Some((shard_id, memtable)) = active_memtable {
+            scanner = scanner.with_active_memtable(shard_id, memtable);
         }
 
         let plan = scanner.create_plan().await.unwrap();
@@ -654,13 +656,13 @@ mod integration_tests {
 
     #[tokio::test]
     async fn test_lsm_scan_deduplication_results() {
-        let (base_dataset, region_snapshots, active_memtable, pk_columns, _temp_path) =
+        let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) =
             setup_multi_level_lsm().await;
 
         // Create scanner
-        let mut scanner = LsmScanner::new(base_dataset, region_snapshots, pk_columns);
-        if let Some((region_id, memtable)) = active_memtable {
-            scanner = scanner.with_active_memtable(region_id, memtable);
+        let mut scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns);
+        if let Some((shard_id, memtable)) = active_memtable {
+            scanner = scanner.with_active_memtable(shard_id, memtable);
         }
 
         // Execute and collect results
@@ -714,14 +716,14 @@ mod integration_tests {
 
     #[tokio::test]
     async fn test_lsm_scan_with_projection() {
-        let (base_dataset, region_snapshots, active_memtable, pk_columns, _temp_path) =
+        let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) =
             setup_multi_level_lsm().await;
 
         // Create scanner with projection (only id column)
         let mut scanner =
-            LsmScanner::new(base_dataset, region_snapshots, pk_columns).project(&["id"]);
-        if let Some((region_id, memtable)) = active_memtable {
-            scanner = scanner.with_active_memtable(region_id, memtable);
+            LsmScanner::new(base_dataset, shard_snapshots, pk_columns).project(&["id"]);
+        if let Some((shard_id, memtable)) = active_memtable {
+            scanner = scanner.with_active_memtable(shard_id, memtable);
         }
 
         // Execute and collect results
@@ -745,14 +747,13 @@ mod integration_tests {
 
     #[tokio::test]
     async fn test_lsm_scan_with_limit() {
-        let (base_dataset, region_snapshots, active_memtable, pk_columns, _temp_path) =
+        let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) =
             setup_multi_level_lsm().await;
 
         // Create scanner with limit
-        let mut scanner =
-            LsmScanner::new(base_dataset, region_snapshots, pk_columns).limit(3, None);
-        if let Some((region_id, memtable)) = active_memtable {
-            scanner = scanner.with_active_memtable(region_id, memtable);
+        let mut scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns).limit(3, None);
+        if let Some((shard_id, memtable)) = active_memtable {
+            scanner = scanner.with_active_memtable(shard_id, memtable);
         }
 
         // Execute and collect results
@@ -773,7 +774,7 @@ mod integration_tests {
     async fn test_lsm_scan_base_only() {
         let (base_dataset, _, _, pk_columns, _temp_path) = setup_multi_level_lsm().await;
 
-        // Create scanner with only base table (no region snapshots or active memtable)
+        // Create scanner with only base table (no shard snapshots or active memtable)
         let scanner = LsmScanner::new(base_dataset, vec![], pk_columns);
 
         let plan = scanner.create_plan().await.unwrap();
@@ -816,11 +817,11 @@ mod integration_tests {
 
     #[tokio::test]
     async fn test_lsm_scan_flushed_only_no_active() {
-        let (base_dataset, region_snapshots, _, pk_columns, _temp_path) =
+        let (base_dataset, shard_snapshots, _, pk_columns, _temp_path) =
             setup_multi_level_lsm().await;
 
         // Create scanner with base + flushed (no active memtable)
-        let scanner = LsmScanner::new(base_dataset, region_snapshots, pk_columns);
+        let scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns);
 
         // Execute and collect results
         let batches: Vec<RecordBatch> = scanner
@@ -867,14 +868,14 @@ mod integration_tests {
 
     #[tokio::test]
     async fn test_lsm_scan_with_row_address() {
-        let (base_dataset, region_snapshots, active_memtable, pk_columns, _temp_path) =
+        let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) =
             setup_multi_level_lsm().await;
 
         // Create scanner requesting _rowaddr
         let mut scanner =
-            LsmScanner::new(base_dataset, region_snapshots, pk_columns).with_row_address();
-        if let Some((region_id, memtable)) = active_memtable {
-            scanner = scanner.with_active_memtable(region_id, memtable);
+            LsmScanner::new(base_dataset, shard_snapshots, pk_columns).with_row_address();
+        if let Some((shard_id, memtable)) = active_memtable {
+            scanner = scanner.with_active_memtable(shard_id, memtable);
         }
 
         let plan = scanner.create_plan().await.unwrap();
@@ -927,15 +928,15 @@ mod integration_tests {
 
     #[tokio::test]
     async fn test_lsm_scan_with_both_memtable_gen_and_row_address() {
-        let (base_dataset, region_snapshots, active_memtable, pk_columns, _temp_path) =
+        let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) =
             setup_multi_level_lsm().await;
 
         // Create scanner requesting both _memtable_gen and _rowaddr
-        let mut scanner = LsmScanner::new(base_dataset, region_snapshots, pk_columns)
+        let mut scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns)
             .with_memtable_gen()
             .with_row_address();
-        if let Some((region_id, memtable)) = active_memtable {
-            scanner = scanner.with_active_memtable(region_id, memtable);
+        if let Some((shard_id, memtable)) = active_memtable {
+            scanner = scanner.with_active_memtable(shard_id, memtable);
         }
 
         let plan = scanner.create_plan().await.unwrap();
@@ -971,7 +972,7 @@ mod integration_tests {
     /// - Flushed datasets have BTree index created (enabling ScalarIndexQuery)
     async fn setup_multi_level_lsm_with_btree_index() -> (
         Arc<Dataset>,
-        Vec<RegionSnapshot>,
+        Vec<ShardSnapshot>,
         Option<(Uuid, ActiveMemTableRef)>,
         Vec<String>,
         String,
@@ -999,8 +1000,8 @@ mod integration_tests {
         let base_dataset = Arc::new(Dataset::open(&base_uri).await.unwrap());
 
         // Create flushed gen1 with BTree index
-        let region_id = Uuid::new_v4();
-        let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, region_id);
+        let shard_id = Uuid::new_v4();
+        let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
         let gen1_batch = create_test_batch(&schema, &[3, 4], "gen1");
         let mut gen1_dataset = create_dataset(&gen1_uri, vec![gen1_batch]).await;
         CreateIndexBuilder::new(&mut gen1_dataset, &["id"], IndexType::BTree, &params)
@@ -1008,15 +1009,15 @@ mod integration_tests {
             .unwrap();
 
         // Create flushed gen2 with BTree index
-        let gen2_uri = format!("{}/_mem_wal/{}/gen_2", base_uri, region_id);
+        let gen2_uri = format!("{}/_mem_wal/{}/gen_2", base_uri, shard_id);
         let gen2_batch = create_test_batch(&schema, &[4, 5, 6], "gen2");
         let mut gen2_dataset = create_dataset(&gen2_uri, vec![gen2_batch]).await;
         CreateIndexBuilder::new(&mut gen2_dataset, &["id"], IndexType::BTree, &params)
             .await
             .unwrap();
 
-        // Build region snapshot
-        let region_snapshot = RegionSnapshot::new(region_id)
+        // Build shard snapshot
+        let shard_snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(3)
             .with_flushed_generation(1, "gen_1".to_string())
             .with_flushed_generation(2, "gen_2".to_string());
@@ -1049,8 +1050,8 @@ mod integration_tests {
 
         (
             base_dataset,
-            vec![region_snapshot],
-            Some((region_id, active_memtable)),
+            vec![shard_snapshot],
+            Some((shard_id, active_memtable)),
             pk_columns,
             temp_path,
         )
@@ -1058,15 +1059,15 @@ mod integration_tests {
 
     #[tokio::test]
     async fn test_lsm_scan_with_btree_index_filter() {
-        let (base_dataset, region_snapshots, active_memtable, pk_columns, _temp_path) =
+        let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) =
             setup_multi_level_lsm_with_btree_index().await;
 
         // Create scanner with filter on the indexed column
-        let mut scanner = LsmScanner::new(base_dataset, region_snapshots, pk_columns)
+        let mut scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns)
             .filter("id = 5")
             .unwrap();
-        if let Some((region_id, memtable)) = active_memtable {
-            scanner = scanner.with_active_memtable(region_id, memtable);
+        if let Some((shard_id, memtable)) = active_memtable {
+            scanner = scanner.with_active_memtable(shard_id, memtable);
         }
 
         let plan = scanner.create_plan().await.unwrap();
@@ -1149,16 +1150,16 @@ mod integration_tests {
     #[tokio::test]
     async fn test_lsm_scan_with_filter_no_index() {
         // Test that filter still works correctly even without BTree index
-        let (base_dataset, region_snapshots, active_memtable, pk_columns, _temp_path) =
+        let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) =
             setup_multi_level_lsm().await;
 
         // Create scanner with SQL filter
         // This tests that type coercion works correctly (Int64 literal -> Int32 column)
-        let mut scanner = LsmScanner::new(base_dataset, region_snapshots, pk_columns)
+        let mut scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns)
             .filter("id = 3")
             .unwrap();
-        if let Some((region_id, memtable)) = active_memtable {
-            scanner = scanner.with_active_memtable(region_id, memtable);
+        if let Some((shard_id, memtable)) = active_memtable {
+            scanner = scanner.with_active_memtable(shard_id, memtable);
         }
 
         // Execute and verify result
diff --git a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
index 3cef6a34dd4..a11ba083139 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
@@ -14,6 +14,7 @@ use datafusion::physical_plan::limit::GlobalLimitExec;
 use datafusion::prelude::Expr;
 use lance_core::Result;
 use lance_index::scalar::bloomfilter::sbbf::Sbbf;
+use tracing::instrument;
 
 use super::collector::LsmDataSourceCollector;
 use super::data_source::LsmDataSource;
@@ -117,6 +118,7 @@ impl LsmPointLookupPlanner {
     ///
     /// An execution plan that returns at most one row - the newest version
     /// of the row with the given primary key.
+    #[instrument(name = "lsm_point_lookup", level = "debug", skip_all, fields(pk_column_count = self.pk_columns.len()))]
     pub async fn plan_lookup(
         &self,
         pk_values: &[ScalarValue],
@@ -293,7 +295,7 @@ mod tests {
     use std::collections::HashMap;
     use uuid::Uuid;
 
-    use crate::dataset::mem_wal::scanner::data_source::RegionSnapshot;
+    use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot;
     use crate::dataset::{Dataset, WriteParams};
 
     fn create_pk_schema() -> Arc<ArrowSchema> {
@@ -374,18 +376,18 @@ mod tests {
         let base_batch = create_test_batch(&schema, &[1, 2, 3], "base");
         let base_dataset = Arc::new(create_dataset(&base_uri, vec![base_batch]).await);
 
-        // Create region snapshot
-        let region_id = Uuid::new_v4();
-        let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, region_id);
+        // Create shard snapshot
+        let shard_id = Uuid::new_v4();
+        let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
         let gen1_batch = create_test_batch(&schema, &[2], "gen1"); // Update id=2
         create_dataset(&gen1_uri, vec![gen1_batch]).await;
 
-        let region_snapshot = RegionSnapshot::new(region_id)
+        let shard_snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(2)
             .with_flushed_generation(1, "gen_1".to_string());
 
         // Create collector
-        let collector = LsmDataSourceCollector::new(base_dataset, vec![region_snapshot]);
+        let collector = LsmDataSourceCollector::new(base_dataset, vec![shard_snapshot]);
 
         let planner = LsmPointLookupPlanner::new(collector, vec!["id".to_string()], schema.clone());
 
diff --git a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs
index 0433929501f..3f9880250c7 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs
@@ -18,6 +18,7 @@ use datafusion::physical_plan::sorts::sort::SortExec;
 use datafusion::physical_plan::union::UnionExec;
 use lance_core::Result;
 use lance_index::scalar::bloomfilter::sbbf::Sbbf;
+use tracing::instrument;
 
 use super::collector::LsmDataSourceCollector;
 use super::data_source::LsmDataSource;
@@ -144,6 +145,7 @@ impl LsmVectorSearchPlanner {
     ///
     /// An execution plan that returns the top-K nearest neighbors across all
     /// LSM levels, with stale results filtered out.
+    #[instrument(name = "lsm_vector_search", level = "info", skip_all, fields(k, nprobes, vector_column = %self.vector_column, distance_type = ?self.distance_type))]
     pub async fn plan_search(
         &self,
         query_vector: &FixedSizeListArray,
diff --git a/rust/lance/src/dataset/mem_wal/util.rs b/rust/lance/src/dataset/mem_wal/util.rs
index 1f8eed7bf1c..12bf6f299f5 100644
--- a/rust/lance/src/dataset/mem_wal/util.rs
+++ b/rust/lance/src/dataset/mem_wal/util.rs
@@ -126,39 +126,39 @@ pub fn parse_bit_reversed_filename(filename: &str) -> Option<u64> {
     Some(bit_reverse_u64(reversed))
 }
 
-/// Base path for a region within the MemWAL directory.
+/// Base path for a shard within the MemWAL directory.
 ///
-/// Returns: `{base_path}/_mem_wal/{region_id}/`
-pub fn region_base_path(base_path: &Path, region_id: &Uuid) -> Path {
+/// Returns: `{base_path}/_mem_wal/{shard_id}/`
+pub fn shard_base_path(base_path: &Path, shard_id: &Uuid) -> Path {
     base_path
         .child("_mem_wal")
-        .child(region_id.as_hyphenated().to_string())
+        .child(shard_id.as_hyphenated().to_string())
 }
 
-/// Path to the WAL directory for a region.
+/// Path to the WAL directory for a shard.
 ///
-/// Returns: `{base_path}/_mem_wal/{region_id}/wal/`
-pub fn region_wal_path(base_path: &Path, region_id: &Uuid) -> Path {
-    region_base_path(base_path, region_id).child("wal")
+/// Returns: `{base_path}/_mem_wal/{shard_id}/wal/`
+pub fn shard_wal_path(base_path: &Path, shard_id: &Uuid) -> Path {
+    shard_base_path(base_path, shard_id).child("wal")
 }
 
-/// Path to the manifest directory for a region.
+/// Path to the manifest directory for a shard.
 ///
-/// Returns: `{base_path}/_mem_wal/{region_id}/manifest/`
-pub fn region_manifest_path(base_path: &Path, region_id: &Uuid) -> Path {
-    region_base_path(base_path, region_id).child("manifest")
+/// Returns: `{base_path}/_mem_wal/{shard_id}/manifest/`
+pub fn shard_manifest_path(base_path: &Path, shard_id: &Uuid) -> Path {
+    shard_base_path(base_path, shard_id).child("manifest")
 }
 
 /// Path to a flushed MemTable directory.
 ///
-/// Returns: `{base_path}/_mem_wal/{region_id}/{random_hash}_gen_{generation}/`
+/// Returns: `{base_path}/_mem_wal/{shard_id}/{random_hash}_gen_{generation}/`
 pub fn flushed_memtable_path(
     base_path: &Path,
-    region_id: &Uuid,
+    shard_id: &Uuid,
     random_hash: &str,
     generation: u64,
 ) -> Path {
-    region_base_path(base_path, region_id).child(format!("{}_gen_{}", random_hash, generation))
+    shard_base_path(base_path, shard_id).child(format!("{}_gen_{}", random_hash, generation))
 }
 
 /// Generate an 8-character random hex string for flushed MemTable directories.
@@ -177,7 +177,7 @@ pub fn wal_entry_filename(wal_entry_position: u64) -> String {
     bit_reversed_filename(wal_entry_position, "arrow")
 }
 
-/// Region manifest filename.
+/// Shard manifest filename.
 ///
 /// Returns bit-reversed filename with .binpb extension.
 pub fn manifest_filename(version: u64) -> String {
@@ -241,34 +241,34 @@ mod tests {
     }
 
     #[test]
-    fn test_region_paths() {
+    fn test_shard_paths() {
         let base_path = Path::from("my/dataset");
-        let region_id = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
+        let shard_id = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
 
         assert_eq!(
-            region_base_path(&base_path, &region_id).as_ref(),
+            shard_base_path(&base_path, &shard_id).as_ref(),
             "my/dataset/_mem_wal/550e8400-e29b-41d4-a716-446655440000"
         );
 
         assert_eq!(
-            region_wal_path(&base_path, &region_id).as_ref(),
+            shard_wal_path(&base_path, &shard_id).as_ref(),
             "my/dataset/_mem_wal/550e8400-e29b-41d4-a716-446655440000/wal"
         );
 
         assert_eq!(
-            region_manifest_path(&base_path, &region_id).as_ref(),
+            shard_manifest_path(&base_path, &shard_id).as_ref(),
             "my/dataset/_mem_wal/550e8400-e29b-41d4-a716-446655440000/manifest"
         );
 
         assert_eq!(
-            flushed_memtable_path(&base_path, &region_id, "a1b2c3d4", 5).as_ref(),
+            flushed_memtable_path(&base_path, &shard_id, "a1b2c3d4", 5).as_ref(),
             "my/dataset/_mem_wal/550e8400-e29b-41d4-a716-446655440000/a1b2c3d4_gen_5"
         );
 
         // Test with empty base path
         let empty_base = Path::from("");
         assert_eq!(
-            region_wal_path(&empty_base, &region_id).as_ref(),
+            shard_wal_path(&empty_base, &shard_id).as_ref(),
             "_mem_wal/550e8400-e29b-41d4-a716-446655440000/wal"
         );
     }
diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs
index a0b7a4c912a..8dd1d9d6e32 100644
--- a/rust/lance/src/dataset/mem_wal/wal.rs
+++ b/rust/lance/src/dataset/mem_wal/wal.rs
@@ -21,9 +21,10 @@ use lance_io::object_store::ObjectStore;
 use object_store::path::Path;
 use tokio::sync::{mpsc, watch};
 
+use tracing::instrument;
 use uuid::Uuid;
 
-use super::util::{WatchableOnceCell, region_wal_path, wal_entry_filename};
+use super::util::{WatchableOnceCell, shard_wal_path, wal_entry_filename};
 
 use super::index::IndexStore;
 use super::memtable::batch_store::{BatchStore, StoredBatch};
@@ -154,8 +155,8 @@ pub struct WalFlusher {
     durable_watermark_rx: watch::Receiver<usize>,
     /// Object store for writing WAL files.
     object_store: Option<Arc<ObjectStore>>,
-    /// Region ID.
-    region_id: Uuid,
+    /// Shard ID.
+    shard_id: Uuid,
     /// Writer epoch (stored in WAL entries for fencing).
     writer_epoch: u64,
     /// Next WAL entry ID to use.
@@ -176,16 +177,16 @@ impl WalFlusher {
     /// # Arguments
     ///
     /// * `base_path` - Base path within the object store (from ObjectStore::from_uri)
-    /// * `region_id` - Region UUID
+    /// * `shard_id` - Shard UUID
     /// * `writer_epoch` - Current writer epoch
-    /// * `next_wal_entry_position` - Next WAL entry ID (from recovery or 1 for new region)
+    /// * `next_wal_entry_position` - Next WAL entry ID (from recovery or 1 for new shard)
     pub fn new(
         base_path: &Path,
-        region_id: Uuid,
+        shard_id: Uuid,
         writer_epoch: u64,
         next_wal_entry_position: u64,
     ) -> Self {
-        let wal_dir = region_wal_path(base_path, &region_id);
+        let wal_dir = shard_wal_path(base_path, &shard_id);
         // Initialize durable watermark at 0 (no batches durable yet)
         let (durable_watermark_tx, durable_watermark_rx) = watch::channel(0);
         // Create initial WAL flush cell for backpressure
@@ -194,7 +195,7 @@ impl WalFlusher {
             durable_watermark_tx,
             durable_watermark_rx,
             object_store: None,
-            region_id,
+            shard_id,
             writer_epoch,
             next_wal_entry_position: AtomicU64::new(next_wal_entry_position),
             flush_tx: None,
@@ -297,6 +298,7 @@ impl WalFlusher {
     ///
     /// A `WalFlushResult` with timing metrics and the WAL entry.
     /// Returns empty result if nothing to flush (already flushed past end_batch_position).
+    #[instrument(name = "wal_flush", level = "info", skip_all, fields(shard_id = %self.shard_id, end_batch_position, has_indexes = indexes.is_some()))]
     pub async fn flush_to_with_index_update(
         &self,
         batch_store: &BatchStore,
@@ -461,9 +463,9 @@ impl WalFlusher {
         self.next_wal_entry_position.load(Ordering::SeqCst)
     }
 
-    /// Get the region ID.
-    pub fn region_id(&self) -> Uuid {
-        self.region_id
+    /// Get the shard ID.
+    pub fn shard_id(&self) -> Uuid {
+        self.shard_id
     }
 
     /// Get the writer epoch.
@@ -498,6 +500,7 @@ impl WalEntryData {
     /// # Returns
     ///
     /// The parsed WAL entry data, or an error if reading/parsing fails.
+    #[instrument(name = "wal_entry_read", level = "debug", skip_all, fields(path = %path))]
     pub async fn read(object_store: &ObjectStore, path: &Path) -> Result<Self> {
         // Read the file
         let data = object_store
@@ -576,8 +579,8 @@ mod tests {
     #[tokio::test]
     async fn test_wal_flusher_track_batch() {
         let (store, base_path, _temp_dir) = create_local_store().await;
-        let region_id = Uuid::new_v4();
-        let mut buffer = WalFlusher::new(&base_path, region_id, 1, 1);
+        let shard_id = Uuid::new_v4();
+        let mut buffer = WalFlusher::new(&base_path, shard_id, 1, 1);
         buffer.set_object_store(store);
 
         // Track a batch
@@ -587,11 +590,44 @@ mod tests {
         assert!(!watcher.is_durable());
     }
 
+    // Regression test: track_batch must return a watcher wired to the real
+    // WAL watermark, NOT a pre-resolved watcher. A pre-resolved watcher would
+    // cause durable writes to return before the WAL is actually flushed.
     #[tokio::test]
-    async fn test_wal_flusher_flush_to_with_index_update() {
+    async fn test_track_batch_watcher_blocks_until_flush() {
         let (store, base_path, _temp_dir) = create_local_store().await;
         let region_id = Uuid::new_v4();
-        let mut buffer = WalFlusher::new(&base_path, region_id, 1, 1);
+        let mut flusher = WalFlusher::new(&base_path, region_id, 1, 1);
+        flusher.set_object_store(store);
+
+        let schema = create_test_schema();
+        let batch_store = BatchStore::with_capacity(10);
+        batch_store.append(create_test_batch(&schema, 10)).unwrap();
+
+        let mut watcher = flusher.track_batch(0);
+
+        // wait() must NOT resolve before the flush happens
+        let result =
+            tokio::time::timeout(std::time::Duration::from_millis(100), watcher.wait()).await;
+        assert!(
+            result.is_err(),
+            "watcher resolved before WAL flush — durability guarantee broken"
+        );
+
+        // Flush, then the watcher should resolve
+        flusher
+            .flush_to_with_index_update(&batch_store, batch_store.len(), None)
+            .await
+            .unwrap();
+        watcher.wait().await.unwrap();
+        assert!(watcher.is_durable());
+    }
+
+    #[tokio::test]
+    async fn test_wal_flusher_flush_to_with_index_update() {
+        let (store, base_path, _temp_dir) = create_local_store().await;
+        let shard_id = Uuid::new_v4();
+        let mut buffer = WalFlusher::new(&base_path, shard_id, 1, 1);
         buffer.set_object_store(store);
 
         // Create a BatchStore with some data
@@ -634,8 +670,8 @@ mod tests {
     #[tokio::test]
     async fn test_wal_entry_read() {
         let (store, base_path, _temp_dir) = create_local_store().await;
-        let region_id = Uuid::new_v4();
-        let mut buffer = WalFlusher::new(&base_path, region_id, 42, 1);
+        let shard_id = Uuid::new_v4();
+        let mut buffer = WalFlusher::new(&base_path, shard_id, 42, 1);
         buffer.set_object_store(store.clone());
 
         // Create a BatchStore with some data
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 65aa1d97586..406e521e10d 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -6,7 +6,7 @@
 //! Write path for MemWAL.
 //!
 //! This module contains all components for the write path:
-//! - [`RegionWriter`] - Main writer interface for a single region
+//! - [`ShardWriter`] - Main writer interface for a single shard
 //! - [`MemTable`] - In-memory table storing Arrow RecordBatches
 //! - [`WalFlusher`] - Write-ahead log buffer for durability (Arrow IPC format)
 //! - [`IndexStore`] - In-memory index management
@@ -23,7 +23,7 @@ use arrow_schema::Schema as ArrowSchema;
 use async_trait::async_trait;
 use lance_core::datatypes::Schema;
 use lance_core::{Error, Result};
-use lance_index::mem_wal::RegionManifest;
+use lance_index::mem_wal::ShardManifest;
 use lance_io::object_store::ObjectStore;
 use log::{debug, error, info, warn};
 use object_store::path::Path;
@@ -31,6 +31,7 @@ use tokio::sync::{RwLock, mpsc};
 use tokio::task::JoinHandle;
 use tokio::time::{Interval, interval_at};
 use tokio_util::sync::CancellationToken;
+use tracing::instrument;
 use uuid::Uuid;
 
 pub use super::index::{
@@ -47,21 +48,21 @@ pub use super::wal::{WalEntry, WalEntryData, WalFlushResult, WalFlusher};
 use super::memtable::flush::TriggerMemTableFlush;
 use super::wal::TriggerWalFlush;
 
-use super::manifest::RegionManifestStore;
+use super::manifest::ShardManifestStore;
 
 // ============================================================================
 // Configuration
 // ============================================================================
 
-/// Configuration for a region writer.
+/// Configuration for a shard writer.
 #[derive(Debug, Clone)]
-pub struct RegionWriterConfig {
-    /// Unique identifier for this region (UUID v4).
-    pub region_id: Uuid,
+pub struct ShardWriterConfig {
+    /// Unique identifier for this shard (UUID v4).
+    pub shard_id: Uuid,
 
-    /// Region spec ID this region was created with.
-    /// A value of 0 indicates a manually-created region not governed by any spec.
-    pub region_spec_id: u32,
+    /// Shard spec ID this shard was created with.
+    /// A value of 0 indicates a manually-created shard not governed by any spec.
+    pub shard_spec_id: u32,
 
     /// Whether to wait for WAL flush before returning from writes.
     ///
@@ -182,11 +183,11 @@ pub struct RegionWriterConfig {
     pub stats_log_interval: Option<Duration>,
 }
 
-impl Default for RegionWriterConfig {
+impl Default for ShardWriterConfig {
     fn default() -> Self {
         Self {
-            region_id: Uuid::new_v4(),
-            region_spec_id: 0,
+            shard_id: Uuid::new_v4(),
+            shard_spec_id: 0,
             durable_write: true,
             sync_indexed_write: true,
             max_wal_buffer_size: 10 * 1024 * 1024, // 10MB
@@ -205,18 +206,18 @@ impl Default for RegionWriterConfig {
     }
 }
 
-impl RegionWriterConfig {
-    /// Create a new configuration with the given region ID.
-    pub fn new(region_id: Uuid) -> Self {
+impl ShardWriterConfig {
+    /// Create a new configuration with the given shard ID.
+    pub fn new(shard_id: Uuid) -> Self {
         Self {
-            region_id,
+            shard_id,
             ..Default::default()
         }
     }
 
-    /// Set the region spec ID.
-    pub fn with_region_spec_id(mut self, spec_id: u32) -> Self {
-        self.region_spec_id = spec_id;
+    /// Set the shard spec ID.
+    pub fn with_shard_spec_id(mut self, spec_id: u32) -> Self {
+        self.shard_spec_id = spec_id;
         self
     }
 
@@ -567,14 +568,14 @@ pub struct BackpressureStatsSnapshot {
 /// Backpressure controller for managing write flow.
 pub struct BackpressureController {
     /// Configuration.
-    config: RegionWriterConfig,
+    config: ShardWriterConfig,
     /// Stats for monitoring.
     stats: Arc<BackpressureStats>,
 }
 
 impl BackpressureController {
     /// Create a new backpressure controller.
-    pub fn new(config: RegionWriterConfig) -> Self {
+    pub fn new(config: ShardWriterConfig) -> Self {
         Self {
             config,
             stats: Arc::new(BackpressureStats::new()),
@@ -653,7 +654,7 @@ pub struct WriteResult {
     pub batch_positions: std::ops::Range<usize>,
 }
 
-/// RegionWriter state shared across tasks.
+/// ShardWriter state shared across tasks.
 struct WriterState {
     memtable: MemTable,
     last_flushed_wal_entry_position: u64,
@@ -685,7 +686,7 @@ struct SharedWriterState {
     wal_flusher: Arc<WalFlusher>,
     wal_flush_tx: mpsc::UnboundedSender<TriggerWalFlush>,
     memtable_flush_tx: mpsc::UnboundedSender<TriggerMemTableFlush>,
-    config: RegionWriterConfig,
+    config: ShardWriterConfig,
     schema: Arc<ArrowSchema>,
     pk_field_ids: Vec<i32>,
     max_memtable_batches: usize,
@@ -701,7 +702,7 @@ impl SharedWriterState {
         wal_flusher: Arc<WalFlusher>,
         wal_flush_tx: mpsc::UnboundedSender<TriggerWalFlush>,
         memtable_flush_tx: mpsc::UnboundedSender<TriggerMemTableFlush>,
-        config: RegionWriterConfig,
+        config: ShardWriterConfig,
         schema: Arc<ArrowSchema>,
         pk_field_ids: Vec<i32>,
         max_memtable_batches: usize,
@@ -799,12 +800,8 @@ impl SharedWriterState {
     }
 
     /// Track batch for WAL durability.
-    fn track_batch_for_wal(&self, batch_position: usize) -> DurabilityWatcher {
-        let _wal_watcher = self.wal_flusher.track_batch(batch_position);
-        // Return pre-resolved watcher for non-durable case
-        let cell: WatchableOnceCell<DurabilityResult> = WatchableOnceCell::new();
-        cell.write(DurabilityResult::ok());
-        cell.reader()
+    fn track_batch_for_wal(&self, batch_position: usize) -> super::wal::BatchDurableWatcher {
+        self.wal_flusher.track_batch(batch_position)
     }
 
     /// Check if memtable flush is needed and trigger if so.
@@ -928,47 +925,48 @@ impl SharedWriterState {
     }
 }
 
-/// Main writer for a MemWAL region.
-pub struct RegionWriter {
-    config: RegionWriterConfig,
+/// Main writer for a MemWAL shard.
+pub struct ShardWriter {
+    config: ShardWriterConfig,
     epoch: u64,
     state: Arc<RwLock<WriterState>>,
     wal_flusher: Arc<WalFlusher>,
     task_executor: Arc<TaskExecutor>,
-    manifest_store: Arc<RegionManifestStore>,
+    manifest_store: Arc<ShardManifestStore>,
     stats: SharedWriteStats,
     writer_state: Arc<SharedWriterState>,
     backpressure: BackpressureController,
 }
 
-impl RegionWriter {
-    /// Open or create a RegionWriter.
+impl ShardWriter {
+    /// Open or create a ShardWriter.
     ///
     /// The `base_path` should come from `ObjectStore::from_uri()` to ensure
     /// WAL files are written inside the dataset directory.
+    #[instrument(name = "sw_open", level = "info", skip_all, fields(shard_id = %config.shard_id, index_count = index_configs.len()))]
     pub async fn open(
         object_store: Arc<ObjectStore>,
         base_path: Path,
         base_uri: impl Into<String>,
-        config: RegionWriterConfig,
+        config: ShardWriterConfig,
         schema: Arc<ArrowSchema>,
         index_configs: Vec<MemIndexConfig>,
     ) -> Result<Self> {
         let base_uri = base_uri.into();
-        let region_id = config.region_id;
-        let manifest_store = Arc::new(RegionManifestStore::new(
+        let shard_id = config.shard_id;
+        let manifest_store = Arc::new(ShardManifestStore::new(
             object_store.clone(),
             &base_path,
-            region_id,
+            shard_id,
             config.manifest_scan_batch_size,
         ));
 
-        // Claim the region (epoch-based fencing)
-        let (epoch, manifest) = manifest_store.claim_epoch(config.region_spec_id).await?;
+        // Claim the shard (epoch-based fencing)
+        let (epoch, manifest) = manifest_store.claim_epoch(config.shard_spec_id).await?;
 
         info!(
-            "Opened RegionWriter for region {} (epoch {}, generation {})",
-            region_id, epoch, manifest.current_generation
+            "Opened ShardWriter for shard {} (epoch {}, generation {})",
+            shard_id, epoch, manifest.current_generation
         );
 
         // Create MemTable with primary key field IDs from schema
@@ -1011,7 +1009,7 @@ impl RegionWriter {
         // Create WAL flusher
         let mut wal_flusher = WalFlusher::new(
             &base_path,
-            region_id,
+            shard_id,
             epoch,
             manifest.wal_entry_position_last_seen + 1,
         );
@@ -1029,7 +1027,7 @@ impl RegionWriter {
             object_store.clone(),
             base_path,
             base_uri,
-            region_id,
+            shard_id,
             manifest_store.clone(),
         ));
 
@@ -1087,7 +1085,7 @@ impl RegionWriter {
         })
     }
 
-    /// Write record batches to the region.
+    /// Write record batches to the shard.
     ///
     /// All batches are inserted atomically with a single lock acquisition.
     /// This is more efficient than calling put() multiple times for Arrow IPC
@@ -1106,6 +1104,7 @@ impl RegionWriter {
     /// Fencing is detected lazily during WAL flush via atomic writes.
     /// If another writer has taken over, the WAL flush will fail with
     /// `AlreadyExists`, indicating this writer has been fenced.
+    #[instrument(name = "sw_put", level = "info", skip_all, fields(batch_count = batches.len(), shard_id = %self.config.shard_id))]
     pub async fn put(&self, batches: Vec<RecordBatch>) -> Result<WriteResult> {
         if batches.is_empty() {
             return Err(Error::invalid_input("Cannot write empty batch list"));
@@ -1132,7 +1131,7 @@ impl RegionWriter {
         let start = std::time::Instant::now();
 
         // Acquire write lock for entire operation (atomic approach)
-        let (batch_positions, durable_watcher, batch_store, indexes) = {
+        let (batch_positions, mut durable_watcher, batch_store, indexes) = {
             let mut state = self.state.write().await;
 
             // 1. Insert all batches into memtable atomically
@@ -1170,7 +1169,7 @@ impl RegionWriter {
             // Must trigger a flush to ensure durability (flush up to and including all batches)
             self.wal_flusher
                 .trigger_flush(batch_store, indexes, batch_positions.end, None)?;
-            durable_watcher.clone().await_value().await.into_result()?;
+            durable_watcher.wait().await?;
         }
 
         Ok(WriteResult { batch_positions })
@@ -1186,8 +1185,8 @@ impl RegionWriter {
         self.stats.clone()
     }
 
-    /// Get the current region manifest.
-    pub async fn manifest(&self) -> Result<Option<RegionManifest>> {
+    /// Get the current shard manifest.
+    pub async fn manifest(&self) -> Result<Option<ShardManifest>> {
         self.manifest_store.read_latest().await
     }
 
@@ -1196,9 +1195,9 @@ impl RegionWriter {
         self.epoch
     }
 
-    /// Get the region ID.
-    pub fn region_id(&self) -> Uuid {
-        self.config.region_id
+    /// Get the shard ID.
+    pub fn shard_id(&self) -> Uuid {
+        self.config.shard_id
     }
 
     /// Get current MemTable statistics.
@@ -1261,8 +1260,9 @@ impl RegionWriter {
     /// Close the writer gracefully.
     ///
     /// Flushes pending data and shuts down background tasks.
+    #[instrument(name = "sw_close", level = "info", skip_all, fields(shard_id = %self.config.shard_id, epoch = self.epoch))]
     pub async fn close(self) -> Result<()> {
-        info!("Closing RegionWriter for region {}", self.config.region_id);
+        info!("Closing ShardWriter for shard {}", self.config.shard_id);
 
         // Send final WAL flush message and wait for completion
         let state = self.state.read().await;
@@ -1298,7 +1298,7 @@ impl RegionWriter {
         // Shutdown background tasks
         self.task_executor.shutdown_all().await?;
 
-        info!("RegionWriter closed for region {}", self.config.region_id);
+        info!("ShardWriter closed for shard {}", self.config.shard_id);
         Ok(())
     }
 }
@@ -1377,6 +1377,12 @@ impl WalFlushHandler {
     /// * `batch_store` - The batch store to flush from
     /// * `indexes` - Optional indexes to update in parallel with WAL I/O
     /// * `end_batch_position` - End batch ID (exclusive). Flush batches in (max_flushed, end_batch_position).
+    #[instrument(
+        name = "wal_do_flush",
+        level = "debug",
+        skip_all,
+        fields(end_batch_position)
+    )]
     async fn do_flush(
         &self,
         batch_store: Arc<BatchStore>,
@@ -1490,6 +1496,7 @@ impl MemTableFlushHandler {
     /// This method waits for the WAL flush to complete (sent at freeze time),
     /// then flushes to Lance storage. The WAL flush is already queued by
     /// freeze_memtable to ensure strict ordering of WAL entries.
+    #[instrument(name = "mt_flush", level = "info", skip_all, fields(generation = memtable.generation(), row_count = memtable.row_count()))]
     async fn flush_memtable(
         &mut self,
         memtable: Arc<MemTable>,
@@ -1863,13 +1870,13 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_region_writer_basic_write() {
+    async fn test_shard_writer_basic_write() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
         let schema = create_test_schema();
 
-        let config = RegionWriterConfig {
-            region_id: Uuid::new_v4(),
-            region_spec_id: 0,
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            shard_spec_id: 0,
             durable_write: false,
             sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
@@ -1879,7 +1886,7 @@ mod tests {
             ..Default::default()
         };
 
-        let writer = RegionWriter::open(
+        let writer = ShardWriter::open(
             store,
             base_path,
             base_uri,
@@ -1906,13 +1913,13 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_region_writer_multiple_writes() {
+    async fn test_shard_writer_multiple_writes() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
         let schema = create_test_schema();
 
-        let config = RegionWriterConfig {
-            region_id: Uuid::new_v4(),
-            region_spec_id: 0,
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            shard_spec_id: 0,
             durable_write: false,
             sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
@@ -1922,7 +1929,7 @@ mod tests {
             ..Default::default()
         };
 
-        let writer = RegionWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
             .await
             .unwrap();
 
@@ -1941,13 +1948,13 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_region_writer_with_indexes() {
+    async fn test_shard_writer_with_indexes() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
         let schema = create_test_schema();
 
-        let config = RegionWriterConfig {
-            region_id: Uuid::new_v4(),
-            region_spec_id: 0,
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            shard_spec_id: 0,
             durable_write: false,
             sync_indexed_write: true,
             max_wal_buffer_size: 1024 * 1024,
@@ -1963,7 +1970,7 @@ mod tests {
             column: "id".to_string(),
         })];
 
-        let writer = RegionWriter::open(
+        let writer = ShardWriter::open(
             store,
             base_path,
             base_uri,
@@ -1986,14 +1993,14 @@ mod tests {
 
     /// Test memtable auto-flush triggered by size threshold.
     #[tokio::test]
-    async fn test_region_writer_auto_flush_by_size() {
+    async fn test_shard_writer_auto_flush_by_size() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
         let schema = create_test_schema();
 
         // Use a small memtable size to trigger auto-flush
-        let config = RegionWriterConfig {
-            region_id: Uuid::new_v4(),
-            region_spec_id: 0,
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            shard_spec_id: 0,
             durable_write: false,
             sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
@@ -2003,7 +2010,7 @@ mod tests {
             ..Default::default()
         };
 
-        let writer = RegionWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
             .await
             .unwrap();
 
@@ -2030,7 +2037,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_no_backpressure_when_under_threshold() {
-        let config = RegionWriterConfig::default().with_max_unflushed_memtable_bytes(1024 * 1024); // 1MB
+        let config = ShardWriterConfig::default().with_max_unflushed_memtable_bytes(1024 * 1024); // 1MB
 
         let controller = BackpressureController::new(config);
 
@@ -2048,7 +2055,7 @@ mod tests {
         use std::sync::atomic::AtomicUsize;
         use std::time::Duration;
 
-        let config = RegionWriterConfig::default()
+        let config = ShardWriterConfig::default()
             .with_max_unflushed_memtable_bytes(100) // Very low threshold
             .with_backpressure_log_interval(Duration::from_millis(50));
 
@@ -2125,19 +2132,20 @@ mod tests {
 }
 
 #[cfg(test)]
-mod region_writer_tests {
+mod shard_writer_tests {
     use std::sync::Arc;
 
+    use crate::index::DatasetIndexExt;
     use arrow_array::{
         FixedSizeListArray, Float32Array, Int64Array, RecordBatch, RecordBatchIterator, StringArray,
     };
     use arrow_schema::{DataType, Field, Schema as ArrowSchema};
     use lance_arrow::FixedSizeListArrayExt;
+    use lance_index::IndexType;
     use lance_index::scalar::ScalarIndexParams;
     use lance_index::scalar::inverted::InvertedIndexParams;
     use lance_index::vector::ivf::IvfBuildParams;
     use lance_index::vector::pq::builder::PQBuildParams;
-    use lance_index::{DatasetIndexExt, IndexType};
     use lance_linalg::distance::MetricType;
     use uuid::Uuid;
 
@@ -2145,7 +2153,7 @@ mod region_writer_tests {
     use crate::dataset::{Dataset, WriteParams};
     use crate::index::vector::VectorIndexParams;
 
-    use super::super::RegionWriterConfig;
+    use super::super::ShardWriterConfig;
 
     fn create_test_schema(vector_dim: i32) -> Arc<ArrowSchema> {
         use std::collections::HashMap;
@@ -2205,16 +2213,16 @@ mod region_writer_tests {
         .unwrap()
     }
 
-    /// Quick smoke test for region writer - runs against memory://
-    /// Run with: cargo test -p lance region_writer_tests::test_region_writer_smoke -- --nocapture
+    /// Quick smoke test for shard writer - runs against memory://
+    /// Run with: cargo test -p lance shard_writer_tests::test_shard_writer_smoke -- --nocapture
     #[tokio::test]
-    async fn test_region_writer_smoke() {
+    async fn test_shard_writer_smoke() {
         let vector_dim = 128;
         let batch_size = 20;
         let num_batches = 100;
 
         let schema = create_test_schema(vector_dim);
-        let uri = format!("memory://test_region_writer_{}", Uuid::new_v4());
+        let uri = format!("memory://test_shard_writer_{}", Uuid::new_v4());
 
         // Create initial dataset
         let initial_batch = create_test_batch(&schema, 0, 100, vector_dim);
@@ -2226,20 +2234,20 @@ mod region_writer_tests {
         // Initialize MemWAL (no indexes for smoke test)
         dataset
             .initialize_mem_wal(MemWalConfig {
-                region_spec: None,
+                shard_spec: None,
                 maintained_indexes: vec![],
             })
             .await
             .expect("Failed to initialize MemWAL");
 
-        // Create region writer
-        let region_id = Uuid::new_v4();
-        let config = RegionWriterConfig::new(region_id)
+        // Create shard writer
+        let shard_id = Uuid::new_v4();
+        let config = ShardWriterConfig::new(shard_id)
             .with_durable_write(false)
             .with_sync_indexed_write(false);
 
         let writer = dataset
-            .mem_wal_writer(region_id, config)
+            .mem_wal_writer(shard_id, config)
             .await
             .expect("Failed to create writer");
 
@@ -2254,11 +2262,11 @@ mod region_writer_tests {
         writer.close().await.expect("Failed to close");
     }
 
-    /// Test region writer against S3 with IVF-PQ, BTree, and FTS indexes (requires DATASET_PREFIX env var)
-    /// Run with: DATASET_PREFIX=s3://bucket/path cargo test -p lance --release region_writer_tests::test_region_writer_s3_ivfpq -- --nocapture --ignored
+    /// Test shard writer against S3 with IVF-PQ, BTree, and FTS indexes (requires DATASET_PREFIX env var)
+    /// Run with: DATASET_PREFIX=s3://bucket/path cargo test -p lance --release shard_writer_tests::test_shard_writer_s3_ivfpq -- --nocapture --ignored
     #[tokio::test]
     #[ignore]
-    async fn test_region_writer_s3_ivfpq() {
+    async fn test_shard_writer_s3_ivfpq() {
         let prefix = std::env::var("DATASET_PREFIX").expect("DATASET_PREFIX not set");
 
         let vector_dim = 512;
@@ -2335,7 +2343,7 @@ mod region_writer_tests {
         // Initialize MemWAL with all three indexes
         dataset
             .initialize_mem_wal(MemWalConfig {
-                region_spec: None,
+                shard_spec: None,
                 maintained_indexes: vec![
                     "id_btree".to_string(),
                     "text_fts".to_string(),
@@ -2345,14 +2353,14 @@ mod region_writer_tests {
             .await
             .expect("Failed to initialize MemWAL");
 
-        // Create region writer with default config
-        let region_id = Uuid::new_v4();
-        let config = RegionWriterConfig::new(region_id)
+        // Create shard writer with default config
+        let shard_id = Uuid::new_v4();
+        let config = ShardWriterConfig::new(shard_id)
             .with_durable_write(false)
             .with_sync_indexed_write(false);
 
         let writer = dataset
-            .mem_wal_writer(region_id, config)
+            .mem_wal_writer(shard_id, config)
             .await
             .expect("Failed to create writer");
 
@@ -2367,7 +2375,7 @@ mod region_writer_tests {
         writer.close().await.expect("Failed to close");
     }
 
-    /// End-to-end correctness test for RegionWriter with multiple memtable flushes.
+    /// End-to-end correctness test for ShardWriter with multiple memtable flushes.
     ///
     /// This test verifies:
     /// 1. Multiple memtable flushes are triggered via small memtable size
@@ -2376,9 +2384,9 @@ mod region_writer_tests {
     /// 4. Data can be read after each flush cycle
     /// 5. Manifest tracks flushed generations correctly
     ///
-    /// Run with: cargo test -p lance region_writer_tests::test_region_writer_e2e_correctness -- --nocapture
+    /// Run with: cargo test -p lance shard_writer_tests::test_shard_writer_e2e_correctness -- --nocapture
     #[tokio::test]
-    async fn test_region_writer_e2e_correctness() {
+    async fn test_shard_writer_e2e_correctness() {
         use std::time::Duration;
         use tempfile::TempDir;
 
@@ -2417,15 +2425,15 @@ mod region_writer_tests {
         // Initialize MemWAL with BTree index only (simpler for this test)
         dataset
             .initialize_mem_wal(MemWalConfig {
-                region_spec: None,
+                shard_spec: None,
                 maintained_indexes: vec!["id_btree".to_string()],
             })
             .await
             .expect("Failed to initialize MemWAL");
 
-        // Create region writer with small memtable size to trigger flushes
-        let region_id = Uuid::new_v4();
-        let config = RegionWriterConfig::new(region_id)
+        // Create shard writer with small memtable size to trigger flushes
+        let shard_id = Uuid::new_v4();
+        let config = ShardWriterConfig::new(shard_id)
             .with_durable_write(true) // Ensure WAL files are written
             .with_sync_indexed_write(true)
             .with_max_memtable_size(50 * 1024) // 50KB - triggers flush after ~8 batches
@@ -2433,7 +2441,7 @@ mod region_writer_tests {
             .with_max_wal_flush_interval(Duration::from_millis(50)); // Fast flush
 
         let writer = dataset
-            .mem_wal_writer(region_id, config)
+            .mem_wal_writer(shard_id, config)
             .await
             .expect("Failed to create writer");
 
@@ -2465,7 +2473,7 @@ mod region_writer_tests {
         writer.close().await.expect("Failed to close");
 
         // === VERIFY FILE SYSTEM LAYOUT ===
-        let mem_wal_dir = temp_dir.path().join("_mem_wal").join(region_id.to_string());
+        let mem_wal_dir = temp_dir.path().join("_mem_wal").join(shard_id.to_string());
         assert!(mem_wal_dir.exists(), "MemWAL directory should exist");
 
         // Check WAL directory
@@ -2497,7 +2505,7 @@ mod region_writer_tests {
             .await
             .expect("Failed to open store");
         let manifest_store =
-            super::super::manifest::RegionManifestStore::new(store, &base_path, region_id, 2);
+            super::super::manifest::ShardManifestStore::new(store, &base_path, shard_id, 2);
         let manifest = manifest_store
             .read_latest()
             .await
@@ -2510,12 +2518,12 @@ mod region_writer_tests {
             "Should have at least one flushed generation"
         );
         for flushed_gen in &manifest.flushed_generations {
-            // The path stored in manifest is relative to the region directory
-            // Construct full path: temp_dir/_mem_wal/region_id/generation_folder
+            // The path stored in manifest is relative to the shard directory
+            // Construct full path: temp_dir/_mem_wal/shard_id/generation_folder
             let gen_path = temp_dir
                 .path()
                 .join("_mem_wal")
-                .join(region_id.to_string())
+                .join(shard_id.to_string())
                 .join(&flushed_gen.path);
 
             // The generation directory should exist
@@ -2550,26 +2558,26 @@ mod region_writer_tests {
         // === VERIFY DATA CAN BE READ FROM NEW WRITER ===
         // Re-open dataset and create new writer to verify recovery
         let dataset = Dataset::open(&uri).await.expect("Failed to reopen dataset");
-        let new_region_id = Uuid::new_v4();
-        let new_config = RegionWriterConfig::new(new_region_id)
+        let new_shard_id = Uuid::new_v4();
+        let new_config = ShardWriterConfig::new(new_shard_id)
             .with_durable_write(false)
             .with_sync_indexed_write(true);
 
         let new_writer = dataset
-            .mem_wal_writer(new_region_id, new_config)
+            .mem_wal_writer(new_shard_id, new_config)
             .await
             .expect("Failed to create new writer");
 
-        // Write a test batch to verify the new region works
+        // Write a test batch to verify the new shard works
         let verify_batch = create_test_batch(&schema, 10000, 10, vector_dim);
         new_writer
             .put(vec![verify_batch])
             .await
-            .expect("Failed to write to new region");
+            .expect("Failed to write to new shard");
 
         let scanner = new_writer.scan().await;
         let result = scanner.try_into_batch().await.expect("Failed to scan");
-        assert_eq!(result.num_rows(), 10, "New region should have 10 rows");
+        assert_eq!(result.num_rows(), 10, "New shard should have 10 rows");
 
         new_writer
             .close()
diff --git a/rust/lance/src/dataset/optimize.rs b/rust/lance/src/dataset/optimize.rs
index c3e1149a2d8..c70eb93bbcd 100644
--- a/rust/lance/src/dataset/optimize.rs
+++ b/rust/lance/src/dataset/optimize.rs
@@ -89,12 +89,15 @@ use std::sync::Arc;
 use super::fragment::FileFragment;
 use super::index::DatasetIndexRemapperOptions;
 use super::rowids::load_row_id_sequences;
-use super::transaction::{Operation, RewriteGroup, RewrittenIndex, Transaction};
+use super::transaction::{
+    Operation, RewriteGroup, RewrittenIndex, Transaction, TransactionBuilder,
+};
 use super::utils::make_rowid_capture_stream;
 use super::{WriteMode, WriteParams, write_fragments_internal};
 use crate::Dataset;
 use crate::Result;
 use crate::dataset::utils::CapturedRowIds;
+use crate::index::DatasetIndexExt;
 use crate::io::commit::{commit_transaction, migrate_fragments};
 use datafusion::physical_plan::SendableRecordBatchStream;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
@@ -103,12 +106,11 @@ use lance_core::Error;
 use lance_core::datatypes::BlobHandling;
 use lance_core::utils::tokio::get_num_compute_intensive_cpus;
 use lance_core::utils::tracing::{DATASET_COMPACTING_EVENT, TRACE_DATASET_EVENTS};
-use lance_index::DatasetIndexExt;
 use lance_index::frag_reuse::FragReuseGroup;
 use lance_table::format::{Fragment, RowIdMeta};
 use roaring::{RoaringBitmap, RoaringTreemap};
 use serde::{Deserialize, Serialize};
-use tracing::info;
+use tracing::{info, warn};
 
 mod binary_copy;
 pub mod remapping;
@@ -201,6 +203,19 @@ pub struct CompactionOptions {
     /// Controls how much data is read at once when performing binary copy.
     /// Defaults to 16MB (16 * 1024 * 1024).
     pub binary_copy_read_batch_bytes: Option<usize>,
+    /// Maximum number of source fragments to compact in a single run. When set,
+    /// tasks are included in the plan until adding the next task would exceed
+    /// this limit. This allows for incremental compaction (e.g., compact 20
+    /// fragments at a time).
+    /// Defaults to `None` (no limit, all eligible fragments are compacted).
+    pub max_source_fragments: Option<usize>,
+    /// Transaction properties to store with this commit.
+    ///
+    /// These key-value pairs are stored in the transaction file
+    /// and can be read later to identify the source of the commit
+    /// (e.g., job_id for tracking completed compaction jobs).
+    #[serde(skip)]
+    pub transaction_properties: Option<Arc<HashMap<String, String>>>,
 }
 
 #[allow(deprecated)]
@@ -220,12 +235,138 @@ impl Default for CompactionOptions {
             enable_binary_copy: false,
             enable_binary_copy_force: false,
             binary_copy_read_batch_bytes: Some(16 * 1024 * 1024),
+            max_source_fragments: None,
+            transaction_properties: None,
         }
     }
 }
 
+/// Config key prefix for compaction options stored in the dataset manifest.
+pub const COMPACTION_CONFIG_PREFIX: &str = "lance.compaction.";
+
 #[allow(deprecated)]
 impl CompactionOptions {
+    /// Create [`CompactionOptions`] by starting with defaults and applying any
+    /// overrides found in the dataset manifest config.
+    ///
+    /// Config keys are prefixed with `lance.compaction.` and map to fields:
+    /// - `lance.compaction.target_rows_per_fragment`
+    /// - `lance.compaction.max_rows_per_group`
+    /// - `lance.compaction.max_bytes_per_file`
+    /// - `lance.compaction.materialize_deletions`
+    /// - `lance.compaction.materialize_deletions_threshold`
+    /// - `lance.compaction.defer_index_remap`
+    /// - `lance.compaction.batch_size`
+    /// - `lance.compaction.compaction_mode`
+    /// - `lance.compaction.binary_copy_read_batch_bytes`
+    /// - `lance.compaction.max_source_fragments`
+    pub fn from_dataset_config(config: &HashMap<String, String>) -> Result<Self> {
+        let mut opts = Self::default();
+        opts.apply_dataset_config(config)?;
+        Ok(opts)
+    }
+
+    /// Apply overrides from the dataset manifest config to this options struct.
+    ///
+    /// Only fields with corresponding config keys are modified; other fields
+    /// retain their current values.
+    pub fn apply_dataset_config(&mut self, config: &HashMap<String, String>) -> Result<()> {
+        for (key, value) in config {
+            let Some(field) = key.strip_prefix(COMPACTION_CONFIG_PREFIX) else {
+                continue;
+            };
+            match field {
+                "target_rows_per_fragment" => {
+                    self.target_rows_per_fragment = value.parse().map_err(|_| {
+                        Error::invalid_input(format!(
+                            "Invalid value for {}: '{}' (expected a non-negative integer)",
+                            key, value
+                        ))
+                    })?;
+                }
+                "max_rows_per_group" => {
+                    self.max_rows_per_group = value.parse().map_err(|_| {
+                        Error::invalid_input(format!(
+                            "Invalid value for {}: '{}' (expected a non-negative integer)",
+                            key, value
+                        ))
+                    })?;
+                }
+                "max_bytes_per_file" => {
+                    self.max_bytes_per_file = Some(value.parse().map_err(|_| {
+                        Error::invalid_input(format!(
+                            "Invalid value for {}: '{}' (expected a non-negative integer)",
+                            key, value
+                        ))
+                    })?);
+                }
+                "materialize_deletions" => {
+                    self.materialize_deletions = match value.to_lowercase().as_str() {
+                        "true" => true,
+                        "false" => false,
+                        _ => {
+                            return Err(Error::invalid_input(format!(
+                                "Invalid value for {}: '{}' (expected 'true' or 'false')",
+                                key, value
+                            )));
+                        }
+                    };
+                }
+                "materialize_deletions_threshold" => {
+                    self.materialize_deletions_threshold = value.parse().map_err(|_| {
+                        Error::invalid_input(format!(
+                            "Invalid value for {}: '{}' (expected a float between 0.0 and 1.0)",
+                            key, value
+                        ))
+                    })?;
+                }
+                "defer_index_remap" => {
+                    self.defer_index_remap = match value.to_lowercase().as_str() {
+                        "true" => true,
+                        "false" => false,
+                        _ => {
+                            return Err(Error::invalid_input(format!(
+                                "Invalid value for {}: '{}' (expected 'true' or 'false')",
+                                key, value
+                            )));
+                        }
+                    };
+                }
+                "batch_size" => {
+                    self.batch_size = Some(value.parse().map_err(|_| {
+                        Error::invalid_input(format!(
+                            "Invalid value for {}: '{}' (expected a non-negative integer)",
+                            key, value
+                        ))
+                    })?);
+                }
+                "compaction_mode" => {
+                    self.compaction_mode = Some(CompactionMode::try_from(value.as_str())?);
+                }
+                "binary_copy_read_batch_bytes" => {
+                    self.binary_copy_read_batch_bytes = Some(value.parse().map_err(|_| {
+                        Error::invalid_input(format!(
+                            "Invalid value for {}: '{}' (expected a non-negative integer)",
+                            key, value
+                        ))
+                    })?);
+                }
+                "max_source_fragments" => {
+                    self.max_source_fragments = Some(value.parse().map_err(|_| {
+                        Error::invalid_input(format!(
+                            "Invalid value for {}: '{}' (expected a non-negative integer)",
+                            key, value
+                        ))
+                    })?);
+                }
+                _ => {
+                    warn!("Ignoring unknown compaction config key: {}", key);
+                }
+            }
+        }
+        Ok(())
+    }
+
     pub fn validate(&mut self) {
         // If threshold is 100%, same as turning off deletion materialization.
         if self.materialize_deletions && self.materialize_deletions_threshold >= 1.0 {
@@ -247,6 +388,12 @@ impl CompactionOptions {
             _ => CompactionMode::Reencode,
         }
     }
+
+    /// Set transaction properties to store in the commit manifest.
+    pub fn transaction_properties(mut self, properties: HashMap<String, String>) -> Self {
+        self.transaction_properties = Some(Arc::new(properties));
+        self
+    }
 }
 
 /// Determine if page-level binary copy can safely merge the provided fragments.
@@ -551,17 +698,31 @@ impl CompactionPlanner for DefaultCompactionPlanner {
             candidate_bins.push(bin);
         }
 
-        let final_bins = candidate_bins
+        let all_tasks: Vec<TaskData> = candidate_bins
             .into_iter()
             .filter(|bin| !bin.is_noop())
             .flat_map(|bin| bin.split_for_size(self.options.target_rows_per_fragment))
             .map(|bin| TaskData {
                 fragments: bin.fragments,
-            });
+            })
+            .collect();
+
+        let tasks = if let Some(max_frags) = self.options.max_source_fragments {
+            let mut total_frags = 0;
+            all_tasks
+                .into_iter()
+                .take_while(|task| {
+                    total_frags += task.fragments.len();
+                    total_frags <= max_frags
+                })
+                .collect()
+        } else {
+            all_tasks
+        };
 
         let mut compaction_plan =
             CompactionPlan::new(dataset.manifest.version, self.options.clone());
-        compaction_plan.extend_tasks(final_bins);
+        compaction_plan.extend_tasks(tasks);
 
         Ok(compaction_plan)
     }
@@ -1401,6 +1562,7 @@ pub async fn commit_compaction(
                 new_id: rewritten.new_id,
                 new_index_details: rewritten.index_details,
                 new_index_version: rewritten.index_version,
+                new_index_files: rewritten.files,
             })
             .collect()
     } else if !options.defer_index_remap && !has_address_style {
@@ -1423,15 +1585,16 @@ pub async fn commit_compaction(
         None
     };
 
-    let transaction = Transaction::new(
+    let transaction = TransactionBuilder::new(
         dataset.manifest.version,
         Operation::Rewrite {
             groups: rewrite_groups,
             rewritten_indices,
             frag_reuse_index,
         },
-        None,
-    );
+    )
+    .transaction_properties(options.transaction_properties.clone())
+    .build();
 
     dataset
         .apply_commit(transaction, &Default::default(), &Default::default())
@@ -2291,6 +2454,71 @@ mod tests {
         assert_eq!(before_scalar_result, after_scalar_result);
     }
 
+    // Regression test for https://github.com/lancedb/lance/issues/6161
+    // When FragReuseIndexDetails exceeds 204800 bytes it is written to an external
+    // file. Previously the file was silently dropped (temp file deleted) because
+    // tokio::io::AsyncWriteExt::shutdown was called instead of
+    // lance_io::traits::Writer::shutdown, which persists the temp file.
+    #[tokio::test]
+    async fn test_defer_index_remap_large_external_file() {
+        let test_dir = TempStrDir::default();
+        let test_uri = &test_dir;
+
+        // Create ~150 fragments × 1000 rows to produce a FragReuseIndexDetails
+        // that exceeds the 204800-byte inline threshold (~302 KB serialized).
+        let num_fragments = 150usize;
+        let rows_per_fragment = 1000usize;
+        let total_rows = num_fragments * rows_per_fragment;
+
+        let schema = Arc::new(Schema::new(vec![Field::new("i", DataType::Int32, false)]));
+
+        let mut dataset = Dataset::write(
+            RecordBatchIterator::new(
+                vec![Ok(RecordBatch::try_new(
+                    schema.clone(),
+                    vec![Arc::new(Int32Array::from_iter_values(0..total_rows as i32)) as ArrayRef],
+                )
+                .unwrap())],
+                schema.clone(),
+            ),
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: rows_per_fragment,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(dataset.get_fragments().len(), num_fragments);
+
+        // Delete a few rows from each fragment so compaction has something to do.
+        dataset.delete("i % 1000 = 0").await.unwrap();
+
+        compact_files(
+            &mut dataset,
+            CompactionOptions {
+                defer_index_remap: true,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+
+        // Loading the FragReuseIndex details must succeed even when the details
+        // were written to an external file.
+        let frag_reuse_meta = dataset
+            .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+            .await
+            .unwrap()
+            .expect("fragment reuse index must exist after compaction");
+
+        load_frag_reuse_index_details(&dataset, &frag_reuse_meta)
+            .await
+            .expect("loading large frag reuse index details must not fail");
+    }
+
     #[tokio::test]
     async fn test_defer_index_remap() {
         let mut data_gen = BatchGenerator::new()
@@ -3978,4 +4206,362 @@ mod tests {
         // make sure options.validate() worked
         assert!(!plan.options.materialize_deletions);
     }
+
+    #[test]
+    fn test_from_dataset_config() {
+        let config = HashMap::from([
+            (
+                "lance.compaction.target_rows_per_fragment".to_string(),
+                "500000".to_string(),
+            ),
+            (
+                "lance.compaction.max_rows_per_group".to_string(),
+                "2048".to_string(),
+            ),
+            (
+                "lance.compaction.max_bytes_per_file".to_string(),
+                "1000000".to_string(),
+            ),
+            (
+                "lance.compaction.materialize_deletions".to_string(),
+                "false".to_string(),
+            ),
+            (
+                "lance.compaction.materialize_deletions_threshold".to_string(),
+                "0.25".to_string(),
+            ),
+            (
+                "lance.compaction.defer_index_remap".to_string(),
+                "true".to_string(),
+            ),
+            (
+                "lance.compaction.batch_size".to_string(),
+                "4096".to_string(),
+            ),
+            (
+                "lance.compaction.compaction_mode".to_string(),
+                "try_binary_copy".to_string(),
+            ),
+            (
+                "lance.compaction.binary_copy_read_batch_bytes".to_string(),
+                "8388608".to_string(),
+            ),
+        ]);
+
+        let opts = CompactionOptions::from_dataset_config(&config).unwrap();
+        assert_eq!(opts.target_rows_per_fragment, 500_000);
+        assert_eq!(opts.max_rows_per_group, 2048);
+        assert_eq!(opts.max_bytes_per_file, Some(1_000_000));
+        assert!(!opts.materialize_deletions);
+        assert!((opts.materialize_deletions_threshold - 0.25).abs() < f32::EPSILON);
+        assert!(opts.defer_index_remap);
+        assert_eq!(opts.batch_size, Some(4096));
+        assert_eq!(opts.compaction_mode, Some(CompactionMode::TryBinaryCopy));
+        assert_eq!(opts.binary_copy_read_batch_bytes, Some(8_388_608));
+    }
+
+    #[test]
+    fn test_from_dataset_config_empty() {
+        let config = HashMap::new();
+        let opts = CompactionOptions::from_dataset_config(&config).unwrap();
+        let defaults = CompactionOptions::default();
+        assert_eq!(
+            opts.target_rows_per_fragment,
+            defaults.target_rows_per_fragment
+        );
+        assert_eq!(opts.max_rows_per_group, defaults.max_rows_per_group);
+        assert_eq!(opts.max_bytes_per_file, defaults.max_bytes_per_file);
+        assert_eq!(opts.materialize_deletions, defaults.materialize_deletions);
+        assert_eq!(
+            opts.materialize_deletions_threshold,
+            defaults.materialize_deletions_threshold
+        );
+        assert_eq!(opts.defer_index_remap, defaults.defer_index_remap);
+        assert_eq!(opts.batch_size, defaults.batch_size);
+        assert_eq!(opts.compaction_mode, defaults.compaction_mode);
+        assert_eq!(
+            opts.binary_copy_read_batch_bytes,
+            defaults.binary_copy_read_batch_bytes
+        );
+    }
+
+    #[test]
+    fn test_from_dataset_config_partial() {
+        let config = HashMap::from([(
+            "lance.compaction.target_rows_per_fragment".to_string(),
+            "500000".to_string(),
+        )]);
+
+        let opts = CompactionOptions::from_dataset_config(&config).unwrap();
+        assert_eq!(opts.target_rows_per_fragment, 500_000);
+        // Other fields should remain at defaults
+        let defaults = CompactionOptions::default();
+        assert_eq!(opts.max_rows_per_group, defaults.max_rows_per_group);
+        assert_eq!(opts.max_bytes_per_file, defaults.max_bytes_per_file);
+        assert_eq!(opts.materialize_deletions, defaults.materialize_deletions);
+        assert_eq!(opts.defer_index_remap, defaults.defer_index_remap);
+        assert_eq!(opts.batch_size, defaults.batch_size);
+        assert_eq!(opts.compaction_mode, defaults.compaction_mode);
+        assert_eq!(
+            opts.binary_copy_read_batch_bytes,
+            defaults.binary_copy_read_batch_bytes
+        );
+    }
+
+    #[test]
+    fn test_from_dataset_config_ignores_other_keys() {
+        let config = HashMap::from([
+            (
+                "lance.compaction.target_rows_per_fragment".to_string(),
+                "500000".to_string(),
+            ),
+            (
+                "lance.auto_cleanup.interval".to_string(),
+                "3600".to_string(),
+            ),
+            ("some.other.key".to_string(), "value".to_string()),
+        ]);
+
+        let opts = CompactionOptions::from_dataset_config(&config).unwrap();
+        assert_eq!(opts.target_rows_per_fragment, 500_000);
+    }
+
+    #[test]
+    fn test_from_dataset_config_invalid_value() {
+        let config = HashMap::from([(
+            "lance.compaction.target_rows_per_fragment".to_string(),
+            "not_a_number".to_string(),
+        )]);
+
+        let result = CompactionOptions::from_dataset_config(&config);
+        let err_msg = result.unwrap_err().to_string();
+        assert!(err_msg.contains("target_rows_per_fragment"));
+        assert!(err_msg.contains("not_a_number"));
+    }
+
+    #[test]
+    fn test_from_dataset_config_invalid_bool() {
+        let config = HashMap::from([(
+            "lance.compaction.materialize_deletions".to_string(),
+            "yes".to_string(),
+        )]);
+
+        let result = CompactionOptions::from_dataset_config(&config);
+        let err_msg = result.unwrap_err().to_string();
+        assert!(err_msg.contains("materialize_deletions"));
+        assert!(err_msg.contains("yes"));
+    }
+
+    #[test]
+    fn test_from_dataset_config_unknown_compaction_key() {
+        // Unknown keys should be ignored (with a warning) for forwards compatibility
+        let config = HashMap::from([(
+            "lance.compaction.unknown_key".to_string(),
+            "value".to_string(),
+        )]);
+
+        let opts = CompactionOptions::from_dataset_config(&config).unwrap();
+        // Should return defaults since the unknown key is skipped
+        let defaults = CompactionOptions::default();
+        assert_eq!(
+            opts.target_rows_per_fragment,
+            defaults.target_rows_per_fragment
+        );
+    }
+
+    #[test]
+    fn test_from_dataset_config_invalid_compaction_mode() {
+        let config = HashMap::from([(
+            "lance.compaction.compaction_mode".to_string(),
+            "invalid_mode".to_string(),
+        )]);
+
+        let result = CompactionOptions::from_dataset_config(&config);
+        let err_msg = result.unwrap_err().to_string();
+        assert!(err_msg.contains("invalid_mode"));
+    }
+
+    #[test]
+    fn test_apply_dataset_config_overrides() {
+        let config = HashMap::from([(
+            "lance.compaction.target_rows_per_fragment".to_string(),
+            "500000".to_string(),
+        )]);
+
+        let mut opts = CompactionOptions {
+            max_rows_per_group: 4096,
+            ..Default::default()
+        };
+        opts.apply_dataset_config(&config).unwrap();
+
+        // Config value should be applied
+        assert_eq!(opts.target_rows_per_fragment, 500_000);
+        // Explicitly set value should be preserved (config didn't have this key)
+        assert_eq!(opts.max_rows_per_group, 4096);
+    }
+
+    #[test]
+    fn test_apply_dataset_config_overwrites_matching_field() {
+        let config = HashMap::from([(
+            "lance.compaction.max_rows_per_group".to_string(),
+            "2048".to_string(),
+        )]);
+
+        let mut opts = CompactionOptions {
+            max_rows_per_group: 4096,
+            ..Default::default()
+        };
+        opts.apply_dataset_config(&config).unwrap();
+
+        // Config value should overwrite the pre-set value
+        assert_eq!(opts.max_rows_per_group, 2048);
+    }
+
+    #[tokio::test]
+    async fn test_max_source_fragments() {
+        let test_dir = TempStrDir::default();
+        let test_uri = &test_dir;
+
+        let data = sample_data();
+        let schema = data.schema();
+
+        // Create 10 small fragments (100 rows each) via 10 appends
+        let write_params = WriteParams {
+            max_rows_per_file: 100,
+            ..Default::default()
+        };
+        Dataset::write(
+            RecordBatchIterator::new(vec![Ok(data.slice(0, 100))], schema.clone()),
+            test_uri,
+            Some(write_params.clone()),
+        )
+        .await
+        .unwrap();
+        for i in 1..10 {
+            let mut append_params = write_params.clone();
+            append_params.mode = WriteMode::Append;
+            Dataset::write(
+                RecordBatchIterator::new(vec![Ok(data.slice(i * 100, 100))], schema.clone()),
+                test_uri,
+                Some(append_params),
+            )
+            .await
+            .unwrap();
+        }
+
+        let dataset = Dataset::open(test_uri).await.unwrap();
+        assert_eq!(dataset.get_fragments().len(), 10);
+
+        // Plan without limit - all 10 fragments should be candidates.
+        // Use a target that splits the 10 fragments into multiple tasks.
+        let opts_no_limit = CompactionOptions {
+            target_rows_per_fragment: 250,
+            ..Default::default()
+        };
+        let plan_all = plan_compaction(&dataset, &opts_no_limit).await.unwrap();
+        let total_source_frags: usize = plan_all.tasks().iter().map(|t| t.fragments.len()).sum();
+        assert_eq!(total_source_frags, 10);
+        assert!(
+            plan_all.num_tasks() > 2,
+            "need multiple tasks to test bounding, got {}",
+            plan_all.num_tasks()
+        );
+
+        // Plan with max_source_fragments=4 should include tasks covering <= 4
+        // source fragments
+        let opts_bounded = CompactionOptions {
+            target_rows_per_fragment: 250,
+            max_source_fragments: Some(4),
+            ..Default::default()
+        };
+        let plan_bounded = plan_compaction(&dataset, &opts_bounded).await.unwrap();
+        let bounded_source_frags: usize =
+            plan_bounded.tasks().iter().map(|t| t.fragments.len()).sum();
+        assert!(
+            bounded_source_frags <= 4,
+            "expected at most 4 source fragments, got {bounded_source_frags}"
+        );
+        assert!(
+            bounded_source_frags > 0,
+            "expected at least 1 source fragment in bounded plan"
+        );
+        assert!(
+            plan_bounded.num_tasks() < plan_all.num_tasks(),
+            "bounded plan ({}) should have fewer tasks than unbounded ({})",
+            plan_bounded.num_tasks(),
+            plan_all.num_tasks()
+        );
+
+        // Execute bounded compaction incrementally
+        let mut dataset = dataset;
+        compact_files(&mut dataset, opts_bounded, None)
+            .await
+            .unwrap();
+        let after_first = dataset.get_fragments().len();
+        assert!(
+            after_first < 10,
+            "expected fewer than 10 fragments after first compaction, got {after_first}"
+        );
+        assert!(
+            after_first > 1,
+            "expected partial compaction (not fully compacted), got {after_first}"
+        );
+
+        // Run again to make more progress
+        let opts_bounded = CompactionOptions {
+            target_rows_per_fragment: 250,
+            max_source_fragments: Some(4),
+            ..Default::default()
+        };
+        compact_files(&mut dataset, opts_bounded, None)
+            .await
+            .unwrap();
+        let after_second = dataset.get_fragments().len();
+        assert!(
+            after_second <= after_first,
+            "expected progress: {after_second} should be <= {after_first}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_compaction_uses_manifest_config() {
+        let test_dir = TempStrDir::default();
+        let test_uri = &test_dir;
+
+        let data = sample_data();
+        let schema = data.schema();
+
+        // Create dataset with small fragments
+        let reader = RecordBatchIterator::new(vec![Ok(data.clone())], schema.clone());
+        let write_params = WriteParams {
+            max_rows_per_file: 2000,
+            ..Default::default()
+        };
+        let mut dataset = Dataset::write(reader, test_uri, Some(write_params))
+            .await
+            .unwrap();
+
+        assert_eq!(dataset.get_fragments().len(), 5);
+
+        // Set compaction config in manifest
+        dataset
+            .update_config([
+                ("lance.compaction.target_rows_per_fragment", "5000"),
+                ("lance.compaction.materialize_deletions_threshold", "2.0"),
+            ])
+            .await
+            .unwrap();
+
+        // Build options from the dataset config (as the bindings do)
+        let opts = CompactionOptions::from_dataset_config(&dataset.manifest.config).unwrap();
+        assert_eq!(opts.target_rows_per_fragment, 5000);
+        assert!((opts.materialize_deletions_threshold - 2.0).abs() < f32::EPSILON);
+
+        // Verify the config flows through plan_compaction
+        let plan = plan_compaction(&dataset, &opts).await.unwrap();
+        assert!(!plan.tasks.is_empty());
+        assert_eq!(plan.options.target_rows_per_fragment, 5000);
+        // validate() should have turned off materialize_deletions since threshold >= 1.0
+        assert!(!plan.options.materialize_deletions);
+    }
 }
diff --git a/rust/lance/src/dataset/optimize/binary_copy.rs b/rust/lance/src/dataset/optimize/binary_copy.rs
index 05299ab95f7..8b205471f49 100644
--- a/rust/lance/src/dataset/optimize/binary_copy.rs
+++ b/rust/lance/src/dataset/optimize/binary_copy.rs
@@ -136,9 +136,10 @@ async fn finalize_current_output_file(
     // Register the newly closed output file as a fragment data file
     let (maj, min) = version.to_numbers();
     let mut fragment = Fragment::new(0);
+    let field_column_indices = compute_field_column_indices(schema, full_field_ids.len(), version);
     let mut data_file = DataFile::new_unstarted(current_filename.take().unwrap(), maj, min);
-    data_file.fields = full_field_ids.to_vec();
-    data_file.column_indices = compute_field_column_indices(schema, full_field_ids.len(), version);
+    data_file.fields = full_field_ids.to_vec().into();
+    data_file.column_indices = field_column_indices.into();
     fragment.files.push(data_file);
     fragment.physical_rows = Some(total_rows_in_current as usize);
     Ok(fragment)
diff --git a/rust/lance/src/dataset/optimize/remapping.rs b/rust/lance/src/dataset/optimize/remapping.rs
index 1f2508beff5..dab62bf6166 100644
--- a/rust/lance/src/dataset/optimize/remapping.rs
+++ b/rust/lance/src/dataset/optimize/remapping.rs
@@ -6,14 +6,14 @@
 
 use crate::Result;
 use crate::dataset::transaction::{Operation, Transaction};
+use crate::index::DatasetIndexExt;
 use crate::index::frag_reuse::{load_frag_reuse_index_details, open_frag_reuse_index};
 use crate::{Dataset, index};
 use async_trait::async_trait;
 use lance_core::Error;
 use lance_core::utils::address::RowAddress;
-use lance_index::DatasetIndexExt;
 use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragDigest};
-use lance_table::format::{Fragment, IndexMetadata};
+use lance_table::format::{Fragment, IndexFile, IndexMetadata};
 use lance_table::io::manifest::read_manifest_indexes;
 use roaring::RoaringTreemap;
 use serde::{Deserialize, Serialize};
@@ -39,6 +39,8 @@ pub struct RemappedIndex {
     pub new_id: Uuid,
     pub index_details: prost_types::Any,
     pub index_version: u32,
+    /// List of files in the index with their sizes.
+    pub files: Option<Vec<IndexFile>>,
 }
 
 /// When compaction runs the row ids will change.  This typically means that
@@ -283,6 +285,7 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> {
                     index_version: curr_index_meta.index_version,
                     created_at: curr_index_meta.created_at,
                     base_id: None,
+                    files: curr_index_meta.files.clone(),
                 },
                 RemapResult::Remapped(remapped_index) => IndexMetadata {
                     uuid: remapped_index.new_id,
@@ -294,6 +297,7 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> {
                     index_version: remapped_index.index_version as i32,
                     created_at: curr_index_meta.created_at,
                     base_id: None,
+                    files: remapped_index.files,
                 },
             };
 
diff --git a/rust/lance/src/dataset/optimize/tests/binary_copy.rs b/rust/lance/src/dataset/optimize/tests/binary_copy.rs
index 1081384ee36..f7041bcf38a 100644
--- a/rust/lance/src/dataset/optimize/tests/binary_copy.rs
+++ b/rust/lance/src/dataset/optimize/tests/binary_copy.rs
@@ -566,11 +566,13 @@ async fn test_can_use_binary_copy_schema_mismatch() {
         .collect();
     // Introduce a column index mismatch in the first data file
     if let Some(df) = frags.get_mut(0).and_then(|f| f.files.get_mut(0)) {
-        if let Some(first) = df.column_indices.get_mut(0) {
+        let mut indices = df.column_indices.to_vec();
+        if let Some(first) = indices.get_mut(0) {
             *first = -*first - 1;
         } else {
-            df.column_indices.push(-1);
+            indices.push(-1);
         }
+        df.column_indices = indices.into();
     }
     assert!(!can_use_binary_copy(&dataset, &options, &frags).await);
 
diff --git a/rust/lance/src/dataset/progress.rs b/rust/lance/src/dataset/progress.rs
index 1e489d0b768..e970adc47ee 100644
--- a/rust/lance/src/dataset/progress.rs
+++ b/rust/lance/src/dataset/progress.rs
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use std::sync::Arc;
+
 use async_trait::async_trait;
 use lance_table::format::Fragment;
 
@@ -26,6 +28,47 @@ pub trait WriteFragmentProgress: std::fmt::Debug + Sync + Send {
     async fn complete(&self, fragment: &Fragment) -> Result<()>;
 }
 
+/// Statistics reported to the write progress callback set via
+/// [`InsertBuilder::progress`](crate::dataset::InsertBuilder::progress) or
+/// [`WriteParams::write_progress`](crate::dataset::WriteParams::write_progress).
+#[derive(Debug, Clone, Default)]
+pub struct WriteStats {
+    /// Cumulative bytes handed to the writer so far.
+    ///
+    /// For local storage this closely tracks bytes flushed to disk. For cloud
+    /// object stores (S3, GCS, Azure) this reflects bytes handed to the
+    /// multipart-upload buffer; actual network I/O may lag slightly.
+    pub bytes_written: u64,
+    /// Cumulative rows written so far.
+    pub rows_written: u64,
+    /// Number of files (fragments) whose writes have completed so far.
+    pub files_written: u32,
+}
+
+/// An opaque wrapper around a write-progress closure.
+///
+/// Stored inside [`WriteParams::write_progress`](crate::dataset::WriteParams::write_progress).
+/// Construct via [`InsertBuilder::progress`](crate::dataset::InsertBuilder::progress) or
+/// directly with [`WriteProgressFn::new`].
+#[derive(Clone)]
+pub struct WriteProgressFn(Arc<dyn Fn(WriteStats) + Send + Sync>);
+
+impl WriteProgressFn {
+    pub fn new(f: impl Fn(WriteStats) + Send + Sync + 'static) -> Self {
+        Self(Arc::new(f))
+    }
+
+    pub(crate) fn call(&self, stats: WriteStats) {
+        (self.0)(stats);
+    }
+}
+
+impl std::fmt::Debug for WriteProgressFn {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("WriteProgressFn").finish_non_exhaustive()
+    }
+}
+
 /// By default, Progress tracker is Noop.
 #[derive(Debug, Clone, Default)]
 pub struct NoopFragmentWriteProgress {}
diff --git a/rust/lance/src/dataset/rowids.rs b/rust/lance/src/dataset/rowids.rs
index b6b4b106798..d40a9adfd50 100644
--- a/rust/lance/src/dataset/rowids.rs
+++ b/rust/lance/src/dataset/rowids.rs
@@ -92,19 +92,25 @@ async fn load_row_id_index(dataset: &Dataset) -> Result<lance_table::rowids::Row
         .try_collect::<Vec<_>>()
         .await?;
 
+    let fragments = dataset.get_fragments();
+    let fragment_map: std::collections::HashMap<u32, &crate::dataset::fragment::FileFragment> =
+        fragments.iter().map(|f| (f.id() as u32, f)).collect();
+
     let fragment_indices: Vec<_> =
-        futures::future::try_join_all(sequences.into_iter().map(|(fragment_id, sequence)| {
-            let dataset = dataset.clone();
+        futures::stream::iter(sequences.into_iter().map(|(fragment_id, sequence)| {
+            let fragment = fragment_map
+                .get(&fragment_id)
+                .expect("Fragment should exist");
+            let has_deletion_file = fragment.metadata().deletion_file.is_some();
+            let fragment_clone = (*fragment).clone();
             async move {
-                let fragments = dataset.get_fragments();
-                let fragment = fragments
-                    .iter()
-                    .find(|f| f.id() as u32 == fragment_id)
-                    .expect("Fragment should exist");
-
-                let deletion_vector = match fragment.get_deletion_vector().await {
-                    Ok(Some(dv)) => dv,
-                    Ok(None) | Err(_) => Arc::new(DeletionVector::default()),
+                let deletion_vector = if has_deletion_file {
+                    match fragment_clone.get_deletion_vector().await {
+                        Ok(Some(dv)) => dv,
+                        Ok(None) | Err(_) => Arc::new(DeletionVector::default()),
+                    }
+                } else {
+                    Arc::new(DeletionVector::default())
                 };
 
                 Ok::<FragmentRowIdIndex, Error>(FragmentRowIdIndex {
@@ -114,6 +120,8 @@ async fn load_row_id_index(dataset: &Dataset) -> Result<lance_table::rowids::Row
                 })
             }
         }))
+        .buffer_unordered(dataset.object_store.io_parallelism())
+        .try_collect()
         .await?;
 
     let index = RowIdIndex::new(&fragment_indices)?;
@@ -130,6 +138,7 @@ mod test {
     use super::*;
 
     use crate::dataset::optimize::{CompactionOptions, compact_files};
+    use crate::index::DatasetIndexExt;
     use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount};
     use arrow_array::cast::AsArray;
     use arrow_array::types::{Float32Type, Int32Type, UInt64Type};
@@ -139,7 +148,7 @@ mod test {
     use lance_core::datatypes::Schema;
     use lance_core::{ROW_ADDR, ROW_ID, utils::address::RowAddress};
     use lance_datagen::Dimension;
-    use lance_index::{DatasetIndexExt, IndexType, scalar::ScalarIndexParams};
+    use lance_index::{IndexType, scalar::ScalarIndexParams};
     use std::collections::HashMap;
     use std::collections::HashSet;
 
diff --git a/rust/lance/src/dataset/scanner.rs b/rust/lance/src/dataset/scanner.rs
index cdaa9408281..5f40c79b4f9 100644
--- a/rust/lance/src/dataset/scanner.rs
+++ b/rust/lance/src/dataset/scanner.rs
@@ -1,11 +1,13 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use std::collections::HashSet;
 use std::ops::Range;
 use std::pin::Pin;
 use std::sync::{Arc, LazyLock};
 use std::task::{Context, Poll};
 
+use crate::index::DatasetIndexExt;
 use arrow::array::AsArray;
 use arrow_array::{Array, Float32Array, Int64Array, RecordBatch};
 use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema, SchemaRef, SortOptions};
@@ -60,19 +62,20 @@ use lance_datafusion::projection::ProjectionPlan;
 use lance_file::reader::FileReaderOptions;
 use lance_index::IndexCriteria;
 use lance_index::scalar::FullTextSearchQuery;
+use lance_index::scalar::expression::ScalarIndexExpr;
 use lance_index::scalar::expression::{INDEX_EXPR_RESULT_SCHEMA, IndexExprResult, PlannerIndexExt};
 use lance_index::scalar::inverted::query::{
     FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, PhraseQuery, fill_fts_query_column,
 };
 use lance_index::scalar::inverted::{SCORE_COL, SCORE_FIELD};
 use lance_index::vector::{DIST_COL, Query};
-use lance_index::{DatasetIndexExt, scalar::expression::ScalarIndexExpr};
 use lance_index::{metrics::NoOpMetricsCollector, scalar::inverted::FTS_SCHEMA};
 use lance_io::stream::RecordBatchStream;
 use lance_linalg::distance::MetricType;
 use lance_table::format::{Fragment, IndexMetadata};
 use roaring::RoaringBitmap;
 use tracing::{Span, info_span, instrument};
+use uuid::Uuid;
 
 use super::Dataset;
 use crate::dataset::row_offsets_to_row_addresses;
@@ -716,6 +719,10 @@ pub struct Scanner {
     /// The batch size controls the maximum size of rows to return for each read.
     batch_size: Option<usize>,
 
+    /// If set, the scanner will produce batches whose total size in bytes
+    /// is approximately this value, overriding the row-based `batch_size`.
+    batch_size_bytes: Option<u64>,
+
     /// Number of batches to prefetch
     batch_readahead: usize,
 
@@ -760,6 +767,9 @@ pub struct Scanner {
     /// If set, this scanner serves only these fragments.
     fragments: Option<Vec<Fragment>>,
 
+    /// If set, this scanner will only search the specified vector index segments.
+    index_segments: Option<Vec<Uuid>>,
+
     /// Only search the data being indexed (weak consistency search).
     ///
     /// Default value is false.
@@ -983,6 +993,7 @@ impl Scanner {
             filter: LanceFilter::default(),
             full_text_query: None,
             batch_size: None,
+            batch_size_bytes: None,
             batch_readahead: get_num_compute_intensive_cpus(),
             fragment_readahead: None,
             io_buffer_size: None,
@@ -993,6 +1004,7 @@ impl Scanner {
             use_stats: true,
             ordered: true,
             fragments: None,
+            index_segments: None,
             fast_search: false,
             use_scalar_index: true,
             include_deleted_rows: false,
@@ -1039,6 +1051,23 @@ impl Scanner {
         self
     }
 
+    /// Restrict vector index search to the specified index segments.
+    ///
+    /// This setting is only supported for vector search.
+    ///
+    /// If [`Self::with_fragments`] is also set then rows from those fragments that are not covered
+    /// by the selected index segments will still be searched with flat KNN. Otherwise, unindexed
+    /// fragments outside the selected index segments are not searched.
+    pub fn with_index_segments(&mut self, segments: Vec<Uuid>) -> Result<&mut Self> {
+        if segments.is_empty() {
+            return Err(Error::invalid_input(
+                "with_index_segments does not accept an empty segment list".to_string(),
+            ));
+        }
+        self.index_segments = Some(segments);
+        Ok(self)
+    }
+
     fn get_batch_size(&self) -> usize {
         // Default batch size to be large enough so that a i32 column can be
         // read in a single range request. For the object store default of
@@ -1237,12 +1266,29 @@ impl Scanner {
         Ok(self)
     }
 
-    /// Set the batch size.
+    /// Set the maximum number of rows per batch.
+    ///
+    /// Note: this can be overridden by [`Self::batch_size_bytes`] or by a dataset-level
+    /// `batch_size_bytes` set via [`ReadParams::file_reader_options`](crate::dataset::ReadParams::file_reader_options).  When a byte-based
+    /// batch size is active, the row-based batch size is used only as an initial estimate.
     pub fn batch_size(&mut self, batch_size: usize) -> &mut Self {
         self.batch_size = Some(batch_size);
         self
     }
 
+    /// Set the target batch size in bytes.
+    ///
+    /// When set, the scanner will produce batches whose total size in bytes
+    /// is approximately this value, overriding the row-based `batch_size`.
+    ///
+    /// This can also be configured at the dataset level via
+    /// [`ReadParams::file_reader_options`](crate::dataset::ReadParams::file_reader_options).  A scanner-level setting takes
+    /// precedence over the dataset-level default.
+    pub fn batch_size_bytes(&mut self, batch_size_bytes: u64) -> &mut Self {
+        self.batch_size_bytes = Some(batch_size_bytes);
+        self
+    }
+
     /// Include deleted rows
     ///
     /// These are rows that have been deleted from the dataset but are still present in the
@@ -1664,6 +1710,30 @@ impl Scanner {
         self
     }
 
+    /// Compute the resolved file reader options, merging the scanner's explicit
+    /// `file_reader_options`, the dataset-level defaults, and the `batch_size_bytes`
+    /// setting.
+    fn resolved_file_reader_options(&self) -> Option<FileReaderOptions> {
+        let base = self
+            .file_reader_options
+            .clone()
+            .or_else(|| self.dataset.file_reader_options.clone());
+        match (base, self.batch_size_bytes) {
+            (Some(mut opts), Some(bsb)) => {
+                if opts.batch_size_bytes.is_none() {
+                    opts.batch_size_bytes = Some(bsb);
+                }
+                Some(opts)
+            }
+            (Some(opts), None) => Some(opts),
+            (None, Some(bsb)) => Some(FileReaderOptions {
+                batch_size_bytes: Some(bsb),
+                ..Default::default()
+            }),
+            (None, None) => None,
+        }
+    }
+
     /// Create a physical expression for a column that may be nested
     fn create_column_expr(
         column_name: &str,
@@ -2166,6 +2236,12 @@ impl Scanner {
             }
         }
 
+        if self.index_segments.is_some() && self.nearest.is_none() {
+            return Err(Error::not_supported(
+                "with_index_segments is only supported for vector search".to_string(),
+            ));
+        }
+
         Ok(())
     }
 
@@ -2628,6 +2704,10 @@ impl Scanner {
             read_options = read_options.with_batch_size(batch_size as u32);
         }
 
+        if let Some(file_reader_options) = self.resolved_file_reader_options() {
+            read_options = read_options.with_file_reader_options(file_reader_options);
+        }
+
         if let Some(fragment_readahead) = self.fragment_readahead {
             read_options = read_options.with_fragment_readahead(fragment_readahead);
         }
@@ -2698,8 +2778,7 @@ impl Scanner {
         let row_addrs = RowAddrTreeMap::from_iter(u64s);
         let row_addr_mask = RowAddrMask::from_allowed(row_addrs);
         let index_result = IndexExprResult::Exact(row_addr_mask);
-        let fragments_covered =
-            RoaringBitmap::from_iter(self.dataset.fragments().iter().map(|f| f.id as u32));
+        let fragments_covered = self.dataset.fragment_bitmap.as_ref().clone();
         let batch = index_result.serialize_to_arrow(&fragments_covered)?;
         let stream = futures::stream::once(async move { Ok(batch) });
         let stream = Box::pin(RecordBatchStreamAdapter::new(
@@ -2719,15 +2798,21 @@ impl Scanner {
             TakeOperation::RowAddrs(addrs) => self.u64s_as_take_input(addrs),
             TakeOperation::RowOffsets(offsets) => {
                 let mut addrs =
-                    row_offsets_to_row_addresses(self.dataset.as_ref(), &offsets).await?;
+                    row_offsets_to_row_addresses(&self.dataset.get_fragments(), &offsets).await?;
                 addrs.retain(|addr| *addr != RowAddress::TOMBSTONE_ROW);
                 self.u64s_as_take_input(addrs)
             }
         }?;
 
+        let mut filtered_read_options = FilteredReadOptions::new(projection);
+        if let Some(fragment) = self.fragments.as_ref() {
+            filtered_read_options =
+                filtered_read_options.with_fragments(Arc::new(fragment.clone()));
+        }
+
         Ok(Arc::new(FilteredReadExec::try_new(
             self.dataset.clone(),
-            FilteredReadOptions::new(projection),
+            filtered_read_options,
             Some(input),
         )?))
     }
@@ -3415,55 +3500,127 @@ impl Scanner {
         } else {
             Arc::new(vec![])
         };
-        // Find an index for the column and check if metric is compatible
-        let matching_index = if let Some(index) =
-            indices.iter().find(|i| i.fields.contains(&column_id))
-        {
-            // TODO: Once we do https://github.com/lance-format/lance/issues/5231, we
-            // should be able to get the metric type directly from the index metadata,
-            // at least for newer indexes.
-            let idx = self
-                .dataset
-                .open_vector_index(
-                    q.column.as_str(),
-                    &index.uuid.to_string(),
-                    &NoOpMetricsCollector,
-                )
-                .await?;
-            let index_metric = idx.metric_type();
+        let index_and_segments = if use_index {
+            if let Some(requested_segments) = self.index_segments.as_ref() {
+                let requested_segment_set =
+                    requested_segments.iter().copied().collect::<HashSet<_>>();
+                let requested_index_segments = indices
+                    .iter()
+                    .filter(|idx| requested_segment_set.contains(&idx.uuid))
+                    .cloned()
+                    .collect::<Vec<_>>();
+
+                if requested_index_segments.len() != requested_segment_set.len() {
+                    let found_segment_set = requested_index_segments
+                        .iter()
+                        .map(|idx| idx.uuid)
+                        .collect::<HashSet<_>>();
+                    let missing_segments = requested_segment_set
+                        .difference(&found_segment_set)
+                        .map(ToString::to_string)
+                        .collect::<Vec<_>>();
+                    return Err(Error::invalid_input(format!(
+                        "with_index_segments referenced unknown index segments: {missing_segments:?}",
+                    )));
+                }
+
+                if requested_index_segments
+                    .iter()
+                    .any(|idx| !idx.fields.contains(&column_id))
+                {
+                    return Err(Error::invalid_input(format!(
+                        "with_index_segments contained a segment that does not belong to vector column '{}'",
+                        q.column
+                    )));
+                }
+
+                let index_name = requested_index_segments[0].name.clone();
+                if requested_index_segments
+                    .iter()
+                    .any(|idx| idx.name != index_name)
+                {
+                    return Err(Error::invalid_input(
+                        "with_index_segments must reference segments from a single logical index"
+                            .to_string(),
+                    ));
+                }
 
-            // Check if user's requested metric is compatible with index
-            let use_this_index = match q.metric_type {
-                Some(user_metric) => {
-                    if user_metric == index_metric {
-                        true
+                let selected_index_segments =
+                    self.retain_relevant_index_segments(requested_index_segments);
+                if selected_index_segments.is_empty() {
+                    None
+                } else {
+                    let idx = self
+                        .dataset
+                        .open_vector_index(
+                            q.column.as_str(),
+                            &selected_index_segments[0].uuid.to_string(),
+                            &NoOpMetricsCollector,
+                        )
+                        .await?;
+                    let index_metric = idx.metric_type();
+                    let use_this_index = match q.metric_type {
+                        Some(user_metric) => {
+                            if user_metric == index_metric {
+                                true
+                            } else {
+                                return Err(Error::invalid_input(format!(
+                                    "with_index_segments requested metric {:?} but the selected index segments use {:?}",
+                                    user_metric, index_metric
+                                )));
+                            }
+                        }
+                        None => true,
+                    };
+                    if use_this_index {
+                        Some((index_name, selected_index_segments, index_metric))
                     } else {
-                        log::warn!(
-                            "Requested metric {:?} is incompatible with index metric {:?}, falling back to brute-force search",
-                            user_metric,
-                            index_metric
-                        );
-                        false
+                        None
                     }
                 }
-                None => true, // No preference, use index's metric
-            };
+            } else if let Some(index) = indices.iter().find(|i| i.fields.contains(&column_id)) {
+                // TODO: Once we do https://github.com/lance-format/lance/issues/5231, we
+                // should be able to get the metric type directly from the index metadata,
+                // at least for newer indexes.
+                let idx = self
+                    .dataset
+                    .open_vector_index(
+                        q.column.as_str(),
+                        &index.uuid.to_string(),
+                        &NoOpMetricsCollector,
+                    )
+                    .await?;
+                let index_metric = idx.metric_type();
 
-            if use_this_index {
-                Some((index, idx, index_metric))
-            } else {
-                None
-            }
-        } else {
-            None
-        };
+                let use_this_index = match q.metric_type {
+                    Some(user_metric) => {
+                        if user_metric == index_metric {
+                            true
+                        } else {
+                            log::warn!(
+                                "Requested metric {:?} is incompatible with index metric {:?}, falling back to brute-force search",
+                                user_metric,
+                                index_metric
+                            );
+                            false
+                        }
+                    }
+                    None => true,
+                };
 
-        // Only return index and deltas if there is an index on the column and at least one of the target fragments are indexed
-        let index_and_deltas = if let Some((index, _idx, index_metric)) = matching_index {
-            let deltas = self.dataset.load_indices_by_name(&index.name).await?;
-            let index_frags = self.get_indexed_frags(&deltas);
-            if !index_frags.is_empty() {
-                Some((index, deltas, index_metric))
+                if use_this_index {
+                    let index_segments = self.retain_relevant_index_segments(
+                        self.dataset.load_indices_by_name(&index.name).await?,
+                    );
+                    let index_frags = self.get_indexed_frags(&index_segments);
+                    if !index_segments.is_empty() && !index_frags.is_empty() {
+                        Some((index.name.clone(), index_segments, index_metric))
+                    } else {
+                        None
+                    }
+                } else {
+                    None
+                }
             } else {
                 None
             }
@@ -3471,7 +3628,7 @@ impl Scanner {
             None
         };
 
-        if let Some((index, deltas, index_metric)) = index_and_deltas {
+        if let Some((index_name, index_segments, index_metric)) = index_and_segments {
             log::trace!("index found for vector search");
             // Use the index's metric type
             q.metric_type = Some(index_metric);
@@ -3483,8 +3640,8 @@ impl Scanner {
                 ));
             }
             let ann_node = match vector_type {
-                DataType::FixedSizeList(_, _) => self.ann(&q, &deltas, filter_plan).await?,
-                DataType::List(_) => self.multivec_ann(&q, &deltas, filter_plan).await?,
+                DataType::FixedSizeList(_, _) => self.ann(&q, &index_segments, filter_plan).await?,
+                DataType::List(_) => self.multivec_ann(&q, &index_segments, filter_plan).await?,
                 _ => unreachable!(),
             };
 
@@ -3501,7 +3658,9 @@ impl Scanner {
             }; // vector, _distance, _rowid
 
             if !self.fast_search {
-                knn_node = self.knn_combined(&q, index, knn_node, filter_plan).await?;
+                knn_node = self
+                    .knn_combined(&q, &index_name, &index_segments, knn_node, filter_plan)
+                    .await?;
             }
 
             Ok(knn_node)
@@ -3551,27 +3710,27 @@ impl Scanner {
     async fn knn_combined(
         &self,
         q: &Query,
-        index: &IndexMetadata,
+        index_name: &str,
+        indexed_segments: &[IndexMetadata],
         mut knn_node: Arc<dyn ExecutionPlan>,
         filter_plan: &ExprFilterPlan,
     ) -> Result<Arc<dyn ExecutionPlan>> {
-        // Get unindexed fragments and filter to target fragments
-        let unindexed_fragments =
-            self.retain_target_fragments(self.dataset.unindexed_fragments(&index.name).await?);
-
-        if !unindexed_fragments.is_empty() {
-            // need to set the metric type to be the same as the index
-            // to make sure the distance is comparable.
-            let idx = self
-                .dataset
-                .open_vector_index(
-                    q.column.as_str(),
-                    &index.uuid.to_string(),
-                    &NoOpMetricsCollector,
-                )
-                .await?;
-            let mut q = q.clone();
-            q.metric_type = Some(idx.metric_type());
+        let fallback_fragments = if let Some(target_fragments) = &self.fragments {
+            let indexed_fragments = self.get_indexed_frags(indexed_segments);
+            target_fragments
+                .iter()
+                .filter(|fragment| !indexed_fragments.contains(fragment.id as u32))
+                .cloned()
+                .collect::<Vec<_>>()
+        } else if self.index_segments.is_some() {
+            Vec::new()
+        } else {
+            self.dataset.unindexed_fragments(index_name).await?
+        };
+
+        if !fallback_fragments.is_empty() {
+            let q = q.clone();
+            debug_assert!(q.metric_type.is_some());
 
             // If the vector column is not present, we need to take the vector column, so
             // that the distance value is comparable with the flat search ones.
@@ -3600,7 +3759,7 @@ impl Scanner {
                 false,
                 false,
                 vector_scan_projection,
-                Arc::new(unindexed_fragments),
+                Arc::new(fallback_fragments),
                 // Can't pushdown limit/offset in an ANN search
                 None,
                 // We are re-ordering anyways, so no need to get data in data
@@ -3894,6 +4053,7 @@ impl Scanner {
             with_row_created_at_version,
             with_make_deletions_null,
             ordered_output: ordered,
+            file_reader_options: self.resolved_file_reader_options(),
         };
         Arc::new(LanceScanExec::new(
             self.dataset.clone(),
@@ -3920,10 +4080,7 @@ impl Scanner {
             with_row_address: self.projection_plan.physical_projection.with_row_addr,
             make_deletions_null,
             ordered_output: self.ordered,
-            file_reader_options: self
-                .file_reader_options
-                .clone()
-                .or_else(|| self.dataset.file_reader_options.clone()),
+            file_reader_options: self.resolved_file_reader_options(),
         };
 
         let fragments = if let Some(fragment) = self.fragments.as_ref() {
@@ -4166,7 +4323,26 @@ impl Scanner {
         if let Some(fragments) = &self.fragments {
             RoaringBitmap::from_iter(fragments.iter().map(|f| f.id as u32))
         } else {
-            RoaringBitmap::from_iter(self.dataset.fragments().iter().map(|f| f.id as u32))
+            self.dataset.fragment_bitmap.as_ref().clone()
+        }
+    }
+
+    fn retain_relevant_index_segments(
+        &self,
+        index_segments: Vec<IndexMetadata>,
+    ) -> Vec<IndexMetadata> {
+        if let Some(fragments) = &self.fragments {
+            let target_fragments = RoaringBitmap::from_iter(fragments.iter().map(|f| f.id as u32));
+            index_segments
+                .into_iter()
+                .filter(|idx| {
+                    idx.fragment_bitmap
+                        .as_ref()
+                        .is_some_and(|fragmap| !(fragmap & &target_fragments).is_empty())
+                })
+                .collect()
+        } else {
+            index_segments
         }
     }
 
@@ -4537,6 +4713,7 @@ pub mod test_dataset {
 
     use arrow_array::{
         ArrayRef, FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator, StringArray,
+        types::Float32Type,
     };
     use arrow_schema::{ArrowError, DataType};
     use lance_arrow::FixedSizeListArrayExt;
@@ -4545,7 +4722,13 @@ pub mod test_dataset {
     use lance_index::{
         IndexType,
         scalar::{ScalarIndexParams, inverted::tokenizer::InvertedIndexParams},
+        vector::{
+            ivf::IvfBuildParams,
+            kmeans::{KMeansParams, train_kmeans},
+        },
     };
+    use lance_linalg::distance::DistanceType;
+    use uuid::Uuid;
 
     use crate::dataset::WriteParams;
     use crate::index::vector::VectorIndexParams;
@@ -4656,6 +4839,63 @@ pub mod test_dataset {
             Ok(())
         }
 
+        pub async fn make_segmented_vector_index(&mut self) -> Result<Vec<Uuid>> {
+            let batch = self
+                .dataset
+                .scan()
+                .project(&["vec"])
+                .unwrap()
+                .try_into_batch()
+                .await?;
+            let vectors = batch
+                .column_by_name("vec")
+                .expect("vector column should exist")
+                .as_fixed_size_list();
+            let values = vectors.values().as_primitive::<Float32Type>();
+            let centroids = train_kmeans::<Float32Type>(
+                values,
+                KMeansParams::new(None, 10, 1, DistanceType::L2),
+                self.dimension as usize,
+                2,
+                2,
+            )
+            .unwrap()
+            .centroids
+            .as_primitive::<Float32Type>()
+            .clone();
+            let centroids = Arc::new(
+                FixedSizeListArray::try_new_from_values(centroids, self.dimension as i32).unwrap(),
+            );
+            let params = VectorIndexParams::with_ivf_flat_params(
+                DistanceType::L2,
+                IvfBuildParams::try_with_centroids(2, centroids).unwrap(),
+            );
+            let fragment_ids = self
+                .dataset
+                .get_fragments()
+                .iter()
+                .map(|fragment| fragment.id() as u32)
+                .collect::<Vec<_>>();
+
+            let mut segments = Vec::with_capacity(fragment_ids.len());
+            for fragment_id in fragment_ids {
+                let mut builder =
+                    self.dataset
+                        .create_index_builder(&["vec"], IndexType::Vector, &params);
+                builder = builder.name("idx".to_string()).fragments(vec![fragment_id]);
+                segments.push(builder.execute_uncommitted().await?);
+            }
+
+            let segment_ids = segments
+                .iter()
+                .map(|segment| segment.uuid)
+                .collect::<Vec<_>>();
+            self.dataset
+                .commit_existing_index_segments("idx", "vec", segments)
+                .await?;
+            Ok(segment_ids)
+        }
+
         pub async fn make_scalar_index(&mut self) -> Result<()> {
             self.dataset
                 .create_index(
@@ -7509,117 +7749,669 @@ mod test {
         .unwrap();
     }
 
-    #[rstest]
     #[tokio::test]
-    async fn test_late_materialization(
-        #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
-        data_storage_version: LanceFileVersion,
-    ) {
-        use lance_io::assert_io_lt;
-        // Create a large dataset with a scalar indexed column and a sorted but not scalar
-        // indexed column
-        use lance_table::io::commit::RenameCommitHandler;
+    async fn test_like_prefix_with_btree_index() {
+        // Create dataset with string data that has various prefixes
+        // Avoid LIKE special characters (%, _) in data to keep tests simple
         let data = gen_batch()
             .col(
-                "vector",
-                array::rand_vec::<Float32Type>(Dimension::from(32)),
+                "name",
+                array::cycle_utf8_literals(&[
+                    "apple",
+                    "application",
+                    "app",
+                    "banana",
+                    "band",
+                    "testns1",
+                    "testns2",
+                    "test",
+                    "testing",
+                    "zoo",
+                ]),
             )
-            .col("indexed", array::step::<Int32Type>())
-            .col("not_indexed", array::step::<Int32Type>())
-            .into_reader_rows(RowCount::from(1000), BatchCount::from(20));
+            .col("id", array::step::<Int32Type>())
+            .into_reader_rows(RowCount::from(100), BatchCount::from(1));
 
-        let mut dataset = Dataset::write(
-            data,
-            "memory://test",
-            Some(WriteParams {
-                commit_handler: Some(Arc::new(RenameCommitHandler)),
-                data_storage_version: Some(data_storage_version),
-                ..Default::default()
-            }),
-        )
-        .await
-        .unwrap();
+        let mut dataset = Dataset::write(data, "memory://test_like", None)
+            .await
+            .unwrap();
+
+        // Create BTree index on string column
         dataset
             .create_index(
-                &["indexed"],
-                IndexType::Scalar,
+                &["name"],
+                IndexType::BTree,
                 None,
                 &ScalarIndexParams::default(),
-                false,
+                true,
             )
             .await
             .unwrap();
 
-        // First run a full scan to get a baseline
-        let _ = dataset.object_store().io_stats_incremental(); // reset
-        dataset.scan().try_into_batch().await.unwrap();
-        let io_stats = dataset.object_store().io_stats_incremental();
-        let full_scan_bytes = io_stats.read_bytes;
+        // Test 1: Verify LIKE 'app%' uses scalar index and returns correct results
+        assert_plan_equals(
+            &dataset,
+            |scanner| scanner.filter("name LIKE 'app%'"),
+            "LanceRead: uri=..., projection=[name, id], num_fragments=1, \
+             range_before=None, range_after=None, row_id=false, row_addr=false, \
+             full_filter=name LIKE Utf8(\"app%\"), refine_filter=--
+               ScalarIndexQuery: query=[name LIKE 'app%']@name_idx",
+        )
+        .await
+        .unwrap();
 
-        // Next do a scan without pushdown, we should still see a benefit from late materialization
-        dataset
+        // Verify correct results for LIKE 'app%'
+        let results = dataset
             .scan()
-            .use_stats(false)
-            .filter("not_indexed = 50")
+            .filter("name LIKE 'app%'")
             .unwrap()
             .try_into_batch()
             .await
             .unwrap();
-        let io_stats = dataset.object_store().io_stats_incremental();
-        assert_io_lt!(io_stats, read_bytes, full_scan_bytes);
-        let filtered_scan_bytes = io_stats.read_bytes;
+        let names: Vec<&str> = results
+            .column_by_name("name")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap()
+            .iter()
+            .map(|s| s.unwrap())
+            .collect();
+        // Should match: apple, application, app (repeated in cycle)
+        assert!(names.iter().all(|n| n.starts_with("app")));
+        assert!(!names.is_empty());
 
-        // Now do a scan with pushdown, the benefit should be even greater
-        // Pushdown only works with the legacy format for now.
-        if data_storage_version == LanceFileVersion::Legacy {
-            dataset
-                .scan()
-                .filter("not_indexed = 50")
-                .unwrap()
-                .try_into_batch()
-                .await
-                .unwrap();
-            let io_stats = dataset.object_store().io_stats_incremental();
-            assert_io_lt!(io_stats, read_bytes, filtered_scan_bytes);
-        }
+        // Test 2: Verify starts_with() uses scalar index (simple prefix without special chars)
+        // Note: DataFusion optimizes starts_with() to LIKE before our index planning
+        assert_plan_equals(
+            &dataset,
+            |scanner| scanner.filter("starts_with(name, 'ban')"),
+            "LanceRead: uri=..., projection=[name, id], num_fragments=1, \
+             range_before=None, range_after=None, row_id=false, row_addr=false, \
+             full_filter=name LIKE Utf8(\"ban%\"), refine_filter=--
+               ScalarIndexQuery: query=[name LIKE 'ban%']@name_idx",
+        )
+        .await
+        .unwrap();
 
-        // Now do a scalar index scan, this should be better than a
-        // full scan but since we have to load the index might be more
-        // expensive than late / pushdown scan
-        dataset
+        // Verify correct results for starts_with
+        let results = dataset
             .scan()
-            .filter("indexed = 50")
+            .filter("starts_with(name, 'ban')")
             .unwrap()
             .try_into_batch()
             .await
             .unwrap();
-        let io_stats = dataset.object_store().io_stats_incremental();
-        assert_io_lt!(io_stats, read_bytes, full_scan_bytes);
-        let index_scan_bytes = io_stats.read_bytes;
+        let names: Vec<&str> = results
+            .column_by_name("name")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap()
+            .iter()
+            .map(|s| s.unwrap())
+            .collect();
+        // Should match: banana, band
+        assert!(names.iter().all(|n| n.starts_with("ban")));
+        assert!(!names.is_empty());
 
-        // A second scalar index scan should be cheaper than the first
-        // since we should have the index in cache
-        dataset
+        // Test 3: LIKE with pattern requiring refine (e.g., 'test%2')
+        assert_plan_equals(
+            &dataset,
+            |scanner| scanner.filter("name LIKE 'test%2'"),
+            "ProjectionExec: expr=[name@0 as name, id@1 as id]
+  LanceRead: uri=..., projection=[name, id], num_fragments=1, \
+range_before=None, range_after=None, row_id=true, row_addr=false, \
+full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
+    ScalarIndexQuery: query=[name LIKE 'test%']@name_idx",
+        )
+        .await
+        .unwrap();
+
+        // Verify correct results for LIKE 'test%2' (needs refine)
+        let results = dataset
             .scan()
-            .filter("indexed = 50")
+            .filter("name LIKE 'test%2'")
             .unwrap()
             .try_into_batch()
             .await
             .unwrap();
-        let io_stats = dataset.object_store().io_stats_incremental();
-        assert_io_lt!(io_stats, read_bytes, index_scan_bytes);
+        let names: Vec<&str> = results
+            .column_by_name("name")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap()
+            .iter()
+            .map(|s| s.unwrap())
+            .collect();
+        // Should match: testns2 (ends with '2')
+        assert!(
+            names
+                .iter()
+                .all(|n| n.starts_with("test") && n.ends_with("2"))
+        );
+
+        // Test 4: LIKE starting with wildcard should NOT use scalar index for pruning
+        // Verify by checking the plan does NOT have ScalarIndexQuery
+        let mut scanner = dataset.scan();
+        scanner.filter("name LIKE '%app%'").unwrap();
+        let plan = scanner.create_plan().await.unwrap();
+        let plan_str = format!("{:?}", plan);
+        assert!(
+            !plan_str.contains("ScalarIndexQuery"),
+            "LIKE '%app%' should not use scalar index, but got: {}",
+            plan_str
+        );
+
+        // Verify correct results for LIKE '%app%'
+        let results = dataset
+            .scan()
+            .filter("name LIKE '%app%'")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let names: Vec<&str> = results
+            .column_by_name("name")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap()
+            .iter()
+            .map(|s| s.unwrap())
+            .collect();
+        // Should match: apple, application, app (contain 'app')
+        assert!(names.iter().all(|n| n.contains("app")));
+
+        // Test 5: NOT LIKE should NOT use scalar index
+        let mut scanner = dataset.scan();
+        scanner.filter("name NOT LIKE 'app%'").unwrap();
+        let plan = scanner.create_plan().await.unwrap();
+        let plan_str = format!("{:?}", plan);
+        assert!(
+            !plan_str.contains("ScalarIndexQuery"),
+            "NOT LIKE should not use scalar index, but got: {}",
+            plan_str
+        );
     }
 
-    #[rstest]
     #[tokio::test]
-    async fn test_project_nested(
-        #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
-        data_storage_version: LanceFileVersion,
-    ) -> Result<()> {
-        let struct_i_field = ArrowField::new("i", DataType::Int32, true);
-        let struct_o_field = ArrowField::new("o", DataType::Utf8, true);
-        let schema = Arc::new(ArrowSchema::new(vec![
-            ArrowField::new(
+    async fn test_like_prefix_correctness_with_btree_index() {
+        // Create dataset with deterministic string data for exact result verification
+        let names: Vec<&str> = vec![
+            "alpha", "alphabet", "beta", "gamma", "delta", "epsilon", "eta", "theta", "iota",
+            "kappa",
+        ];
+        let data = RecordBatch::try_new(
+            Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("name", DataType::Utf8, false),
+                ArrowField::new("id", DataType::Int32, false),
+            ])),
+            vec![
+                Arc::new(StringArray::from(names.clone())),
+                Arc::new(Int32Array::from_iter_values(0..10)),
+            ],
+        )
+        .unwrap();
+
+        let reader = RecordBatchIterator::new(
+            vec![Ok(data)],
+            Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("name", DataType::Utf8, false),
+                ArrowField::new("id", DataType::Int32, false),
+            ])),
+        );
+
+        let mut dataset = Dataset::write(reader, "memory://test_like_correctness", None)
+            .await
+            .unwrap();
+
+        // Create BTree index
+        dataset
+            .create_index(
+                &["name"],
+                IndexType::BTree,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Test with index
+        let with_index = dataset
+            .scan()
+            .filter("name LIKE 'alpha%'")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+
+        // Test without index (for comparison)
+        let without_index = dataset
+            .scan()
+            .use_scalar_index(false)
+            .filter("name LIKE 'alpha%'")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+
+        // Both should return same results: alpha, alphabet
+        assert_eq!(with_index.num_rows(), without_index.num_rows());
+        assert_eq!(with_index.num_rows(), 2);
+
+        let with_index_names: BTreeSet<String> = with_index
+            .column_by_name("name")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap()
+            .iter()
+            .map(|s| s.unwrap().to_string())
+            .collect();
+
+        let without_index_names: BTreeSet<String> = without_index
+            .column_by_name("name")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap()
+            .iter()
+            .map(|s| s.unwrap().to_string())
+            .collect();
+
+        assert_eq!(with_index_names, without_index_names);
+        assert_eq!(
+            with_index_names,
+            BTreeSet::from(["alpha".to_string(), "alphabet".to_string()])
+        );
+
+        // Test starts_with correctness
+        let starts_with_result = dataset
+            .scan()
+            .filter("starts_with(name, 'e')")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+
+        let starts_with_names: BTreeSet<String> = starts_with_result
+            .column_by_name("name")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap()
+            .iter()
+            .map(|s| s.unwrap().to_string())
+            .collect();
+
+        // Should match: epsilon, eta
+        assert_eq!(
+            starts_with_names,
+            BTreeSet::from(["epsilon".to_string(), "eta".to_string()])
+        );
+    }
+
+    #[tokio::test]
+    async fn test_like_prefix_with_zone_map() {
+        use lance_index::scalar::BuiltinIndexType;
+
+        // Create dataset with string data that has various prefixes
+        let data = gen_batch()
+            .col(
+                "name",
+                array::cycle_utf8_literals(&[
+                    "apple",
+                    "application",
+                    "app",
+                    "banana",
+                    "band",
+                    "testns1",
+                    "testns2",
+                    "test",
+                    "testing",
+                    "zoo",
+                ]),
+            )
+            .col("id", array::step::<Int32Type>())
+            .into_reader_rows(RowCount::from(100), BatchCount::from(1));
+
+        let mut dataset = Dataset::write(data, "memory://test_like_zonemap", None)
+            .await
+            .unwrap();
+
+        // Create ZoneMap index on string column
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap);
+        dataset
+            .create_index(
+                &["name"],
+                IndexType::Scalar,
+                Some("name_zonemap".to_string()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Test 1: Verify LIKE 'app%' uses zone map index
+        let mut scanner = dataset.scan();
+        scanner.filter("name LIKE 'app%'").unwrap();
+        let plan = scanner.create_plan().await.unwrap();
+        let plan_str = format!("{:?}", plan);
+        // Zone map uses ScalarIndexExec with LikePrefix query
+        assert!(
+            plan_str.contains("ScalarIndexExec") && plan_str.contains("LikePrefix"),
+            "LIKE 'app%' should use zone map index with LikePrefix, but got: {}",
+            plan_str
+        );
+
+        // Verify correct results for LIKE 'app%'
+        let results = dataset
+            .scan()
+            .filter("name LIKE 'app%'")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let names: Vec<&str> = results
+            .column_by_name("name")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap()
+            .iter()
+            .map(|s| s.unwrap())
+            .collect();
+        assert!(names.iter().all(|n| n.starts_with("app")));
+        assert!(!names.is_empty());
+
+        // Test 2: Verify starts_with() uses zone map index
+        let mut scanner = dataset.scan();
+        scanner.filter("starts_with(name, 'ban')").unwrap();
+        let plan = scanner.create_plan().await.unwrap();
+        let plan_str = format!("{:?}", plan);
+        assert!(
+            plan_str.contains("ScalarIndexExec") && plan_str.contains("LikePrefix"),
+            "starts_with should use zone map index with LikePrefix, but got: {}",
+            plan_str
+        );
+
+        // Verify correct results
+        let results = dataset
+            .scan()
+            .filter("starts_with(name, 'ban')")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let names: Vec<&str> = results
+            .column_by_name("name")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap()
+            .iter()
+            .map(|s| s.unwrap())
+            .collect();
+        assert!(names.iter().all(|n| n.starts_with("ban")));
+
+        // Test 3: LIKE with refine pattern still uses zone map for prefix pruning
+        let mut scanner = dataset.scan();
+        scanner.filter("name LIKE 'test%2'").unwrap();
+        let plan = scanner.create_plan().await.unwrap();
+        let plan_str = format!("{:?}", plan);
+        assert!(
+            plan_str.contains("ScalarIndexExec") && plan_str.contains("LikePrefix"),
+            "LIKE 'test%2' should use zone map index for prefix, but got: {}",
+            plan_str
+        );
+
+        // Test 4: LIKE starting with wildcard should NOT use zone map
+        let mut scanner = dataset.scan();
+        scanner.filter("name LIKE '%app%'").unwrap();
+        let plan = scanner.create_plan().await.unwrap();
+        let plan_str = format!("{:?}", plan);
+        assert!(
+            !plan_str.contains("LikePrefix"),
+            "LIKE '%app%' should not use LikePrefix index, but got: {}",
+            plan_str
+        );
+    }
+
+    #[tokio::test]
+    async fn test_like_prefix_correctness_with_zone_map() {
+        use lance_index::scalar::BuiltinIndexType;
+
+        // Create dataset with deterministic string data for exact result verification
+        let names: Vec<&str> = vec![
+            "alpha", "alphabet", "beta", "gamma", "delta", "epsilon", "eta", "theta", "iota",
+            "kappa",
+        ];
+        let data = RecordBatch::try_new(
+            Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("name", DataType::Utf8, false),
+                ArrowField::new("id", DataType::Int32, false),
+            ])),
+            vec![
+                Arc::new(StringArray::from(names.clone())),
+                Arc::new(Int32Array::from_iter_values(0..10)),
+            ],
+        )
+        .unwrap();
+
+        let reader = RecordBatchIterator::new(
+            vec![Ok(data)],
+            Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("name", DataType::Utf8, false),
+                ArrowField::new("id", DataType::Int32, false),
+            ])),
+        );
+
+        let mut dataset = Dataset::write(reader, "memory://test_like_correctness_zonemap", None)
+            .await
+            .unwrap();
+
+        // Create ZoneMap index
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap);
+        dataset
+            .create_index(
+                &["name"],
+                IndexType::Scalar,
+                Some("name_zonemap".to_string()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Test with zone map index
+        let with_index = dataset
+            .scan()
+            .filter("name LIKE 'alpha%'")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+
+        // Test without index (for comparison)
+        let without_index = dataset
+            .scan()
+            .use_scalar_index(false)
+            .filter("name LIKE 'alpha%'")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+
+        // Both should return same results: alpha, alphabet
+        assert_eq!(with_index.num_rows(), without_index.num_rows());
+        assert_eq!(with_index.num_rows(), 2);
+
+        let with_index_names: BTreeSet<String> = with_index
+            .column_by_name("name")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap()
+            .iter()
+            .map(|s| s.unwrap().to_string())
+            .collect();
+
+        let without_index_names: BTreeSet<String> = without_index
+            .column_by_name("name")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap()
+            .iter()
+            .map(|s| s.unwrap().to_string())
+            .collect();
+
+        assert_eq!(with_index_names, without_index_names);
+        assert_eq!(
+            with_index_names,
+            BTreeSet::from(["alpha".to_string(), "alphabet".to_string()])
+        );
+
+        // Test starts_with correctness with zone map
+        let starts_with_result = dataset
+            .scan()
+            .filter("starts_with(name, 'e')")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+
+        let starts_with_names: BTreeSet<String> = starts_with_result
+            .column_by_name("name")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap()
+            .iter()
+            .map(|s| s.unwrap().to_string())
+            .collect();
+
+        // Should match: epsilon, eta
+        assert_eq!(
+            starts_with_names,
+            BTreeSet::from(["epsilon".to_string(), "eta".to_string()])
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn test_late_materialization(
+        #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
+        data_storage_version: LanceFileVersion,
+    ) {
+        use lance_io::assert_io_lt;
+        // Create a large dataset with a scalar indexed column and a sorted but not scalar
+        // indexed column
+        use lance_table::io::commit::RenameCommitHandler;
+        let data = gen_batch()
+            .col(
+                "vector",
+                array::rand_vec::<Float32Type>(Dimension::from(32)),
+            )
+            .col("indexed", array::step::<Int32Type>())
+            .col("not_indexed", array::step::<Int32Type>())
+            .into_reader_rows(RowCount::from(1000), BatchCount::from(20));
+
+        let mut dataset = Dataset::write(
+            data,
+            "memory://test",
+            Some(WriteParams {
+                commit_handler: Some(Arc::new(RenameCommitHandler)),
+                data_storage_version: Some(data_storage_version),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        dataset
+            .create_index(
+                &["indexed"],
+                IndexType::Scalar,
+                None,
+                &ScalarIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+
+        // First run a full scan to get a baseline
+        let _ = dataset.object_store().io_stats_incremental(); // reset
+        dataset.scan().try_into_batch().await.unwrap();
+        let io_stats = dataset.object_store().io_stats_incremental();
+        let full_scan_bytes = io_stats.read_bytes;
+
+        // Next do a scan without pushdown, we should still see a benefit from late materialization
+        dataset
+            .scan()
+            .use_stats(false)
+            .filter("not_indexed = 50")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let io_stats = dataset.object_store().io_stats_incremental();
+        assert_io_lt!(io_stats, read_bytes, full_scan_bytes);
+        let filtered_scan_bytes = io_stats.read_bytes;
+
+        // Now do a scan with pushdown, the benefit should be even greater
+        // Pushdown only works with the legacy format for now.
+        if data_storage_version == LanceFileVersion::Legacy {
+            dataset
+                .scan()
+                .filter("not_indexed = 50")
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            let io_stats = dataset.object_store().io_stats_incremental();
+            assert_io_lt!(io_stats, read_bytes, filtered_scan_bytes);
+        }
+
+        // Now do a scalar index scan, this should be better than a
+        // full scan but since we have to load the index might be more
+        // expensive than late / pushdown scan
+        dataset
+            .scan()
+            .filter("indexed = 50")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let io_stats = dataset.object_store().io_stats_incremental();
+        assert_io_lt!(io_stats, read_bytes, full_scan_bytes);
+        let index_scan_bytes = io_stats.read_bytes;
+
+        // A second scalar index scan should be cheaper than the first
+        // since we should have the index in cache
+        dataset
+            .scan()
+            .filter("indexed = 50")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let io_stats = dataset.object_store().io_stats_incremental();
+        assert_io_lt!(io_stats, read_bytes, index_scan_bytes);
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn test_project_nested(
+        #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
+        data_storage_version: LanceFileVersion,
+    ) -> Result<()> {
+        let struct_i_field = ArrowField::new("i", DataType::Int32, true);
+        let struct_o_field = ArrowField::new("o", DataType::Utf8, true);
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new(
                 "struct",
                 DataType::Struct(vec![struct_i_field.clone(), struct_o_field.clone()].into()),
                 true,
@@ -9700,8 +10492,8 @@ mod test {
             .await
             .unwrap();
 
-        // Create index on first 2 fragments
-        test_ds.make_vector_index().await.unwrap();
+        // Create one segment per indexed fragment so fragment filtering must prune ANN fan-out.
+        test_ds.make_segmented_vector_index().await.unwrap();
 
         let query: Float32Array = (0..32).map(|v| v as f32).collect();
 
@@ -9722,6 +10514,207 @@ mod test {
         .await;
     }
 
+    #[tokio::test]
+    async fn test_vector_search_fragment_filter_prunes_segment_fanout() {
+        let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false)
+            .await
+            .unwrap();
+        test_ds.make_segmented_vector_index().await.unwrap();
+
+        let query: Float32Array = (0..32).map(|v| v as f32).collect();
+        test_ds.append_data_with_range(400, 410).await.unwrap();
+        test_ds.append_data_with_range(410, 420).await.unwrap();
+        let fragments = test_ds.dataset.fragments();
+
+        let mut scanner = test_ds.dataset.scan();
+        scanner.nearest("vec", &query, 420).unwrap();
+        let full_plan = scanner.explain_plan(true).await.unwrap();
+        assert!(
+            full_plan.contains("ANNSubIndex: name=idx, k=420, deltas=2, metric=L2"),
+            "expected two ANN deltas without fragment filter, plan was:\n{full_plan}"
+        );
+
+        let mut scanner = test_ds.dataset.scan();
+        scanner
+            .nearest("vec", &query, 420)
+            .unwrap()
+            .with_fragments(vec![fragments[0].clone()]);
+        let filtered_plan = scanner.explain_plan(true).await.unwrap();
+        assert!(
+            filtered_plan.contains("ANNSubIndex: name=idx, k=420, deltas=1, metric=L2"),
+            "expected one ANN delta with fragment filter, plan was:\n{filtered_plan}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_search_respects_index_segments() {
+        let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false)
+            .await
+            .unwrap();
+        let segment_ids = test_ds.make_segmented_vector_index().await.unwrap();
+
+        let query: Float32Array = (0..32).map(|v| v as f32).collect();
+        test_ds.append_data_with_range(400, 410).await.unwrap();
+        test_ds.append_data_with_range(410, 420).await.unwrap();
+
+        let mut scanner = test_ds.dataset.scan();
+        scanner
+            .nearest("vec", &query, 420)
+            .unwrap()
+            .with_index_segments(vec![segment_ids[0]])
+            .unwrap();
+        let batch = scanner.try_into_batch().await.unwrap();
+        let i_array = batch
+            .column_by_name("i")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        assert_eq!(batch.num_rows(), 200);
+        assert_values_in_range(
+            i_array,
+            0..200,
+            "Should only get results from the selected index segment",
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_search_intersects_fragments_and_index_segments() {
+        let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false)
+            .await
+            .unwrap();
+        let segment_ids = test_ds.make_segmented_vector_index().await.unwrap();
+
+        let query: Float32Array = (0..32).map(|v| v as f32).collect();
+        test_ds.append_data_with_range(400, 410).await.unwrap();
+        test_ds.append_data_with_range(410, 420).await.unwrap();
+        let fragments = test_ds.dataset.fragments();
+
+        let mut scanner = test_ds.dataset.scan();
+        scanner
+            .nearest("vec", &query, 420)
+            .unwrap()
+            .with_fragments(vec![fragments[0].clone(), fragments[2].clone()])
+            .with_index_segments(vec![segment_ids[0]])
+            .unwrap();
+        let batch = scanner.try_into_batch().await.unwrap();
+        let i_array = batch
+            .column_by_name("i")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        assert!(
+            i_array
+                .iter()
+                .all(|v| v.is_some_and(|val| (0..200).contains(&val) || (400..410).contains(&val)))
+                && i_array
+                    .iter()
+                    .any(|v| v.is_some_and(|val| (0..200).contains(&val)))
+                && i_array
+                    .iter()
+                    .any(|v| v.is_some_and(|val| (400..410).contains(&val))),
+            "Should get selected segment rows plus flat fallback for target fragments outside the selected segments"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_search_rejects_unknown_index_segment() {
+        let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false)
+            .await
+            .unwrap();
+        test_ds.make_segmented_vector_index().await.unwrap();
+
+        let query: Float32Array = (0..32).map(|v| v as f32).collect();
+        let err = test_ds
+            .dataset
+            .scan()
+            .nearest("vec", &query, 10)
+            .unwrap()
+            .with_index_segments(vec![Uuid::new_v4()])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string().contains("unknown index segments"),
+            "unexpected error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_search_rejects_metric_mismatch_for_index_segments() {
+        let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false)
+            .await
+            .unwrap();
+        let segment_ids = test_ds.make_segmented_vector_index().await.unwrap();
+
+        let query: Float32Array = (0..32).map(|v| v as f32).collect();
+        let err = test_ds
+            .dataset
+            .scan()
+            .nearest("vec", &query, 10)
+            .unwrap()
+            .distance_metric(DistanceType::Dot)
+            .with_index_segments(vec![segment_ids[0]])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("with_index_segments requested metric"),
+            "unexpected error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_with_index_segments_rejects_empty_list() {
+        let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false)
+            .await
+            .unwrap();
+        let query: Float32Array = (0..32).map(|v| v as f32).collect();
+
+        let Err(err) = test_ds
+            .dataset
+            .scan()
+            .nearest("vec", &query, 10)
+            .unwrap()
+            .with_index_segments(vec![])
+        else {
+            panic!("expected empty index segments to be rejected");
+        };
+        assert!(
+            err.to_string()
+                .contains("with_index_segments does not accept an empty segment list"),
+            "unexpected error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_with_index_segments_rejected_for_non_vector_query() {
+        let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false)
+            .await
+            .unwrap();
+        let segment_ids = test_ds.make_segmented_vector_index().await.unwrap();
+
+        let err = test_ds
+            .dataset
+            .scan()
+            .project(&["i"])
+            .unwrap()
+            .with_index_segments(vec![segment_ids[0]])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("with_index_segments is only supported for vector search"),
+            "unexpected error: {err}"
+        );
+    }
+
     #[tokio::test]
     async fn test_fts_respects_fragment_list() {
         let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false)
diff --git a/rust/lance/src/dataset/schema_evolution.rs b/rust/lance/src/dataset/schema_evolution.rs
index b79cb283956..f5d792979df 100644
--- a/rust/lance/src/dataset/schema_evolution.rs
+++ b/rust/lance/src/dataset/schema_evolution.rs
@@ -12,6 +12,7 @@ use crate::{Error, Result, io::exec::Planner};
 use arrow::compute::CastOptions;
 use arrow::compute::can_cast_types;
 use arrow_array::{Array, RecordBatch, RecordBatchReader};
+use arrow_cast::cast_with_options;
 use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
 use datafusion::execution::SendableRecordBatchStream;
 use futures::stream::{StreamExt, TryStreamExt};
@@ -635,7 +636,7 @@ pub(super) async fn alter_columns(
             let mut columns = Vec::with_capacity(batch.num_columns());
             for (old, new) in &cast_fields {
                 let old_column = batch[&old.name].clone();
-                let new_column = lance_arrow::cast::cast_with_options(
+                let new_column = cast_with_options(
                     &old_column,
                     &new.data_type(),
                     // Safe: false means it will error if the cast is lossy.
@@ -1783,11 +1784,12 @@ mod test {
     ) -> Result<()> {
         // Create a table with 2 scalar columns, 1 vector column
 
+        use crate::index::DatasetIndexExt;
         use arrow::datatypes::{Int32Type, Int64Type};
         use arrow_array::{Float16Array, Float32Array, Int64Array, ListArray};
         use half::f16;
         use lance_arrow::FixedSizeListArrayExt;
-        use lance_index::{DatasetIndexExt, IndexType, scalar::ScalarIndexParams};
+        use lance_index::{IndexType, scalar::ScalarIndexParams};
         use lance_linalg::distance::MetricType;
         use lance_testing::datagen::generate_random_array;
 
@@ -1962,7 +1964,7 @@ mod test {
                 Arc::new(Float16Array::from_iter_values(
                     (0..nrows).map(|i| f16::from_f32(i as f32)),
                 )),
-                lance_arrow::cast::cast_with_options(
+                cast_with_options(
                     batch["vec"].as_ref(),
                     &DataType::FixedSizeList(
                         Arc::new(ArrowField::new("item", DataType::Float16, true)),
diff --git a/rust/lance/src/dataset/take.rs b/rust/lance/src/dataset/take.rs
index 73625a171e0..f304f732978 100644
--- a/rust/lance/src/dataset/take.rs
+++ b/rust/lance/src/dataset/take.rs
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::{collections::BTreeMap, ops::Range, pin::Pin, sync::Arc};
+use std::{collections::BTreeMap, collections::HashMap, ops::Range, pin::Pin, sync::Arc};
 
 use crate::dataset::fragment::FragReadConfig;
 use crate::dataset::rowids::get_row_id_index;
@@ -43,12 +43,10 @@ use super::{Dataset, fragment::FileFragment, scanner::DatasetRecordBatchStream};
 /// to the address (1, 10), assuming the second fragment starts with 10 undeleted rows.
 ///
 /// If any offsets are beyond the end of the dataset, they will be mapped to a tombstone row address.
-pub(super) async fn row_offsets_to_row_addresses(
-    dataset: &Dataset,
+pub async fn row_offsets_to_row_addresses(
+    fragments: &[FileFragment],
     row_indices: &[u64],
 ) -> Result<Vec<u64>> {
-    let fragments = dataset.get_fragments();
-
     let mut perm = permutation::sort(row_indices);
     let sorted_offsets = perm.apply_slice(row_indices);
 
@@ -115,7 +113,8 @@ pub async fn take(
     }
 
     // First, convert the dataset offsets into row addresses
-    let addrs = row_offsets_to_row_addresses(dataset, offsets).await?;
+    let fragments = dataset.get_fragments();
+    let addrs = row_offsets_to_row_addresses(&fragments, offsets).await?;
 
     let builder = TakeBuilder::try_new_from_addresses(
         Arc::new(dataset.clone()),
@@ -334,14 +333,14 @@ async fn do_take_rows(
             .as_primitive::<UInt64Type>()
             .values();
 
+        let addr_to_pos: HashMap<u64, u64> = returned_row_addr
+            .iter()
+            .enumerate()
+            .map(|(i, addr)| (*addr, i as u64))
+            .collect();
         let remapping_index: UInt64Array = row_addrs
             .iter()
-            .filter_map(|o| {
-                returned_row_addr
-                    .iter()
-                    .position(|id| id == o)
-                    .map(|pos| pos as u64)
-            })
+            .filter_map(|o| addr_to_pos.get(o).copied())
             .collect();
 
         // remapping_index may be greater than the number of rows in one_batch
@@ -374,14 +373,18 @@ async fn do_take_rows(
                 AddRowOffsetExec::compute_row_offset_array(&row_addr_col, builder.dataset).await?;
             let row_offset_field =
                 ArrowField::new(ROW_OFFSET, arrow::datatypes::DataType::UInt64, false);
-            batch = batch.try_with_column(row_offset_field, row_offset_col)?;
+            if batch.schema().column_with_name(ROW_OFFSET).is_none() {
+                batch = batch.try_with_column(row_offset_field, row_offset_col)?;
+            }
         }
 
         if builder.with_row_address {
             // inject `ROW_ADDR` column
             let row_addr_field =
                 ArrowField::new(ROW_ADDR, arrow::datatypes::DataType::UInt64, false);
-            batch = batch.try_with_column(row_addr_field, row_addr_col)?;
+            if batch.schema().column_with_name(ROW_ADDR).is_none() {
+                batch = batch.try_with_column(row_addr_field, row_addr_col)?;
+            }
         }
     }
 
@@ -450,7 +453,7 @@ fn check_row_addrs(row_addrs: &[u64]) -> RowAddressStats {
     let first_fragment_id = row_addrs[0] >> 32;
 
     for addr in row_addrs.iter().skip(1) {
-        sorted &= *addr > last_offset;
+        sorted &= *addr >= last_offset;
         contiguous &= *addr == last_offset + 1;
         // Contiguous also requires the fragment ids are all the same
         contiguous &= (*addr >> 32) == first_fragment_id;
@@ -825,6 +828,46 @@ mod test {
         assert_eq!(struct_arr.fields()[4].name(), "blob_uri");
     }
 
+    #[tokio::test]
+    async fn test_projection_plan_accepts_unloaded_legacy_blob_schema() {
+        let mut metadata = HashMap::new();
+        metadata.insert(lance_arrow::BLOB_META_KEY.to_string(), "true".to_string());
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("blob", DataType::LargeBinary, true).with_metadata(metadata),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(LargeBinaryArray::from(vec![Some(
+                b"hello".as_slice(),
+            )]))],
+        )
+        .unwrap();
+        let write_params = WriteParams {
+            data_storage_version: Some(LanceFileVersion::Legacy),
+            ..Default::default()
+        };
+        let batches = RecordBatchIterator::new([Ok(batch)], schema);
+        let dataset = Dataset::write(batches, "memory://", Some(write_params))
+            .await
+            .unwrap();
+
+        let mut projection = dataset.schema().project(&["blob"]).unwrap();
+        projection.fields[0].unloaded_mut();
+
+        let projection = ProjectionRequest::from_schema(projection)
+            .into_projection_plan(Arc::new(dataset))
+            .unwrap();
+
+        let output_schema = projection.output_schema().unwrap();
+        let blob_field = output_schema.field_with_name("blob").unwrap();
+        let DataType::Struct(fields) = blob_field.data_type() else {
+            panic!("expected blob output schema to be a struct, got {blob_field:?}");
+        };
+        assert_eq!(fields.len(), 2);
+        assert_eq!(fields[0].name(), "position");
+        assert_eq!(fields[1].name(), "size");
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_take_rowid_rowaddr_with_projection_enable_stable_row_ids_projection_from_sql(
diff --git a/rust/lance/src/dataset/tests/dataset_aggregate.rs b/rust/lance/src/dataset/tests/dataset_aggregate.rs
index cdc954a9997..ef2a90e6315 100644
--- a/rust/lance/src/dataset/tests/dataset_aggregate.rs
+++ b/rust/lance/src/dataset/tests/dataset_aggregate.rs
@@ -38,12 +38,13 @@ use tempfile::tempdir;
 
 use crate::Dataset;
 use crate::dataset::scanner::AggregateExpr;
+use crate::index::DatasetIndexExt;
 use crate::index::vector::VectorIndexParams;
 use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount, assert_plan_node_equals};
 use lance_arrow::FixedSizeListArrayExt;
+use lance_index::IndexType;
 use lance_index::scalar::FullTextSearchQuery;
 use lance_index::scalar::inverted::InvertedIndexParams;
-use lance_index::{DatasetIndexExt, IndexType};
 use lance_linalg::distance::MetricType;
 
 /// Helper to create a field reference expression for a column index
diff --git a/rust/lance/src/dataset/tests/dataset_concurrency_store.rs b/rust/lance/src/dataset/tests/dataset_concurrency_store.rs
index 7ce57a4c0f4..a9c2aa44c38 100644
--- a/rust/lance/src/dataset/tests/dataset_concurrency_store.rs
+++ b/rust/lance/src/dataset/tests/dataset_concurrency_store.rs
@@ -8,12 +8,12 @@ use crate::dataset::WriteDestination;
 use crate::{Dataset, Error, Result};
 
 use crate::dataset::write::{WriteMode, WriteParams};
+use crate::index::DatasetIndexExt;
 use arrow_array::RecordBatch;
 use arrow_array::{Int32Array, RecordBatchIterator};
 use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
 use futures::TryStreamExt;
 use lance_core::utils::tempfile::TempStrDir;
-use lance_index::DatasetIndexExt;
 use lance_index::{IndexType, scalar::ScalarIndexParams};
 
 #[tokio::test]
diff --git a/rust/lance/src/dataset/tests/dataset_geo.rs b/rust/lance/src/dataset/tests/dataset_geo.rs
index dc1e79dc455..a43718dd7d4 100644
--- a/rust/lance/src/dataset/tests/dataset_geo.rs
+++ b/rust/lance/src/dataset/tests/dataset_geo.rs
@@ -7,6 +7,7 @@ use std::vec;
 use crate::Dataset;
 use crate::dataset::tests::dataset_transactions::execute_sql;
 
+use crate::index::DatasetIndexExt;
 use arrow_array::RecordBatch;
 use arrow_array::RecordBatchIterator;
 use arrow_array::cast::AsArray;
@@ -19,8 +20,8 @@ use geoarrow_array::{
 };
 use geoarrow_schema::{Dimension, LineStringType, PointType, PolygonType};
 use lance_core::utils::tempfile::TempStrDir;
+use lance_index::IndexType;
 use lance_index::scalar::ScalarIndexParams;
-use lance_index::{DatasetIndexExt, IndexType};
 
 #[tokio::test]
 async fn test_geo_types() {
diff --git a/rust/lance/src/dataset/tests/dataset_index.rs b/rust/lance/src/dataset/tests/dataset_index.rs
index 562252ea8e3..10a3203e669 100644
--- a/rust/lance/src/dataset/tests/dataset_index.rs
+++ b/rust/lance/src/dataset/tests/dataset_index.rs
@@ -6,13 +6,16 @@ use std::sync::Arc;
 use std::vec;
 
 use crate::dataset::ROW_ID;
+use crate::dataset::builder::DatasetBuilder;
 use crate::dataset::tests::dataset_migrations::scan_dataset;
 use crate::dataset::tests::dataset_transactions::{assert_results, execute_sql};
 use crate::index::vector::VectorIndexParams;
+use crate::session::Session;
 use crate::{Dataset, Error, Result};
 use lance_arrow::FixedSizeListArrayExt;
 
 use crate::dataset::write::{WriteMode, WriteParams};
+use crate::index::DatasetIndexExt;
 use arrow::array::{AsArray, GenericListBuilder, GenericStringBuilder};
 use arrow::datatypes::UInt64Type;
 use arrow_array::RecordBatch;
@@ -26,16 +29,20 @@ use arrow_schema::{
     DataType, Field as ArrowField, Field, Fields as ArrowFields, Schema as ArrowSchema,
 };
 use lance_arrow::ARROW_EXT_NAME_KEY;
+use lance_core::cache::LanceCache;
 use lance_core::utils::tempfile::TempStrDir;
 use lance_datagen::{BatchCount, Dimension, RowCount, array, gen_batch};
+use lance_file::reader::{FileReader, FileReaderOptions};
 use lance_file::version::LanceFileVersion;
-use lance_index::DatasetIndexExt;
 use lance_index::scalar::FullTextSearchQuery;
 use lance_index::scalar::inverted::{
     query::{BooleanQuery, MatchQuery, Occur, Operator, PhraseQuery},
     tokenizer::InvertedIndexParams,
 };
+use lance_index::{FtsPrewarmOptions, PrewarmOptions};
 use lance_index::{IndexType, scalar::ScalarIndexParams, vector::DIST_COL};
+use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
+use lance_io::utils::CachedFileSize;
 use lance_linalg::distance::MetricType;
 
 use datafusion::common::{assert_contains, assert_not_contains};
@@ -1772,6 +1779,300 @@ async fn test_fts_phrase_query() {
     assert_eq!(result.num_rows(), 0);
 }
 
+async fn open_dataset_with_fresh_session(uri: &str) -> Dataset {
+    DatasetBuilder::from_uri(uri)
+        .with_session(Arc::new(Session::new(1 << 20, 1 << 20, Default::default())))
+        .load()
+        .await
+        .unwrap()
+}
+
+#[tokio::test]
+async fn test_fts_prewarm_with_position_controls_phrase_query_cache() {
+    let tmpdir = TempStrDir::default();
+    let uri = tmpdir.to_owned();
+    drop(tmpdir);
+
+    let doc_col: Arc<dyn Array> = Arc::new(GenericStringArray::<i32>::from(vec![
+        "lance search",
+        "lance search with tail",
+        "phrase query",
+    ]));
+    let ids = UInt64Array::from_iter_values(0..doc_col.len() as u64);
+    let batch = RecordBatch::try_new(
+        arrow_schema::Schema::new(vec![
+            arrow_schema::Field::new("doc", doc_col.data_type().to_owned(), true),
+            arrow_schema::Field::new("id", DataType::UInt64, false),
+        ])
+        .into(),
+        vec![Arc::new(doc_col) as ArrayRef, Arc::new(ids) as ArrayRef],
+    )
+    .unwrap();
+    let schema = batch.schema();
+    let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    let mut dataset = Dataset::write(batches, &uri, None).await.unwrap();
+    dataset
+        .create_index(
+            &["doc"],
+            IndexType::Inverted,
+            Some("fts_idx".to_owned()),
+            &InvertedIndexParams::default().with_position(true),
+            true,
+        )
+        .await
+        .unwrap();
+
+    let dataset = open_dataset_with_fresh_session(&uri).await;
+    dataset.prewarm_index("fts_idx").await.unwrap();
+    let cache_entries_after_prewarm = dataset.index_cache_entry_count().await;
+    let result = dataset
+        .scan()
+        .project(&["id"])
+        .unwrap()
+        .full_text_search(FullTextSearchQuery::new_query(
+            PhraseQuery::new("lance search".to_owned()).into(),
+        ))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(result.num_rows(), 2);
+    let cache_entries_after_query = dataset.index_cache_entry_count().await;
+    assert!(
+        cache_entries_after_query > cache_entries_after_prewarm,
+        "phrase query should populate positions cache when prewarm skipped positions"
+    );
+
+    let dataset = open_dataset_with_fresh_session(&uri).await;
+    dataset
+        .prewarm_index_with_options(
+            "fts_idx",
+            &PrewarmOptions::Fts(FtsPrewarmOptions::new().with_position(true)),
+        )
+        .await
+        .unwrap();
+    let cache_entries_after_prewarm = dataset.index_cache_entry_count().await;
+    let result = dataset
+        .scan()
+        .project(&["id"])
+        .unwrap()
+        .full_text_search(FullTextSearchQuery::new_query(
+            PhraseQuery::new("lance search".to_owned()).into(),
+        ))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(result.num_rows(), 2);
+    let cache_entries_after_query = dataset.index_cache_entry_count().await;
+    assert_eq!(
+        cache_entries_after_query, cache_entries_after_prewarm,
+        "phrase query should not add cache entries after prewarming positions"
+    );
+}
+
+#[tokio::test]
+async fn test_prewarm_index_with_position_validation() {
+    let tmpdir = TempStrDir::default();
+    let uri = tmpdir.to_owned();
+    drop(tmpdir);
+
+    let doc_col: Arc<dyn Array> = Arc::new(GenericStringArray::<i32>::from(vec![
+        "lance search",
+        "phrase query",
+    ]));
+    let ids = UInt64Array::from_iter_values(0..doc_col.len() as u64);
+    let batch = RecordBatch::try_new(
+        arrow_schema::Schema::new(vec![
+            arrow_schema::Field::new("doc", doc_col.data_type().to_owned(), true),
+            arrow_schema::Field::new("id", DataType::UInt64, false),
+        ])
+        .into(),
+        vec![Arc::new(doc_col) as ArrayRef, Arc::new(ids) as ArrayRef],
+    )
+    .unwrap();
+    let schema = batch.schema();
+    let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    let mut dataset = Dataset::write(batches, &uri, None).await.unwrap();
+    dataset
+        .create_index(
+            &["doc"],
+            IndexType::Inverted,
+            Some("fts_idx".to_owned()),
+            &InvertedIndexParams::default().with_position(false),
+            true,
+        )
+        .await
+        .unwrap();
+
+    let dataset = open_dataset_with_fresh_session(&uri).await;
+    let err = dataset
+        .prewarm_index_with_options(
+            "fts_idx",
+            &PrewarmOptions::Fts(FtsPrewarmOptions::new().with_position(true)),
+        )
+        .await
+        .unwrap_err()
+        .to_string();
+    assert_contains!(
+        err,
+        "cannot prewarm positions for an inverted index that was built without positions"
+    );
+
+    let tmpdir = TempStrDir::default();
+    let uri = tmpdir.to_owned();
+    drop(tmpdir);
+
+    let batch = RecordBatch::try_from_iter(vec![(
+        "id",
+        Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef,
+    )])
+    .unwrap();
+    let schema = batch.schema();
+    let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    let mut dataset = Dataset::write(batches, &uri, None).await.unwrap();
+    dataset
+        .create_index(
+            &["id"],
+            IndexType::BTree,
+            Some("id_idx".to_owned()),
+            &ScalarIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+
+    let dataset = open_dataset_with_fresh_session(&uri).await;
+    let err = dataset
+        .prewarm_index_with_options("id_idx", &PrewarmOptions::Fts(FtsPrewarmOptions::default()))
+        .await
+        .unwrap_err()
+        .to_string();
+    assert_contains!(
+        err,
+        "FTS prewarm options are only supported for inverted indices"
+    );
+}
+
+#[tokio::test]
+async fn test_fts_phrase_query_with_removed_stop_words() {
+    let tmpdir = TempStrDir::default();
+    let uri = tmpdir.to_owned();
+    drop(tmpdir);
+
+    let doc_col: Arc<dyn Array> = Arc::new(GenericStringArray::<i32>::from(vec![
+        "want the apple",
+        "want an apple",
+        "want green apple",
+        "apple want the",
+    ]));
+    let ids = UInt64Array::from_iter_values(0..doc_col.len() as u64);
+    let batch = RecordBatch::try_new(
+        arrow_schema::Schema::new(vec![
+            arrow_schema::Field::new("doc", doc_col.data_type().to_owned(), true),
+            arrow_schema::Field::new("id", DataType::UInt64, false),
+        ])
+        .into(),
+        vec![Arc::new(doc_col) as ArrayRef, Arc::new(ids) as ArrayRef],
+    )
+    .unwrap();
+    let schema = batch.schema();
+    let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    let mut dataset = Dataset::write(batches, &uri, None).await.unwrap();
+
+    dataset
+        .create_index(
+            &["doc"],
+            IndexType::Inverted,
+            None,
+            &InvertedIndexParams::default()
+                .with_position(true)
+                .remove_stop_words(true),
+            true,
+        )
+        .await
+        .unwrap();
+
+    for query in ["want the apple", "want an apple"] {
+        let result = dataset
+            .scan()
+            .project(&["id"])
+            .unwrap()
+            .full_text_search(FullTextSearchQuery::new_query(
+                PhraseQuery::new(query.to_owned()).into(),
+            ))
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+
+        let ids = result["id"].as_primitive::<UInt64Type>().values();
+        assert_eq!(result.num_rows(), 3, "query={query}, ids={ids:?}");
+        assert!(ids.contains(&0), "query={query}, ids={ids:?}");
+        assert!(ids.contains(&1), "query={query}, ids={ids:?}");
+        assert!(ids.contains(&2), "query={query}, ids={ids:?}");
+    }
+}
+
+#[tokio::test]
+async fn test_fts_phrase_query_preserves_stop_word_gaps() {
+    let tmpdir = TempStrDir::default();
+    let uri = tmpdir.to_owned();
+    drop(tmpdir);
+
+    let doc_col: Arc<dyn Array> = Arc::new(GenericStringArray::<i32>::from(vec![
+        "the united states of america",
+        "the united states and america",
+        "united states america",
+        "the united states of north america",
+    ]));
+    let ids = UInt64Array::from_iter_values(0..doc_col.len() as u64);
+    let batch = RecordBatch::try_new(
+        arrow_schema::Schema::new(vec![
+            arrow_schema::Field::new("doc", doc_col.data_type().to_owned(), true),
+            arrow_schema::Field::new("id", DataType::UInt64, false),
+        ])
+        .into(),
+        vec![Arc::new(doc_col) as ArrayRef, Arc::new(ids) as ArrayRef],
+    )
+    .unwrap();
+    let schema = batch.schema();
+    let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    let mut dataset = Dataset::write(batches, &uri, None).await.unwrap();
+
+    dataset
+        .create_index(
+            &["doc"],
+            IndexType::Inverted,
+            None,
+            &InvertedIndexParams::default()
+                .with_position(true)
+                .remove_stop_words(true),
+            true,
+        )
+        .await
+        .unwrap();
+
+    let result = dataset
+        .scan()
+        .project(&["id"])
+        .unwrap()
+        .full_text_search(FullTextSearchQuery::new_query(
+            PhraseQuery::new("the united states of america".to_owned()).into(),
+        ))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+
+    let ids = result["id"].as_primitive::<UInt64Type>().values();
+    assert_eq!(result.num_rows(), 2, "ids={ids:?}");
+    assert!(ids.contains(&0), "ids={ids:?}");
+    assert!(ids.contains(&1), "ids={ids:?}");
+    assert!(!ids.contains(&2), "ids={ids:?}");
+    assert!(!ids.contains(&3), "ids={ids:?}");
+}
+
 async fn prepare_json_dataset() -> (Dataset, String) {
     let text_col = Arc::new(StringArray::from(vec![
         r#"{
@@ -2510,3 +2811,190 @@ async fn test_auto_infer_lance_tokenizer() {
         .unwrap();
     assert_eq!(1, batch.num_rows());
 }
+
+#[tokio::test]
+async fn test_index_inherits_dataset_file_version() {
+    // Test that index files use the same format version as the dataset
+    let test_uri = TempStrDir::default();
+
+    let dimension = 16;
+    let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "embeddings",
+        DataType::FixedSizeList(
+            Arc::new(ArrowField::new("item", DataType::Float32, true)),
+            dimension,
+        ),
+        false,
+    )]));
+
+    let float_arr = generate_random_array(512 * dimension as usize);
+    let vectors = Arc::new(
+        <arrow_array::FixedSizeListArray as FixedSizeListArrayExt>::try_new_from_values(
+            float_arr, dimension,
+        )
+        .unwrap(),
+    );
+    let batches = vec![RecordBatch::try_new(schema.clone(), vec![vectors.clone()]).unwrap()];
+
+    let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+
+    // Create dataset with V2_1 file version
+    let dataset_version = LanceFileVersion::V2_1;
+    let mut dataset = Dataset::write(
+        reader,
+        &test_uri,
+        Some(WriteParams {
+            data_storage_version: Some(dataset_version),
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    // Create a vector index
+    let params = VectorIndexParams::ivf_pq(10, 8, 2, MetricType::L2, 50);
+    let index_meta = dataset
+        .create_index(&["embeddings"], IndexType::Vector, None, &params, true)
+        .await
+        .unwrap();
+
+    // Get the index directory
+    let index_dir = dataset.indices_dir().child(index_meta.uuid.to_string());
+
+    // Open the index file and check its version
+    let index_path = index_dir.child("index.idx");
+    let scheduler = ScanScheduler::new(
+        dataset.object_store.clone(),
+        SchedulerConfig::max_bandwidth(&dataset.object_store),
+    );
+
+    let file_handle = scheduler
+        .open_file(&index_path, &CachedFileSize::unknown())
+        .await
+        .unwrap();
+
+    let index_reader = FileReader::try_open(
+        file_handle,
+        None,
+        Arc::default(),
+        &LanceCache::no_cache(),
+        FileReaderOptions::default(),
+    )
+    .await
+    .unwrap();
+
+    // Verify that the index file uses the same version as the dataset
+    assert_eq!(
+        index_reader.metadata().version(),
+        dataset_version,
+        "Index file should use the same format version as the dataset"
+    );
+
+    // Also check the auxiliary file if it exists
+    let aux_path = index_dir.child("auxiliary.idx");
+    if dataset
+        .object_store
+        .exists(&aux_path)
+        .await
+        .unwrap_or(false)
+    {
+        let aux_handle = scheduler
+            .open_file(&aux_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+
+        let aux_reader = FileReader::try_open(
+            aux_handle,
+            None,
+            Arc::default(),
+            &LanceCache::no_cache(),
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(
+            aux_reader.metadata().version(),
+            dataset_version,
+            "Auxiliary index file should use the same format version as the dataset"
+        );
+    }
+}
+
+#[tokio::test]
+async fn test_legacy_dataset_uses_v2_0_for_indexes() {
+    // Test that datasets with legacy format still use V2_0 for indexes (not legacy)
+    let test_uri = TempStrDir::default();
+
+    let dimension = 16;
+    let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "embeddings",
+        DataType::FixedSizeList(
+            Arc::new(ArrowField::new("item", DataType::Float32, true)),
+            dimension,
+        ),
+        false,
+    )]));
+
+    let float_arr = generate_random_array(512 * dimension as usize);
+    let vectors = Arc::new(
+        <arrow_array::FixedSizeListArray as FixedSizeListArrayExt>::try_new_from_values(
+            float_arr, dimension,
+        )
+        .unwrap(),
+    );
+    let batches = vec![RecordBatch::try_new(schema.clone(), vec![vectors.clone()]).unwrap()];
+
+    let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+
+    // Create dataset with legacy file version
+    let mut dataset = Dataset::write(
+        reader,
+        &test_uri,
+        Some(WriteParams {
+            data_storage_version: Some(LanceFileVersion::Legacy),
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    // Create a vector index
+    let params = VectorIndexParams::ivf_pq(10, 8, 2, MetricType::L2, 50);
+    let index_meta = dataset
+        .create_index(&["embeddings"], IndexType::Vector, None, &params, true)
+        .await
+        .unwrap();
+
+    // Get the index directory
+    let index_dir = dataset.indices_dir().child(index_meta.uuid.to_string());
+
+    // Open the index file and check its version
+    let index_path = index_dir.child("index.idx");
+    let scheduler = ScanScheduler::new(
+        dataset.object_store.clone(),
+        SchedulerConfig::max_bandwidth(&dataset.object_store),
+    );
+
+    let file_handle = scheduler
+        .open_file(&index_path, &CachedFileSize::unknown())
+        .await
+        .unwrap();
+
+    let index_reader = FileReader::try_open(
+        file_handle,
+        None,
+        Arc::default(),
+        &LanceCache::no_cache(),
+        FileReaderOptions::default(),
+    )
+    .await
+    .unwrap();
+
+    // Verify that the index file uses V2_0 (not legacy)
+    assert_eq!(
+        index_reader.metadata().version(),
+        LanceFileVersion::V2_0,
+        "Index files should never use legacy format, even for legacy datasets"
+    );
+}
diff --git a/rust/lance/src/dataset/tests/dataset_io.rs b/rust/lance/src/dataset/tests/dataset_io.rs
index 325564cc8ee..306580f4aaa 100644
--- a/rust/lance/src/dataset/tests/dataset_io.rs
+++ b/rust/lance/src/dataset/tests/dataset_io.rs
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use std::collections::HashMap;
 use std::sync::Arc;
 use std::vec;
 
@@ -35,11 +36,13 @@ use lance_datagen::{BatchCount, RowCount, array, gen_batch};
 use lance_file::version::LanceFileVersion;
 use lance_io::assert_io_eq;
 use lance_table::feature_flags;
+use lance_table::format::BasePath;
 
+use crate::index::DatasetIndexExt;
 use futures::TryStreamExt;
+use lance_index::IndexType;
 use lance_index::scalar::ScalarIndexParams;
-use lance_index::{DatasetIndexExt, IndexType};
-use lance_io::object_store::{ObjectStore, ObjectStoreParams};
+use lance_io::object_store::{ObjectStore, ObjectStoreParams, StorageOptionsAccessor};
 use lance_io::utils::tracking_store::IOTracker;
 use lance_table::io::manifest::read_manifest;
 use object_store::path::Path;
@@ -166,6 +169,67 @@ async fn test_with_object_store_enables_isolated_per_request_io_tracking() {
     assert_eq!(tracker_b.incremental_stats().read_iops, 0);
 }
 
+#[cfg(feature = "azure")]
+#[tokio::test]
+async fn test_object_store_for_base_uses_runtime_base_store_params() {
+    let test_dir = TempStdDir::default();
+    create_file(&test_dir, WriteMode::Create, LanceFileVersion::Stable).await;
+    let uri = test_dir.to_str().unwrap();
+    let dataset = Arc::new(Dataset::open(uri).await.unwrap());
+
+    let base_a = BasePath::new(
+        1,
+        "az://container/path-a".to_string(),
+        Some("base-a".to_string()),
+        true,
+    );
+    let base_b = BasePath::new(
+        2,
+        "az://container/path-b".to_string(),
+        Some("base-b".to_string()),
+        true,
+    );
+    dataset
+        .add_bases(vec![base_a.clone(), base_b.clone()], None)
+        .await
+        .unwrap();
+
+    let base_a_store_params = ObjectStoreParams {
+        storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+            HashMap::from([
+                ("account_name".to_string(), "account-a".to_string()),
+                ("account_key".to_string(), "dGVzdA==".to_string()),
+            ]),
+        ))),
+        ..Default::default()
+    };
+    let default_store_params = ObjectStoreParams {
+        storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+            HashMap::from([
+                ("account_name".to_string(), "account-b".to_string()),
+                ("account_key".to_string(), "dGVzdA==".to_string()),
+            ]),
+        ))),
+        ..Default::default()
+    };
+
+    let dataset = DatasetBuilder::from_uri(uri)
+        .with_store_params(default_store_params)
+        .with_base_store_params(&base_a.path, base_a_store_params)
+        .load()
+        .await
+        .unwrap();
+
+    let store_a = dataset.object_store_for_base(1).await.unwrap();
+    let store_a_again = dataset.object_store_for_base(1).await.unwrap();
+    let store_b = dataset.object_store_for_base(2).await.unwrap();
+
+    assert!(Arc::ptr_eq(&store_a, &store_a_again));
+    assert!(!Arc::ptr_eq(&store_a, &store_b));
+    assert_eq!(store_a.store_prefix, "az$container@account-a");
+    assert_eq!(store_b.store_prefix, "az$container@account-b");
+}
+
 #[rstest]
 #[lance_test_macros::test(tokio::test)]
 async fn test_create_dataset(
@@ -1437,6 +1501,118 @@ async fn test_fast_count_rows(
     );
 }
 
+#[rstest]
+#[tokio::test]
+async fn test_sample_with_fragment_ids(
+    #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
+    data_storage_version: LanceFileVersion,
+) {
+    let test_uri = TempStrDir::default();
+    let data = gen_batch()
+        .col("i", array::step::<Int32Type>())
+        .into_reader_rows(RowCount::from(12), BatchCount::from(1));
+    let mut dataset = Dataset::write(
+        data,
+        &test_uri,
+        Some(WriteParams {
+            max_rows_per_file: 4,
+            max_rows_per_group: 2,
+            data_storage_version: Some(data_storage_version),
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    dataset.delete("i IN (1, 9)").await.unwrap();
+
+    let projection = dataset.schema().project(&["i"]).unwrap();
+    let sampled = dataset
+        .sample(8, &projection, Some(&[0, 0, 2]))
+        .await
+        .unwrap();
+    let sampled_values = sampled
+        .column_by_name("i")
+        .unwrap()
+        .as_any()
+        .downcast_ref::<Int32Array>()
+        .unwrap()
+        .values()
+        .to_vec();
+
+    assert_eq!(sampled_values, vec![0, 2, 3, 8, 10, 11]);
+}
+
+#[rstest]
+#[tokio::test]
+async fn test_sample_with_empty_fragment_ids_rejected(
+    #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
+    data_storage_version: LanceFileVersion,
+) {
+    let test_uri = TempStrDir::default();
+    let data = gen_batch()
+        .col("i", array::step::<Int32Type>())
+        .into_reader_rows(RowCount::from(8), BatchCount::from(1));
+    let dataset = Dataset::write(
+        data,
+        &test_uri,
+        Some(WriteParams {
+            max_rows_per_file: 4,
+            max_rows_per_group: 2,
+            data_storage_version: Some(data_storage_version),
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    let projection = dataset.schema().project(&["i"]).unwrap();
+    let err = dataset.sample(1, &projection, Some(&[])).await.unwrap_err();
+
+    assert!(matches!(err, Error::InvalidInput { .. }));
+    assert!(
+        err.to_string()
+            .contains("does not accept an empty fragment_ids list")
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn test_sample_with_unknown_fragment_ids_rejected(
+    #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
+    data_storage_version: LanceFileVersion,
+) {
+    let test_uri = TempStrDir::default();
+    let data = gen_batch()
+        .col("i", array::step::<Int32Type>())
+        .into_reader_rows(RowCount::from(8), BatchCount::from(1));
+    let dataset = Dataset::write(
+        data,
+        &test_uri,
+        Some(WriteParams {
+            max_rows_per_file: 4,
+            max_rows_per_group: 2,
+            data_storage_version: Some(data_storage_version),
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    let projection = dataset.schema().project(&["i"]).unwrap();
+    let err = dataset
+        .sample(1, &projection, Some(&[0, 999]))
+        .await
+        .unwrap_err();
+
+    assert!(matches!(err, Error::InvalidInput { .. }));
+    assert!(
+        err.to_string()
+            .contains("not part of the current dataset version")
+    );
+    assert!(err.to_string().contains("999"));
+}
+
 #[rstest]
 #[tokio::test]
 async fn test_bfloat16_roundtrip(
@@ -1616,3 +1792,57 @@ async fn test_dataset_uri_roundtrips() {
         dataset.latest_version_id().await.unwrap()
     );
 }
+
+/// A commit handler whose resolve_latest_location always returns an IO error.
+/// Used to verify that non-NotFound errors from resolve_latest_location are
+/// propagated as-is rather than being wrapped as DatasetNotFound.
+#[derive(Debug)]
+struct ErroringCommitHandler;
+
+#[async_trait::async_trait]
+impl lance_table::io::commit::CommitHandler for ErroringCommitHandler {
+    async fn resolve_latest_location(
+        &self,
+        _base_path: &Path,
+        _object_store: &ObjectStore,
+    ) -> Result<lance_table::io::commit::ManifestLocation> {
+        Err(Error::io("simulated I/O error".to_string()))
+    }
+
+    async fn commit(
+        &self,
+        _manifest: &mut lance_table::format::Manifest,
+        _indices: Option<Vec<lance_table::format::IndexMetadata>>,
+        _base_path: &Path,
+        _object_store: &ObjectStore,
+        _manifest_writer: lance_table::io::commit::ManifestWriter,
+        _naming_scheme: lance_table::io::commit::ManifestNamingScheme,
+        _transaction: Option<lance_table::format::Transaction>,
+    ) -> std::result::Result<
+        lance_table::io::commit::ManifestLocation,
+        lance_table::io::commit::CommitError,
+    > {
+        unimplemented!()
+    }
+}
+
+#[tokio::test]
+async fn test_open_dataset_non_not_found_error_is_not_masked() {
+    // When resolve_latest_location returns an IO error, it should propagate
+    // as an IO error, not be wrapped as DatasetNotFound.
+    let store = Arc::new(object_store::memory::InMemory::new());
+    let location = url::Url::parse("memory://test").unwrap();
+
+    #[allow(deprecated)]
+    let result = DatasetBuilder::from_uri("memory://test")
+        .with_object_store(store, location, Arc::new(ErroringCommitHandler))
+        .load()
+        .await;
+
+    let err = result.unwrap_err();
+    assert!(
+        matches!(err, Error::IO { .. }),
+        "Expected IO error but got: {:?}",
+        err,
+    );
+}
diff --git a/rust/lance/src/dataset/tests/dataset_merge_update.rs b/rust/lance/src/dataset/tests/dataset_merge_update.rs
index ef64773cc3c..371e8769c8a 100644
--- a/rust/lance/src/dataset/tests/dataset_merge_update.rs
+++ b/rust/lance/src/dataset/tests/dataset_merge_update.rs
@@ -9,11 +9,12 @@ use crate::dataset::WriteDestination;
 use crate::dataset::optimize::{CompactionOptions, compact_files};
 use crate::dataset::transaction::{DataReplacementGroup, Operation};
 use crate::dataset::{AutoCleanupParams, MergeInsertBuilder, ProjectionRequest};
+use crate::index::DatasetIndexExt;
 use crate::{Dataset, Error};
 use lance_core::ROW_ADDR;
+use lance_index::IndexType;
 use lance_index::optimize::OptimizeOptions;
 use lance_index::scalar::ScalarIndexParams;
-use lance_index::{DatasetIndexExt, IndexType};
 use mock_instant::thread_local::MockClock;
 
 use crate::dataset::write::{InsertBuilder, WriteMode, WriteParams};
@@ -375,7 +376,7 @@ async fn test_insert_subschema() {
     let fragments = dataset.get_fragments();
     assert_eq!(fragments.len(), 1);
     assert_eq!(fragments[0].metadata.files.len(), 1);
-    assert_eq!(&fragments[0].metadata.files[0].fields, &[0]);
+    assert_eq!(fragments[0].metadata.files[0].fields.as_ref(), &[0]);
 
     // When reading back, columns that are missing are null
     let data = dataset.scan().try_into_batch().await.unwrap();
@@ -423,7 +424,7 @@ async fn test_insert_subschema() {
     let fragments = dataset.get_fragments();
     assert_eq!(fragments.len(), 1);
     assert_eq!(fragments[0].metadata.files.len(), 1);
-    assert_eq!(&fragments[0].metadata.files[0].fields, &[0, 1]);
+    assert_eq!(fragments[0].metadata.files[0].fields.as_ref(), &[0, 1]);
 
     // Can scan and get expected data.
     let data = dataset.scan().try_into_batch().await.unwrap();
@@ -479,8 +480,11 @@ async fn test_insert_nested_subschemas() {
     let fragments = dataset.get_fragments();
     assert_eq!(fragments.len(), 1);
     assert_eq!(fragments[0].metadata.files.len(), 1);
-    assert_eq!(&fragments[0].metadata.files[0].fields, &[0, 2, 1]);
-    assert_eq!(&fragments[0].metadata.files[0].column_indices, &[0, 1, 2]);
+    assert_eq!(fragments[0].metadata.files[0].fields.as_ref(), &[2, 1]);
+    assert_eq!(
+        fragments[0].metadata.files[0].column_indices.as_ref(),
+        &[0, 1]
+    );
 
     // Can insert c, b
     let just_c_b = Arc::new(ArrowSchema::new(vec![ArrowField::new(
@@ -507,8 +511,11 @@ async fn test_insert_nested_subschemas() {
     let fragments = dataset.get_fragments();
     assert_eq!(fragments.len(), 2);
     assert_eq!(fragments[1].metadata.files.len(), 1);
-    assert_eq!(&fragments[1].metadata.files[0].fields, &[0, 3, 2]);
-    assert_eq!(&fragments[1].metadata.files[0].column_indices, &[0, 1, 2]);
+    assert_eq!(fragments[1].metadata.files[0].fields.as_ref(), &[3, 2]);
+    assert_eq!(
+        fragments[1].metadata.files[0].column_indices.as_ref(),
+        &[0, 1]
+    );
 
     // Can't insert a, c (b is non-nullable)
     let just_a_c = Arc::new(ArrowSchema::new(vec![ArrowField::new(
@@ -611,7 +618,7 @@ async fn test_insert_balanced_subschemas() {
     let fragments = dataset.get_fragments();
     assert_eq!(fragments.len(), 1);
     assert_eq!(fragments[0].metadata.files.len(), 1);
-    assert_eq!(&fragments[0].metadata.files[0].fields, &[0]);
+    assert_eq!(fragments[0].metadata.files[0].fields.as_ref(), &[0]);
 
     // Insert right side
     let just_b = Arc::new(ArrowSchema::new(vec![field_b.clone()]));
@@ -627,7 +634,7 @@ async fn test_insert_balanced_subschemas() {
     let fragments = dataset.get_fragments();
     assert_eq!(fragments.len(), 2);
     assert_eq!(fragments[1].metadata.files.len(), 1);
-    assert_eq!(&fragments[1].metadata.files[0].fields, &[1]);
+    assert_eq!(fragments[1].metadata.files[0].fields.as_ref(), &[1]);
 
     let data = dataset
         .take(
@@ -859,9 +866,9 @@ async fn test_datafile_partial_replacement() {
     let new_data_file = DataFile {
         path: "test.lance".to_string(),
         // the second column in the dataset
-        fields: vec![1],
+        fields: Arc::from([1]),
         // is located in the first column of this datafile
-        column_indices: vec![0],
+        column_indices: Arc::from([0]),
         file_major_version: major,
         file_minor_version: minor,
         file_size_bytes: CachedFileSize::unknown(),
@@ -885,8 +892,14 @@ async fn test_datafile_partial_replacement() {
     assert_eq!(dataset.version().version, 4);
     assert_eq!(dataset.get_fragments().len(), 1);
     assert_eq!(dataset.get_fragments()[0].metadata.files.len(), 2);
-    assert_eq!(dataset.get_fragments()[0].metadata.files[0].fields, vec![0]);
-    assert_eq!(dataset.get_fragments()[0].metadata.files[1].fields, vec![1]);
+    assert_eq!(
+        dataset.get_fragments()[0].metadata.files[0].fields.as_ref(),
+        &[0]
+    );
+    assert_eq!(
+        dataset.get_fragments()[0].metadata.files[1].fields.as_ref(),
+        &[1]
+    );
 
     let batch = dataset.scan().try_into_batch().await.unwrap();
     assert_eq!(batch.num_rows(), 3);
@@ -914,9 +927,9 @@ async fn test_datafile_partial_replacement() {
     let new_data_file = DataFile {
         path: "test.lance".to_string(),
         // the first column in the dataset
-        fields: vec![0],
+        fields: Arc::from([0]),
         // is located in the first column of this datafile
-        column_indices: vec![0],
+        column_indices: Arc::from([0]),
         file_major_version: major,
         file_minor_version: minor,
         file_size_bytes: CachedFileSize::unknown(),
@@ -1013,9 +1026,9 @@ async fn test_datafile_replacement_error() {
     let new_data_file = DataFile {
         path: "test.lance".to_string(),
         // the second column in the dataset
-        fields: vec![1],
+        fields: Arc::from([1]),
         // is located in the first column of this datafile
-        column_indices: vec![0],
+        column_indices: Arc::from([0]),
         file_major_version: 2,
         file_minor_version: 0,
         file_size_bytes: CachedFileSize::unknown(),
@@ -1023,7 +1036,7 @@ async fn test_datafile_replacement_error() {
     };
 
     let new_data_file = DataFile {
-        fields: vec![0, 1],
+        fields: Arc::from([0, 1]),
         ..new_data_file
     };
 
@@ -1705,3 +1718,498 @@ async fn test_data_replacement_invalidates_index_bitmap() {
         "Fragment 0 should be removed from index bitmap after DataReplacement on indexed column"
     );
 }
+/// Regression test: inverted (FTS) index should not carry stale data after
+/// merge_insert + compact + optimize_indices.
+///
+/// This is the FTS equivalent of test_merge_insert_with_reordered_columns_and_index.
+/// The inverted index's update() ignores the valid_old_fragments filter, so stale
+/// posting list entries from pruned fragments survive the merge and cause errors
+/// when queries try to resolve the old row addresses.
+#[tokio::test]
+async fn test_fts_index_stale_data_after_merge_insert_compact_optimize() {
+    use lance_index::scalar::{FullTextSearchQuery, inverted::InvertedIndexParams};
+
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, false),
+        ArrowField::new("text", DataType::Utf8, true),
+    ]));
+
+    // Step 1: Create dataset with 2 rows in separate fragments
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from(vec![0, 1])),
+            Arc::new(StringArray::from(vec![
+                "the quick brown fox",
+                "the lazy dog",
+            ])),
+        ],
+    )
+    .unwrap();
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+    let mut dataset = Dataset::write(
+        reader,
+        "memory://test_fts_stale",
+        Some(WriteParams {
+            max_rows_per_file: 1, // Force 2 fragments
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    // Step 2: Create FTS inverted index on 'text'
+    let params = InvertedIndexParams::default();
+    dataset
+        .create_index(&["text"], IndexType::Inverted, None, &params, true)
+        .await
+        .unwrap();
+
+    // Sanity check: searching "quick" should return 1 result
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("quick".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(results.num_rows(), 1);
+
+    // Step 3: merge_insert with reversed column order (text, id)
+    // This triggers the RewriteColumns/DataReplacement path, which prunes the
+    // index fragment bitmap for the 'text' column.
+    let reversed_schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("text", DataType::Utf8, true),
+        ArrowField::new("id", DataType::Int32, false),
+    ]));
+    let source_batch = RecordBatch::try_new(
+        reversed_schema.clone(),
+        vec![
+            Arc::new(StringArray::from(vec![
+                "updated fox text",
+                "new entry here",
+            ])),
+            Arc::new(Int32Array::from(vec![1, 2])),
+        ],
+    )
+    .unwrap();
+
+    let merge_job = MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .try_build()
+        .unwrap();
+
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(source_batch)],
+        reversed_schema.clone(),
+    ));
+    let (dataset, _stats) = merge_job.execute(reader_to_stream(reader)).await.unwrap();
+    let mut dataset = dataset.as_ref().clone();
+
+    // Step 4: compact_files — moves rows to new fragment(s)
+    compact_files(&mut dataset, CompactionOptions::default(), None)
+        .await
+        .unwrap();
+
+    // Step 5: optimize_indices — should rebuild the FTS index without stale data.
+    // With the current bug, the inverted index ignores valid_old_fragments and
+    // merges stale posting list entries pointing at now-deleted fragments.
+    dataset
+        .optimize_indices(&OptimizeOptions::default())
+        .await
+        .unwrap();
+
+    // Step 6: FTS search should not error and should return correct results.
+    // "quick" appeared in the original data for id=0 (never updated), so it
+    // should still be found.
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("quick".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(
+        results.num_rows(),
+        1,
+        "Expected 1 result for 'quick' after optimize, got {}",
+        results.num_rows()
+    );
+
+    // "lazy" was in the original text for id=1, but id=1 was updated to
+    // "updated fox text". The old posting for "lazy" should have been filtered
+    // out during the index update.
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("lazy".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(
+        results.num_rows(),
+        0,
+        "Expected 0 results for 'lazy' (stale data should be filtered), got {}",
+        results.num_rows()
+    );
+
+    // "updated" should be found (new text for id=1)
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("updated".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(results.num_rows(), 1);
+
+    // "entry" should be found (new row id=2)
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("entry".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(results.num_rows(), 1);
+
+    // Step 7: Another merge_insert should NOT error
+    let source_batch2 = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from(vec![1])),
+            Arc::new(StringArray::from(vec!["final text"])),
+        ],
+    )
+    .unwrap();
+
+    let merge_job2 = MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .try_build()
+        .unwrap();
+
+    let reader2 = Box::new(RecordBatchIterator::new(
+        vec![Ok(source_batch2)],
+        schema.clone(),
+    ));
+    let (final_dataset, _) = merge_job2.execute(reader_to_stream(reader2)).await.unwrap();
+    final_dataset.validate().await.unwrap();
+}
+
+/// Regression test: when rows are updated in-place, the FTS index must
+/// invalidate old entries and allow re-indexing incrementally.
+///
+/// Sequence:
+/// 1. Write fragments 1 and 2.
+/// 2. Create FTS index covering fragments 1 and 2.
+/// 3. Update fragment 1 in-place via merge_insert (DataReplacement path).
+///    This removes fragment 1 from the index's fragment_bitmap.
+/// 4. Call optimize_indices (append) to create a new index segment covering
+///    the updated fragment 1.
+/// 5. Call optimize_indices (merge) to merge both segments. The first segment
+///    contains the old, invalidated values for fragment 1; the second segment
+///    contains the new, valid values. We must keep only the new values.
+#[tokio::test]
+async fn test_fts_index_incremental_reindex_after_in_place_update() {
+    use lance_index::scalar::{FullTextSearchQuery, inverted::InvertedIndexParams};
+
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, false),
+        ArrowField::new("text", DataType::Utf8, true),
+    ]));
+
+    // Step 1: Create dataset with 2 rows in separate fragments
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from(vec![0, 1])),
+            Arc::new(StringArray::from(vec![
+                "the quick brown fox",
+                "the lazy dog",
+            ])),
+        ],
+    )
+    .unwrap();
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+    let mut dataset = Dataset::write(
+        reader,
+        "memory://test_fts_incremental_reindex",
+        Some(WriteParams {
+            max_rows_per_file: 1, // Force 2 fragments
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    // Step 2: Create FTS inverted index on 'text'
+    let params = InvertedIndexParams::default();
+    dataset
+        .create_index(&["text"], IndexType::Inverted, None, &params, true)
+        .await
+        .unwrap();
+
+    // Sanity check: "quick" and "lazy" should each return 1 result
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("quick".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(results.num_rows(), 1);
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("lazy".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(results.num_rows(), 1);
+
+    // Step 3: merge_insert with reversed column order to trigger
+    // RewriteColumns/DataReplacement path, which prunes the index
+    // fragment bitmap for the updated fragment.
+    // Update id=1 ("the lazy dog" -> "a speedy cat")
+    let reversed_schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("text", DataType::Utf8, true),
+        ArrowField::new("id", DataType::Int32, false),
+    ]));
+    let source_batch = RecordBatch::try_new(
+        reversed_schema.clone(),
+        vec![
+            Arc::new(StringArray::from(vec!["a speedy cat"])),
+            Arc::new(Int32Array::from(vec![1])),
+        ],
+    )
+    .unwrap();
+
+    let merge_job = MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::DoNothing)
+        .try_build()
+        .unwrap();
+
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(source_batch)],
+        reversed_schema.clone(),
+    ));
+    let (dataset, _stats) = merge_job.execute(reader_to_stream(reader)).await.unwrap();
+    let mut dataset = dataset.as_ref().clone();
+
+    // Step 4: First optimize_indices (append) — creates a new index segment
+    // covering the updated (previously unindexed) fragment.
+    dataset
+        .optimize_indices(&OptimizeOptions::append())
+        .await
+        .unwrap();
+
+    // At this point we have two index segments:
+    //  - Segment 1: original index (has old data for fragment with id=1)
+    //  - Segment 2: new delta index (has new data for the updated fragment)
+
+    // Step 5: Second optimize_indices (merge all) — merges both segments.
+    // The merge must discard old invalidated entries from segment 1 for
+    // the updated fragment and keep only the new entries from segment 2.
+    dataset
+        .optimize_indices(&OptimizeOptions::default())
+        .await
+        .unwrap();
+
+    // Step 6: Verify search correctness after merge.
+
+    // "quick" was in the original data for id=0 (not updated), should still be found.
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("quick".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(
+        results.num_rows(),
+        1,
+        "Expected 1 result for 'quick' (id=0 was not updated), got {}",
+        results.num_rows()
+    );
+
+    // "lazy" was in the old text for id=1 which was updated to "a speedy cat".
+    // The old posting for "lazy" must have been filtered out during the merge.
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("lazy".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(
+        results.num_rows(),
+        0,
+        "Expected 0 results for 'lazy' (stale data should be filtered), got {}",
+        results.num_rows()
+    );
+
+    // "speedy" is in the new text for id=1, should be found.
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("speedy".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(
+        results.num_rows(),
+        1,
+        "Expected 1 result for 'speedy' (new text for id=1), got {}",
+        results.num_rows()
+    );
+
+    // "cat" is in the new text for id=1, should be found.
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("cat".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(
+        results.num_rows(),
+        1,
+        "Expected 1 result for 'cat' (new text for id=1), got {}",
+        results.num_rows()
+    );
+}
+
+/// Regression test for https://github.com/lance-format/lance/issues/6338
+/// Sub-schema merge_insert with binary columns on v2.2 causes data corruption
+/// when the binary values are >= 256 bytes.
+#[tokio::test]
+async fn test_sub_schema_merge_insert_binary_v2_2() {
+    use crate::dataset::write::merge_insert::WhenMatched;
+    use arrow_array::BinaryArray;
+
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int64, false),
+        ArrowField::new("a", DataType::Binary, true),
+        ArrowField::new("b", DataType::Utf8, true),
+    ]));
+
+    let test_uri = TempStrDir::default();
+
+    // Initial write: 2 rows with null binary values
+    let initial_batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(arrow_array::Int64Array::from(vec![0, 1])),
+            Arc::new(BinaryArray::from(vec![None::<&[u8]>, None])),
+            Arc::new(StringArray::from(vec![None::<&str>, None])),
+        ],
+    )
+    .unwrap();
+
+    let write_params = WriteParams {
+        data_storage_version: Some(LanceFileVersion::V2_2),
+        ..Default::default()
+    };
+    let batches = RecordBatchIterator::new(vec![initial_batch].into_iter().map(Ok), schema.clone());
+    Dataset::write(batches, &test_uri, Some(write_params))
+        .await
+        .unwrap();
+
+    let sub_schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int64, false),
+        ArrowField::new("a", DataType::Binary, true),
+    ]));
+
+    // Sub-schema merge_insert for row 0 (binary value >= 256 bytes)
+    let data_a: Vec<u8> = (0..256).map(|i| (i % 251) as u8).collect();
+    {
+        let update_batch = RecordBatch::try_new(
+            sub_schema.clone(),
+            vec![
+                Arc::new(arrow_array::Int64Array::from(vec![0])),
+                Arc::new(BinaryArray::from(vec![Some(data_a.as_slice())])),
+            ],
+        )
+        .unwrap();
+        let dataset = Dataset::open(&test_uri).await.unwrap();
+        let source = Box::new(RecordBatchIterator::new(
+            vec![update_batch].into_iter().map(Ok),
+            sub_schema.clone(),
+        ));
+        MergeInsertBuilder::try_new(dataset.into(), vec!["id".into()])
+            .unwrap()
+            .when_matched(WhenMatched::UpdateAll)
+            .try_build()
+            .unwrap()
+            .execute_reader(source)
+            .await
+            .unwrap();
+    }
+
+    // Read back and verify first merge worked
+    let dataset = Dataset::open(&test_uri).await.unwrap();
+    let table = dataset
+        .scan()
+        .project(&["id", "a"])
+        .unwrap()
+        .try_into_stream()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+    let table = concat_batches(&table[0].schema(), &table).unwrap();
+    assert_eq!(table.num_rows(), 2);
+
+    // Sub-schema merge_insert for row 1 (binary value >= 256 bytes)
+    let data_b: Vec<u8> = (0..256).map(|i| ((i + 100) % 251) as u8).collect();
+    {
+        let update_batch = RecordBatch::try_new(
+            sub_schema.clone(),
+            vec![
+                Arc::new(arrow_array::Int64Array::from(vec![1])),
+                Arc::new(BinaryArray::from(vec![Some(data_b.as_slice())])),
+            ],
+        )
+        .unwrap();
+        let dataset = Dataset::open(&test_uri).await.unwrap();
+        let source = Box::new(RecordBatchIterator::new(
+            vec![update_batch].into_iter().map(Ok),
+            sub_schema.clone(),
+        ));
+        MergeInsertBuilder::try_new(dataset.into(), vec!["id".into()])
+            .unwrap()
+            .when_matched(WhenMatched::UpdateAll)
+            .try_build()
+            .unwrap()
+            .execute_reader(source)
+            .await
+            .unwrap();
+    }
+
+    // Read back and verify - this is where the bug manifests
+    let dataset = Dataset::open(&test_uri).await.unwrap();
+    let table = dataset
+        .scan()
+        .project(&["id", "a"])
+        .unwrap()
+        .try_into_stream()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+    let table = concat_batches(&table[0].schema(), &table).unwrap();
+    assert_eq!(table.num_rows(), 2);
+
+    let a_col = table.column_by_name("a").unwrap();
+    let binary_arr = a_col.as_any().downcast_ref::<BinaryArray>().unwrap();
+    assert_eq!(binary_arr.value(0), data_a.as_slice());
+    assert_eq!(binary_arr.value(1), data_b.as_slice());
+}
diff --git a/rust/lance/src/dataset/tests/dataset_migrations.rs b/rust/lance/src/dataset/tests/dataset_migrations.rs
index f205c880416..d71a65bfa69 100644
--- a/rust/lance/src/dataset/tests/dataset_migrations.rs
+++ b/rust/lance/src/dataset/tests/dataset_migrations.rs
@@ -4,18 +4,19 @@
 use std::sync::Arc;
 use std::vec;
 
+use crate::dataset::InsertBuilder;
 use crate::dataset::optimize::{CompactionOptions, compact_files};
 use crate::utils::test::copy_test_data_to_tmp;
 use crate::{Dataset, Result};
 use lance_table::format::IndexMetadata;
 
 use crate::dataset::write::{WriteMode, WriteParams};
+use crate::index::DatasetIndexExt;
 use arrow::compute::concat_batches;
 use arrow_array::RecordBatch;
 use arrow_array::{Float32Array, Int64Array, RecordBatchIterator};
 use arrow_schema::Schema as ArrowSchema;
 use lance_file::version::LanceFileVersion;
-use lance_index::DatasetIndexExt;
 
 use futures::{StreamExt, TryStreamExt};
 use rstest::rstest;
@@ -376,6 +377,107 @@ async fn test_max_fragment_id_migration() {
     }
 }
 
+#[tokio::test]
+async fn test_index_without_file_sizes() {
+    // Test that we can open indices created before the `files` field was added
+    // to IndexMetadata. The index should still work correctly, falling back to
+    // HEAD calls for file sizes.
+
+    let test_dir = copy_test_data_to_tmp("pre_file_sizes/index_without_file_sizes").unwrap();
+    let test_uri = test_dir.path_str();
+
+    // Open the dataset
+    let dataset = Dataset::open(&test_uri).await.unwrap();
+
+    // Verify the index exists and has no file size info
+    let indices = dataset.load_indices().await.unwrap();
+    assert_eq!(indices.len(), 1);
+    let index = &indices[0];
+    assert_eq!(index.name, "values_idx");
+    assert!(
+        index.files.is_none() || index.files.as_ref().unwrap().is_empty(),
+        "Index should not have file size info (created with old version)"
+    );
+
+    // Verify the index still works - scan with a filter that uses the index
+    let batch = dataset
+        .scan()
+        .filter("values = 'value_42'")
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(batch.num_rows(), 1);
+
+    // Verify describe_indices returns None for total_size_bytes for old indices
+    let descriptions = dataset.describe_indices(None).await.unwrap();
+    assert_eq!(descriptions.len(), 1);
+    assert!(
+        descriptions[0].total_size_bytes().is_none(),
+        "Old index without file sizes should return None for total_size_bytes"
+    );
+}
+
+#[tokio::test]
+async fn test_index_file_size_migration() {
+    // Test that file sizes are migrated when a write operation is performed
+    // on a dataset with an index missing file sizes.
+
+    let test_dir = copy_test_data_to_tmp("pre_file_sizes/index_without_file_sizes").unwrap();
+    let test_uri = test_dir.path_str();
+
+    // Open the dataset and verify the index has no file sizes
+    let dataset = Dataset::open(&test_uri).await.unwrap();
+    let indices = dataset.load_indices().await.unwrap();
+    assert!(
+        indices[0].files.is_none() || indices[0].files.as_ref().unwrap().is_empty(),
+        "Index should not have file size info before migration"
+    );
+
+    // Perform a write operation (append) to trigger migration
+    let batch = arrow_array::record_batch!(
+        ("id", Int64, [100, 101]),
+        ("values", Utf8, ["value_100", "value_101"])
+    )
+    .unwrap();
+    let dataset = InsertBuilder::new(Arc::new(dataset))
+        .with_params(&WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        })
+        .execute(vec![batch])
+        .await
+        .unwrap();
+
+    // Verify the index now has file sizes after migration
+    let indices = dataset.load_indices().await.unwrap();
+    let index = &indices[0];
+    assert!(
+        index.files.is_some() && !index.files.as_ref().unwrap().is_empty(),
+        "Index should have file size info after migration"
+    );
+
+    // Verify each file has a positive size
+    for file in index.files.as_ref().unwrap() {
+        assert!(
+            file.size_bytes > 0,
+            "File {} should have positive size after migration",
+            file.path
+        );
+    }
+
+    // Verify describe_indices now returns total_size_bytes
+    let descriptions = dataset.describe_indices(None).await.unwrap();
+    assert!(
+        descriptions[0].total_size_bytes().is_some(),
+        "Index should have total_size_bytes after migration"
+    );
+    assert!(
+        descriptions[0].total_size_bytes().unwrap() > 0,
+        "Total size should be positive after migration"
+    );
+}
+
 /// Regression test for issue #5702: project_by_schema should reorder fields inside List<Struct>.
 ///
 /// This test reads a dataset with:
diff --git a/rust/lance/src/dataset/tests/dataset_scanner.rs b/rust/lance/src/dataset/tests/dataset_scanner.rs
index e5e72b48f84..d5dac4c8562 100644
--- a/rust/lance/src/dataset/tests/dataset_scanner.rs
+++ b/rust/lance/src/dataset/tests/dataset_scanner.rs
@@ -9,6 +9,7 @@ use crate::index::vector::VectorIndexParams;
 use lance_arrow::FixedSizeListArrayExt;
 use lance_arrow::json::{JsonArray, is_arrow_json_field, json_field};
 
+use crate::index::DatasetIndexExt;
 use arrow::compute::concat_batches;
 use arrow_array::UInt64Array;
 use arrow_array::{Array, FixedSizeListArray};
@@ -24,7 +25,7 @@ use lance_index::scalar::FullTextSearchQuery;
 use lance_index::scalar::inverted::{
     SCORE_FIELD, query::PhraseQuery, tokenizer::InvertedIndexParams,
 };
-use lance_index::{DatasetIndexExt, IndexType, vector::DIST_COL};
+use lance_index::{IndexType, vector::DIST_COL};
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
 use lance_io::utils::CachedFileSize;
 use lance_linalg::distance::MetricType;
diff --git a/rust/lance/src/dataset/tests/dataset_schema_evolution.rs b/rust/lance/src/dataset/tests/dataset_schema_evolution.rs
index e8e81acf609..d311d62f555 100644
--- a/rust/lance/src/dataset/tests/dataset_schema_evolution.rs
+++ b/rust/lance/src/dataset/tests/dataset_schema_evolution.rs
@@ -4,8 +4,8 @@
 use crate::Dataset;
 use crate::dataset::{NewColumnTransform, WriteMode, WriteParams};
 use arrow_array::{
-    Array, ArrayRef, FixedSizeListArray, Int32Array, ListArray, RecordBatch, RecordBatchIterator,
-    StringArray, StructArray,
+    Array, ArrayRef, FixedSizeListArray, Int32Array, ListArray, NullArray, RecordBatch,
+    RecordBatchIterator, StringArray, StructArray,
 };
 use arrow_schema::{
     DataType, Field as ArrowField, Field, Fields as ArrowFields, Fields, Schema as ArrowSchema,
@@ -549,3 +549,91 @@ async fn prepare_initial_dataset_with_list_struct_col(version: LanceFileVersion)
 
     dataset
 }
+
+/// Reproduces ENT-990: panic in `adjust_child_validity` when reading a dataset where:
+/// - Fragment 0 has `meta.extra: Null` (Arrow infers DataType::Null when the user inserts
+///   rows where every value in `extra` is null, e.g. from Python/pandas with an all-None column)
+/// - Fragment 1 (appended later) has a nullable `meta` struct with null rows, but no `extra`
+///   sub-field (Lance allows this because `extra: Null` is nullable in the dataset schema)
+///
+/// When Fragment 1 is read, Lance adds a `NullReader` for the missing `meta.extra: Null`
+/// sub-field. `MergeStream` calls `RecordBatchExt::merge` on the real batch (with null `meta`
+/// rows) and the `NullReader` batch (all-null `meta` struct). The recursive merge descends into
+/// `meta`, where the parent's null validity is non-empty and the child column has `DataType::Null`
+/// — causing `ArrayData::try_new` to panic.
+#[tokio::test]
+async fn test_scan_with_null_typed_struct_subfield_across_fragments() {
+    // Fragment 0: struct column with an `extra` sub-field of DataType::Null.
+    // This simulates a user inserting rows from Python/pandas where `extra` is all None.
+    let meta0 = StructArray::new(
+        ArrowFields::from(vec![
+            ArrowField::new("name", DataType::Utf8, true),
+            ArrowField::new("extra", DataType::Null, true),
+        ]),
+        vec![
+            Arc::new(StringArray::from(vec![Some("alice"), Some("bob")])) as ArrayRef,
+            Arc::new(NullArray::new(2)) as ArrayRef,
+        ],
+        None,
+    );
+    let schema0 = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, false),
+        ArrowField::new("meta", meta0.data_type().clone(), true),
+    ]));
+    let batch0 = RecordBatch::try_new(
+        schema0.clone(),
+        vec![
+            Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef,
+            Arc::new(meta0) as ArrayRef,
+        ],
+    )
+    .unwrap();
+
+    let mut ds = Dataset::write(
+        RecordBatchIterator::new(vec![Ok(batch0)], schema0),
+        "memory://",
+        Some(WriteParams::default()),
+    )
+    .await
+    .unwrap();
+
+    // Fragment 1: same struct column but WITHOUT `extra`. Lance's `allow_missing_if_nullable`
+    // permits omitting `extra` (it's nullable), so a NullReader will fill it when reading
+    // Fragment 1. The struct has null rows, which is what exposes the bug.
+    let meta1 = StructArray::new(
+        ArrowFields::from(vec![ArrowField::new("name", DataType::Utf8, true)]),
+        vec![Arc::new(StringArray::from(vec![Some("charlie"), None])) as ArrayRef],
+        Some(vec![true, false].into()), // row 1 is a null struct
+    );
+    let schema1 = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, false),
+        ArrowField::new("meta", meta1.data_type().clone(), true),
+    ]));
+    let batch1 = RecordBatch::try_new(
+        schema1.clone(),
+        vec![
+            Arc::new(Int32Array::from(vec![3, 4])) as ArrayRef,
+            Arc::new(meta1) as ArrayRef,
+        ],
+    )
+    .unwrap();
+
+    ds.append(
+        RecordBatchIterator::new(vec![Ok(batch1)], schema1),
+        Some(WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    // Scanning reads both fragments. Fragment 1 is missing `meta.extra: Null`, so Lance adds
+    // a NullReader for it. MergeStream merges the real batch (with null struct rows) and the
+    // NullReader batch (all-null `meta` struct). The recursive merge in `merge()` descends into
+    // `meta`, where `right_validity` (from the all-null NullReader struct) has non-zero null
+    // count and the child column has DataType::Null — previously panicked:
+    // "Arrays of type Null cannot contain a null bitmask".
+    let result = ds.scan().try_into_batch().await.unwrap();
+    assert_eq!(result.num_rows(), 4);
+}
diff --git a/rust/lance/src/dataset/tests/dataset_transactions.rs b/rust/lance/src/dataset/tests/dataset_transactions.rs
index 16d21528562..2b49d0963e1 100644
--- a/rust/lance/src/dataset/tests/dataset_transactions.rs
+++ b/rust/lance/src/dataset/tests/dataset_transactions.rs
@@ -14,13 +14,13 @@ use crate::{Dataset, Result};
 use lance_table::io::commit::ManifestNamingScheme;
 
 use crate::dataset::write::{CommitBuilder, InsertBuilder, WriteMode, WriteParams};
+use crate::index::DatasetIndexExt;
 use arrow_array::Array;
 use arrow_array::RecordBatch;
 use arrow_array::{Int32Array, RecordBatchIterator, StringArray, types::Int32Type};
 use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
 use lance_core::utils::tempfile::{TempDir, TempStrDir};
 use lance_datagen::{BatchCount, RowCount, array};
-use lance_index::DatasetIndexExt;
 
 use crate::datafusion::LanceTableProvider;
 use datafusion::prelude::SessionContext;
@@ -377,3 +377,122 @@ async fn test_inline_transaction() {
     let read_tx = ds_new.read_transaction().await.unwrap().unwrap();
     assert_eq!(read_tx, tx);
 }
+
+#[tokio::test]
+async fn test_list_detached_manifests() {
+    let test_uri = TempStrDir::default();
+
+    // Create initial dataset
+    let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "id",
+        DataType::Int32,
+        false,
+    )]));
+
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+    )
+    .unwrap();
+
+    let dataset = Arc::new(
+        Dataset::write(
+            RecordBatchIterator::new([Ok(batch.clone())], schema.clone()),
+            &test_uri,
+            None,
+        )
+        .await
+        .unwrap(),
+    );
+
+    // Initially there should be no detached manifests
+    let detached = dataset.list_detached_manifests().await.unwrap();
+    assert!(detached.is_empty());
+
+    // Create a detached transaction with properties
+    let mut properties = HashMap::new();
+    properties.insert("detached_key".to_string(), "detached_value".to_string());
+
+    let batch2 = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int32Array::from(vec![4, 5, 6]))],
+    )
+    .unwrap();
+
+    // Use execute_uncommitted + CommitBuilder with_detached(true)
+    let transaction = InsertBuilder::new(dataset.clone())
+        .with_params(&WriteParams {
+            mode: WriteMode::Append,
+            transaction_properties: Some(Arc::new(properties.clone())),
+            ..Default::default()
+        })
+        .execute_uncommitted(vec![batch2])
+        .await
+        .unwrap();
+
+    CommitBuilder::new(dataset.clone())
+        .with_detached(true)
+        .execute(transaction)
+        .await
+        .unwrap();
+
+    // Now there should be one detached manifest
+    let detached = dataset.list_detached_manifests().await.unwrap();
+    assert_eq!(detached.len(), 1);
+
+    // The detached version should have the high bit set
+    let detached_version = detached[0].version;
+    assert!(lance_table::format::is_detached_version(detached_version));
+
+    // We should be able to checkout the detached version and read transaction properties
+    let checked_out = dataset.checkout_version(detached_version).await.unwrap();
+    let tx = checked_out.read_transaction().await.unwrap().unwrap();
+    let tx_props = tx.transaction_properties.unwrap();
+    assert_eq!(
+        tx_props.get("detached_key"),
+        Some(&"detached_value".to_string())
+    );
+
+    // The detached dataset should have more rows
+    assert_eq!(checked_out.count_rows(None).await.unwrap(), 6);
+
+    // Create another detached transaction
+    let batch3 = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int32Array::from(vec![7, 8, 9]))],
+    )
+    .unwrap();
+
+    let mut properties2 = HashMap::new();
+    properties2.insert("second_key".to_string(), "second_value".to_string());
+
+    let transaction2 = InsertBuilder::new(dataset.clone())
+        .with_params(&WriteParams {
+            mode: WriteMode::Append,
+            transaction_properties: Some(Arc::new(properties2)),
+            ..Default::default()
+        })
+        .execute_uncommitted(vec![batch3])
+        .await
+        .unwrap();
+
+    CommitBuilder::new(dataset.clone())
+        .with_detached(true)
+        .execute(transaction2)
+        .await
+        .unwrap();
+
+    // Now there should be two detached manifests
+    let detached = dataset.list_detached_manifests().await.unwrap();
+    assert_eq!(detached.len(), 2);
+
+    // Both should be detached versions
+    for loc in &detached {
+        assert!(lance_table::format::is_detached_version(loc.version));
+    }
+
+    // Regular versions() should not include detached manifests
+    let versions = dataset.versions().await.unwrap();
+    assert_eq!(versions.len(), 1);
+    assert_eq!(versions[0].version, 1);
+}
diff --git a/rust/lance/src/dataset/tests/dataset_versioning.rs b/rust/lance/src/dataset/tests/dataset_versioning.rs
index 024c4254af6..e9253cc69fe 100644
--- a/rust/lance/src/dataset/tests/dataset_versioning.rs
+++ b/rust/lance/src/dataset/tests/dataset_versioning.rs
@@ -161,6 +161,53 @@ async fn test_strict_overwrite() {
         .expect("Unstrict overwrite should succeed when committing to a stale version");
 }
 
+#[tokio::test]
+async fn test_version_id_fast_path() {
+    let test_uri = TempStrDir::default();
+    let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "i",
+        DataType::UInt32,
+        false,
+    )]));
+
+    let data = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(UInt32Array::from_iter_values(0..5))],
+    )
+    .unwrap();
+    let reader = RecordBatchIterator::new(vec![data].into_iter().map(Ok), schema.clone());
+
+    let original = Dataset::write(reader, &test_uri, None).await.unwrap();
+    assert_eq!(original.version_id(), 1);
+    assert_eq!(original.version_id(), original.version().version);
+    assert_eq!(original.latest_version_id().await.unwrap(), 1);
+
+    let data = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(UInt32Array::from_iter_values(5..10))],
+    )
+    .unwrap();
+    let reader = RecordBatchIterator::new(vec![data].into_iter().map(Ok), schema);
+    let updated = Dataset::write(
+        reader,
+        &test_uri,
+        Some(WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+    assert_eq!(updated.version_id(), 2);
+    assert_eq!(updated.version_id(), updated.version().version);
+    assert_eq!(updated.latest_version_id().await.unwrap(), 2);
+
+    let historical = updated.checkout_version(1).await.unwrap();
+    assert_eq!(historical.version_id(), 1);
+    assert_eq!(historical.version_id(), historical.version().version);
+    assert_eq!(historical.latest_version_id().await.unwrap(), 2);
+}
+
 #[rstest]
 #[tokio::test]
 async fn test_restore(
diff --git a/rust/lance/src/dataset/transaction.rs b/rust/lance/src/dataset/transaction.rs
index fc0d4f4dc19..62f30696234 100644
--- a/rust/lance/src/dataset/transaction.rs
+++ b/rust/lance/src/dataset/transaction.rs
@@ -9,41 +9,8 @@
 //! one another. We can also rebuild manifests when retrying committing a
 //! manifest.
 //!
-//! ## Conflict Resolution
-//!
-//! Transactions are compatible with one another if they don't conflict.
-//! Currently, conflict resolution always assumes a Serializable isolation
-//! level.
-//!
-//! Below are the compatibilities between conflicting transactions. The columns
-//! represent the operation that has been applied, while the rows represent the
-//! operation that is being checked for compatibility to see if it can retry.
-//! ✅ indicates that the operation is compatible, while ❌ indicates that it is
-//! a conflict. Some operations have additional conditions that must be met for
-//! them to be compatible.
-//!
-//! NOTE/TODO(rmeng): DataReplacement conflict resolution is not fully implemented
-//!
-//! |                  | Append | Delete / Update | Overwrite/Create | Create Index | Rewrite | Merge | Project | UpdateConfig | DataReplacement |
-//! |------------------|--------|-----------------|------------------|--------------|---------|-------|---------|--------------|-----------------|
-//! | Append           | ✅     | ✅              | ❌                | ✅           | ✅      | ❌     | ❌      | ✅           | ✅
-//! | Delete / Update  | ✅     | 1️⃣              | ❌                | ✅           | 1️⃣      | ❌     | ❌      | ✅           | ✅
-//! | Overwrite/Create | ✅     | ✅              | ✅                | ✅           | ✅      | ✅     | ✅      | 2️⃣           | ✅
-//! | Create index     | ✅     | ✅              | ❌                | ✅           | ✅      | ✅     | ✅      | ✅           | 3️⃣
-//! | Rewrite          | ✅     | 1️⃣              | ❌                | ❌           | 1️⃣      | ❌     | ❌      | ✅           | 3️⃣
-//! | Merge            | ❌     | ❌              | ❌                | ❌           | ✅      | ❌     | ❌      | ✅           | ✅
-//! | Project          | ✅     | ✅              | ❌                | ❌           | ✅      | ❌     | ✅      | ✅           | ✅
-//! | UpdateConfig     | ✅     | ✅              | 2️⃣                | ✅           | ✅      | ✅     | ✅      | 2️⃣           | ✅
-//! | DataReplacement  | ✅     | ✅              | ❌                | 3️⃣           | 1️⃣      | ✅     | 3️⃣      | ✅           | 3️⃣
-//!
-//! 1️⃣ Delete, update, and rewrite are compatible with each other and themselves only if
-//! they affect distinct fragments. Otherwise, they conflict.
-//! 2️⃣ Operations that mutate the config conflict if one of the operations upserts a key
-//! that if referenced by another concurrent operation or if both operations modify the schema
-//! metadata or the same field metadata.
-//! 3️⃣ DataReplacement on a column without index is compatible with any operation AS LONG AS
-//! the operation does not modify the region of the column being replaced.
-//!
+//! For more details please refer to the
+//! [Transaction Specification](https://lance.org/format/table/transaction/#transaction-types).
 
 use super::ManifestWriteConfig;
 use super::write::merge_insert::inserted_rows::KeyExistenceFilter;
@@ -60,7 +27,8 @@ use lance_table::feature_flags::{FLAG_STABLE_ROW_IDS, apply_feature_flags};
 use lance_table::rowids::read_row_ids;
 use lance_table::{
     format::{
-        BasePath, DataFile, DataStorageFormat, Fragment, IndexMetadata, Manifest, RowIdMeta, pb,
+        BasePath, DataFile, DataStorageFormat, Fragment, IndexFile, IndexMetadata, Manifest,
+        RowIdMeta, pb,
     },
     io::{
         commit::CommitHandler,
@@ -1170,6 +1138,9 @@ pub struct RewrittenIndex {
     pub new_id: Uuid,
     pub new_index_details: prost_types::Any,
     pub new_index_version: u32,
+    /// Files in the new index with their sizes.
+    /// Empty list from older writers that didn't persist this field.
+    pub new_index_files: Option<Vec<IndexFile>>,
 }
 
 impl DeepSizeOf for RewrittenIndex {
@@ -1968,13 +1939,17 @@ impl Transaction {
                 removed_indices,
             } => {
                 final_fragments.extend(maybe_existing_fragments?.clone());
+                let removed_uuids = removed_indices
+                    .iter()
+                    .map(|old_index| old_index.uuid)
+                    .collect::<HashSet<_>>();
+                let new_uuids = new_indices
+                    .iter()
+                    .map(|new_index| new_index.uuid)
+                    .collect::<HashSet<_>>();
                 final_indices.retain(|existing_index| {
-                    !new_indices
-                        .iter()
-                        .any(|new_index| new_index.name == existing_index.name)
-                        && !removed_indices
-                            .iter()
-                            .any(|old_index| old_index.uuid == existing_index.uuid)
+                    !removed_uuids.contains(&existing_index.uuid)
+                        && !new_uuids.contains(&existing_index.uuid)
                 });
                 final_indices.extend(new_indices.clone());
             }
@@ -2092,8 +2067,8 @@ impl Transaction {
                     if columns_covered.is_disjoint(&new_file.fields.iter().collect()) {
                         new_frag.add_file(
                             new_file.path.clone(),
-                            new_file.fields.clone(),
-                            new_file.column_indices.clone(),
+                            new_file.fields.to_vec(),
+                            new_file.column_indices.to_vec(),
                             &LanceFileVersion::try_from_major_minor(
                                 new_file.file_major_version,
                                 new_file.file_minor_version,
@@ -2545,6 +2520,10 @@ impl Transaction {
                 groups,
             )?);
             index.uuid = rewritten_index.new_id;
+            // Update file sizes to match the new index files. When not available
+            // (e.g., from older writers), clear the old file sizes to avoid
+            // using stale sizes from the pre-remap index.
+            index.files = rewritten_index.new_index_files.clone();
         }
         Ok(())
     }
@@ -3060,6 +3039,20 @@ impl TryFrom<&pb::transaction::rewrite::RewrittenIndex> for RewrittenIndex {
                 })?
                 .clone(),
             new_index_version: message.new_index_version,
+            new_index_files: if message.new_index_files.is_empty() {
+                None
+            } else {
+                Some(
+                    message
+                        .new_index_files
+                        .iter()
+                        .map(|f| IndexFile {
+                            path: f.path.clone(),
+                            size_bytes: f.size_bytes,
+                        })
+                        .collect(),
+                )
+            },
         })
     }
 }
@@ -3293,6 +3286,19 @@ impl From<&RewrittenIndex> for pb::transaction::rewrite::RewrittenIndex {
             new_id: Some((&value.new_id).into()),
             new_index_details: Some(value.new_index_details.clone()),
             new_index_version: value.new_index_version,
+            new_index_files: value
+                .new_index_files
+                .as_ref()
+                .map(|files| {
+                    files
+                        .iter()
+                        .map(|f| pb::IndexFile {
+                            path: f.path.clone(),
+                            size_bytes: f.size_bytes,
+                        })
+                        .collect()
+                })
+                .unwrap_or_default(),
         }
     }
 }
@@ -3355,7 +3361,12 @@ pub fn validate_operation(manifest: Option<&Manifest>, operation: &Operation) ->
             schema,
             config_upsert_values: None,
             initial_bases: _,
-        } => schema_fragments_valid(Some(manifest), schema, fragments),
+        } => {
+            // Pass None for manifest because Overwrite replaces all fragments.
+            // The old manifest's storage format is irrelevant for validating
+            // the new fragments (e.g., LEGACY→STABLE transitions).
+            schema_fragments_valid(None, schema, fragments)
+        }
         Operation::Update {
             updated_fragments,
             new_fragments,
@@ -3474,7 +3485,37 @@ fn merge_fragments_valid(manifest: &Manifest, new_fragments: &[Fragment]) -> Res
 #[cfg(test)]
 mod tests {
     use super::*;
+    use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
+    use chrono::Utc;
+    use lance_core::datatypes::Schema as LanceSchema;
     use lance_io::utils::CachedFileSize;
+    use std::sync::Arc;
+    use uuid::Uuid;
+
+    fn sample_manifest() -> Manifest {
+        let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]);
+        Manifest::new(
+            LanceSchema::try_from(&schema).unwrap(),
+            Arc::new(vec![Fragment::new(0)]),
+            DataStorageFormat::new(LanceFileVersion::V2_0),
+            HashMap::new(),
+        )
+    }
+
+    fn sample_index_metadata(name: &str) -> IndexMetadata {
+        IndexMetadata {
+            uuid: Uuid::new_v4(),
+            fields: vec![0],
+            name: name.to_string(),
+            dataset_version: 0,
+            fragment_bitmap: Some([0].into_iter().collect()),
+            index_details: None,
+            index_version: 1,
+            created_at: Some(Utc::now()),
+            base_id: None,
+            files: None,
+        }
+    }
 
     #[test]
     fn test_rewrite_fragments() {
@@ -3529,11 +3570,6 @@ mod tests {
 
     #[test]
     fn test_merge_fragments_valid() {
-        use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
-        use lance_core::datatypes::Schema as LanceSchema;
-        use lance_table::format::Manifest;
-        use std::sync::Arc;
-
         // Create a simple schema for testing
         let schema = ArrowSchema::new(vec![
             ArrowField::new("id", DataType::Int32, false),
@@ -3610,6 +3646,82 @@ mod tests {
         assert!(result.is_ok());
     }
 
+    #[test]
+    fn test_create_index_build_manifest_keeps_unremoved_same_name_indices() {
+        let manifest = sample_manifest();
+        let first_index = sample_index_metadata("vector_idx");
+        let second_index = sample_index_metadata("vector_idx");
+        let third_index = sample_index_metadata("vector_idx");
+
+        let transaction = Transaction::new(
+            manifest.version,
+            Operation::CreateIndex {
+                new_indices: vec![third_index.clone()],
+                removed_indices: vec![second_index.clone()],
+            },
+            None,
+        );
+
+        let (_, final_indices) = transaction
+            .build_manifest(
+                Some(&manifest),
+                vec![first_index.clone(), second_index.clone()],
+                "txn",
+                &ManifestWriteConfig::default(),
+            )
+            .unwrap();
+
+        assert_eq!(final_indices.len(), 2);
+        assert!(final_indices.iter().any(|idx| idx.uuid == first_index.uuid));
+        assert!(final_indices.iter().any(|idx| idx.uuid == third_index.uuid));
+        assert!(
+            !final_indices
+                .iter()
+                .any(|idx| idx.uuid == second_index.uuid)
+        );
+    }
+
+    #[test]
+    fn test_create_index_build_manifest_deduplicates_relisted_indices_by_uuid() {
+        let manifest = sample_manifest();
+        let first_index = sample_index_metadata("vector_idx");
+        let second_index = sample_index_metadata("vector_idx");
+        let third_index = sample_index_metadata("vector_idx");
+
+        let transaction = Transaction::new(
+            manifest.version,
+            Operation::CreateIndex {
+                new_indices: vec![first_index.clone(), third_index.clone()],
+                removed_indices: vec![second_index.clone()],
+            },
+            None,
+        );
+
+        let (_, final_indices) = transaction
+            .build_manifest(
+                Some(&manifest),
+                vec![first_index.clone(), second_index.clone()],
+                "txn",
+                &ManifestWriteConfig::default(),
+            )
+            .unwrap();
+
+        assert_eq!(final_indices.len(), 2);
+        assert_eq!(
+            final_indices
+                .iter()
+                .filter(|idx| idx.uuid == first_index.uuid)
+                .count(),
+            1
+        );
+        assert!(final_indices.iter().any(|idx| idx.uuid == third_index.uuid));
+        assert!(
+            !final_indices
+                .iter()
+                .any(|idx| idx.uuid == second_index.uuid)
+        );
+    }
+
     #[test]
     fn test_remove_tombstoned_data_files() {
         // Create a fragment with mixed data files: some normal, some fully tombstoned
@@ -3618,8 +3730,8 @@ mod tests {
         // Add a normal data file with valid field IDs
         fragment.files.push(DataFile {
             path: "normal.lance".to_string(),
-            fields: vec![1, 2, 3],
-            column_indices: vec![],
+            fields: Arc::from([1, 2, 3]),
+            column_indices: Arc::from([]),
             file_major_version: 2,
             file_minor_version: 0,
             file_size_bytes: CachedFileSize::new(1000),
@@ -3629,8 +3741,8 @@ mod tests {
         // Add a data file with all fields tombstoned
         fragment.files.push(DataFile {
             path: "all_tombstoned.lance".to_string(),
-            fields: vec![-2, -2, -2],
-            column_indices: vec![],
+            fields: Arc::from([-2, -2, -2]),
+            column_indices: Arc::from([]),
             file_major_version: 2,
             file_minor_version: 0,
             file_size_bytes: CachedFileSize::new(500),
@@ -3640,8 +3752,8 @@ mod tests {
         // Add a data file with mixed tombstoned and valid fields
         fragment.files.push(DataFile {
             path: "mixed.lance".to_string(),
-            fields: vec![4, -2, 5],
-            column_indices: vec![],
+            fields: Arc::from([4, -2, 5]),
+            column_indices: Arc::from([]),
             file_major_version: 2,
             file_minor_version: 0,
             file_size_bytes: CachedFileSize::new(750),
@@ -3651,8 +3763,8 @@ mod tests {
         // Add another fully tombstoned file
         fragment.files.push(DataFile {
             path: "another_tombstoned.lance".to_string(),
-            fields: vec![-2],
-            column_indices: vec![],
+            fields: Arc::from([-2_i32]),
+            column_indices: Arc::from([]),
             file_major_version: 2,
             file_minor_version: 0,
             file_size_bytes: CachedFileSize::new(250),
@@ -3908,6 +4020,7 @@ mod tests {
             index_version: 1,
             created_at: None,
             base_id: None,
+            files: None,
         }
     }
 
@@ -3929,6 +4042,7 @@ mod tests {
             index_version: 1,
             created_at: None,
             base_id: None,
+            files: None,
         }
     }
 
@@ -4340,6 +4454,7 @@ mod tests {
                 value: vec![],
             },
             new_index_version: 1,
+            new_index_files: None,
         }];
 
         // Should succeed (skip missing index) instead of error
@@ -4347,4 +4462,69 @@ mod tests {
         assert!(result.is_ok());
         assert!(indices.is_empty());
     }
+
+    /// Regression test for https://github.com/lance-format/lance/issues/6417
+    ///
+    /// When overwriting a LEGACY dataset with STABLE-format fragments, the
+    /// validation should not use the old manifest's format. STABLE fragments
+    /// omit struct parent fields, which the strict legacy check rejects.
+    #[test]
+    fn test_overwrite_legacy_to_stable_with_struct_fields() {
+        use arrow_schema::Fields;
+
+        // Schema: id (field 0), name (field 1), address (field 2, struct parent),
+        //   city (field 3), country (field 4)
+        let arrow_schema = ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int32, false),
+            ArrowField::new("name", DataType::Utf8, false),
+            ArrowField::new(
+                "address",
+                DataType::Struct(Fields::from(vec![
+                    ArrowField::new("city", DataType::Utf8, false),
+                    ArrowField::new("country", DataType::Utf8, false),
+                ])),
+                false,
+            ),
+        ]);
+        let schema = LanceSchema::try_from(&arrow_schema).unwrap();
+
+        // Old manifest is LEGACY format
+        let legacy_manifest = Manifest::new(
+            schema.clone(),
+            Arc::new(vec![Fragment::new(0)]),
+            DataStorageFormat::new(LanceFileVersion::Legacy),
+            HashMap::new(),
+        );
+
+        // New fragments in STABLE format omit struct parent field (id=2),
+        // only including leaf fields: id=0, name=1, city=3, country=4
+        let stable_fragment = Fragment {
+            id: 0,
+            files: vec![DataFile::new(
+                "data.lance",
+                vec![0, 1, 3, 4], // no field 2 (struct parent)
+                vec![0, 1, 2, 3],
+                lance_file::format::MAJOR_VERSION as u32,
+                lance_file::format::MINOR_VERSION as u32,
+                None,
+                None,
+            )],
+            physical_rows: Some(10),
+            deletion_file: None,
+            row_id_meta: None,
+            last_updated_at_version_meta: None,
+            created_at_version_meta: None,
+        };
+
+        let operation = Operation::Overwrite {
+            fragments: vec![stable_fragment],
+            schema,
+            config_upsert_values: None,
+            initial_bases: None,
+        };
+
+        // This should succeed — the old manifest's LEGACY format should not
+        // cause strict validation of the new STABLE fragments.
+        validate_operation(Some(&legacy_manifest), &operation).unwrap();
+    }
 }
diff --git a/rust/lance/src/dataset/udtf.rs b/rust/lance/src/dataset/udtf.rs
index d25aec45009..75c0388bc24 100644
--- a/rust/lance/src/dataset/udtf.rs
+++ b/rust/lance/src/dataset/udtf.rs
@@ -256,16 +256,17 @@ impl FtsQueryUDTFBuilder {
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
     use crate::Dataset;
     use crate::dataset::udtf::FtsQueryUDTFBuilder;
+    use crate::index::DatasetIndexExt;
     use arrow_array::{
         Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray, UInt64Array,
     };
     use arrow_schema::{DataType, Field};
     use datafusion::prelude::SessionContext;
+    use lance_index::IndexType;
     use lance_index::scalar::InvertedIndexParams;
-    use lance_index::{DatasetIndexExt, IndexType};
     use std::sync::Arc;
 
     #[tokio::test]
diff --git a/rust/lance/src/dataset/write.rs b/rust/lance/src/dataset/write.rs
index aac88289299..f31b1e2bf2a 100644
--- a/rust/lance/src/dataset/write.rs
+++ b/rust/lance/src/dataset/write.rs
@@ -52,6 +52,7 @@ pub mod merge_insert;
 mod retry;
 pub mod update;
 
+pub use super::progress::{WriteProgressFn, WriteStats};
 pub use commit::CommitBuilder;
 pub use delete::{DeleteBuilder, DeleteResult};
 pub use insert::InsertBuilder;
@@ -132,6 +133,43 @@ impl TryFrom<&str> for WriteMode {
     }
 }
 
+/// The strategy for handling external blob URIs on write.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub enum ExternalBlobMode {
+    /// Store the URI as an external blob reference.
+    #[default]
+    Reference,
+    /// Read the external bytes during write and store them in Lance-managed storage.
+    Ingest,
+}
+
+impl TryFrom<&str> for ExternalBlobMode {
+    type Error = Error;
+
+    fn try_from(value: &str) -> Result<Self> {
+        match value.to_lowercase().as_str() {
+            "reference" => Ok(Self::Reference),
+            "ingest" => Ok(Self::Ingest),
+            _ => Err(Error::invalid_input(format!(
+                "Invalid external blob mode: {}",
+                value
+            ))),
+        }
+    }
+}
+
+fn validate_external_blob_write_params(params: &WriteParams) -> Result<()> {
+    if params.external_blob_mode == ExternalBlobMode::Ingest
+        && params.allow_external_blob_outside_bases
+    {
+        return Err(Error::invalid_input(
+            "allow_external_blob_outside_bases only applies when external_blob_mode=\"reference\"",
+        ));
+    }
+
+    Ok(())
+}
+
 /// Auto cleanup parameters
 #[derive(Debug, Clone)]
 pub struct AutoCleanupParams {
@@ -175,8 +213,17 @@ pub struct WriteParams {
 
     pub store_params: Option<ObjectStoreParams>,
 
+    pub base_store_params: Option<HashMap<String, ObjectStoreParams>>,
+
     pub progress: Arc<dyn WriteFragmentProgress>,
 
+    /// Optional callback invoked after each batch is written.
+    ///
+    /// Receives cumulative [`WriteStats`] so callers can render a progress bar
+    /// or compute throughput. The callback must be cheap and non-blocking;
+    /// spawn a task if you need async work.
+    pub write_progress: Option<WriteProgressFn>,
+
     /// If present, dataset will use this to update the latest version
     ///
     /// If not set, the default will be based on the object store.  Generally this will
@@ -250,6 +297,14 @@ pub struct WriteParams {
     /// Allow writing external blob URIs that cannot be mapped to any registered
     /// non-dataset-root base path. When disabled, such rows are rejected.
     pub allow_external_blob_outside_bases: bool,
+
+    /// The strategy used when writing external blob URIs.
+    pub external_blob_mode: ExternalBlobMode,
+
+    /// Maximum size in bytes for blob v2 pack (.blob) sidecar files.
+    /// When a pack file reaches this size, a new one is started.
+    /// If not set, defaults to 1 GiB.
+    pub blob_pack_file_size_threshold: Option<usize>,
 }
 
 impl Default for WriteParams {
@@ -262,7 +317,9 @@ impl Default for WriteParams {
             max_bytes_per_file: 90 * 1024 * 1024 * 1024, // 90 GB
             mode: WriteMode::Create,
             store_params: None,
+            base_store_params: None,
             progress: Arc::new(NoopFragmentWriteProgress::new()),
+            write_progress: None,
             commit_handler: None,
             data_storage_version: None,
             enable_stable_row_ids: false,
@@ -275,6 +332,8 @@ impl Default for WriteParams {
             target_bases: None,
             target_base_names_or_paths: None,
             allow_external_blob_outside_bases: false,
+            external_blob_mode: ExternalBlobMode::Reference,
+            blob_pack_file_size_threshold: None,
         }
     }
 }
@@ -300,6 +359,21 @@ impl WriteParams {
             .unwrap_or_default()
     }
 
+    /// Set exact runtime object store params for a registered base path.
+    ///
+    /// These params are used as-is for that base. The write-level default
+    /// `store_params` remain the fallback for bases without an explicit binding.
+    pub fn with_base_store_params(
+        mut self,
+        base_path: impl AsRef<str>,
+        store_params: ObjectStoreParams,
+    ) -> Self {
+        self.base_store_params
+            .get_or_insert_with(HashMap::new)
+            .insert(base_path.as_ref().to_string(), store_params);
+        self
+    }
+
     /// Set the properties for this WriteParams.
     pub fn with_transaction_properties(self, properties: HashMap<String, String>) -> Self {
         Self {
@@ -361,6 +435,22 @@ impl WriteParams {
             ..self
         }
     }
+
+    /// Configure how external blob URIs are handled during writes.
+    pub fn with_external_blob_mode(self, mode: ExternalBlobMode) -> Self {
+        Self {
+            external_blob_mode: mode,
+            ..self
+        }
+    }
+
+    /// Set the maximum size in bytes for blob v2 pack (.blob) sidecar files.
+    pub fn with_blob_pack_file_size_threshold(self, max_bytes: usize) -> Self {
+        Self {
+            blob_pack_file_size_threshold: Some(max_bytes),
+            ..self
+        }
+    }
 }
 
 /// Writes the given data to the dataset and returns fragments.
@@ -370,7 +460,7 @@ impl WriteParams {
 /// IDs can be assigned after writing is complete.
 #[deprecated(
     since = "0.20.0",
-    note = "Use [`InsertBuilder::write_uncommitted_stream`] instead"
+    note = "Use [`InsertBuilder::execute_uncommitted_stream`] instead"
 )]
 pub async fn write_fragments(
     dest: impl Into<WriteDestination<'_>>,
@@ -417,6 +507,10 @@ pub async fn do_write_fragments(
     } else {
         None
     };
+    let source_store_registry = dataset
+        .map(|ds| ds.session.store_registry())
+        .unwrap_or_else(|| params.store_registry());
+    let source_store_params = params.store_params.clone().unwrap_or_default();
 
     let writer_generator = WriterGenerator::new(
         object_store,
@@ -426,10 +520,17 @@ pub async fn do_write_fragments(
         target_bases_info,
         external_base_resolver,
         params.allow_external_blob_outside_bases,
+        params.external_blob_mode,
+        source_store_registry,
+        source_store_params,
+        params.blob_pack_file_size_threshold,
     );
     let mut writer: Option<Box<dyn GenericWriter>> = None;
     let mut num_rows_in_current_file = 0;
     let mut fragments = Vec::new();
+    let mut bytes_completed: u64 = 0;
+    let mut rows_completed: u64 = 0;
+    let mut files_written: u32 = 0;
     while let Some(batch_chunk) = buffered_reader.next().await {
         let batch_chunk = batch_chunk?;
 
@@ -441,16 +542,28 @@ pub async fn do_write_fragments(
         }
 
         writer.as_mut().unwrap().write(&batch_chunk).await?;
-        for batch in batch_chunk {
+        for batch in &batch_chunk {
             num_rows_in_current_file += batch.num_rows() as u32;
         }
 
+        if let Some(cb) = &params.write_progress {
+            let current_bytes = writer.as_mut().unwrap().tell().await?;
+            cb.call(WriteStats {
+                bytes_written: bytes_completed + current_bytes,
+                rows_written: rows_completed + num_rows_in_current_file as u64,
+                files_written,
+            });
+        }
+
         if num_rows_in_current_file >= params.max_rows_per_file as u32
             || writer.as_mut().unwrap().tell().await? >= params.max_bytes_per_file as u64
         {
             let (num_rows, data_file) = writer.take().unwrap().finish().await?;
             info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_DATA, path = &data_file.path);
             debug_assert_eq!(num_rows, num_rows_in_current_file);
+            bytes_completed += data_file.file_size_bytes.get().map_or(0, |s| s.get());
+            rows_completed += num_rows as u64;
+            files_written += 1;
             params.progress.complete(fragments.last().unwrap()).await?;
             let last_fragment = fragments.last_mut().unwrap();
             last_fragment.physical_rows = Some(num_rows as usize);
@@ -463,6 +576,16 @@ pub async fn do_write_fragments(
     if let Some(mut writer) = writer.take() {
         let (num_rows, data_file) = writer.finish().await?;
         info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_DATA, path = &data_file.path);
+        bytes_completed += data_file.file_size_bytes.get().map_or(0, |s| s.get());
+        rows_completed += num_rows as u64;
+        files_written += 1;
+        if let Some(cb) = &params.write_progress {
+            cb.call(WriteStats {
+                bytes_written: bytes_completed,
+                rows_written: rows_completed,
+                files_written,
+            });
+        }
         let last_fragment = fragments.last_mut().unwrap();
         last_fragment.physical_rows = Some(num_rows as usize);
         last_fragment.files.push(data_file);
@@ -537,7 +660,6 @@ pub async fn validate_and_resolve_target_bases(
         .unwrap_or_default();
 
     if let Some(target_bases) = &target_base_ids {
-        let store_params = params.store_params.clone().unwrap_or_default();
         let mut bases_info = Vec::new();
 
         for &target_base_id in target_bases {
@@ -548,6 +670,7 @@ pub async fn validate_and_resolve_target_bases(
                 ))
             })?;
 
+            let store_params = write_store_params_for_base(params, &base_path.path);
             let (target_object_store, extracted_path) = ObjectStore::from_uri_and_params(
                 store_registry.clone(),
                 &base_path.path,
@@ -573,6 +696,7 @@ fn append_external_base_candidate(
     base_path: &BasePath,
     store_prefix: String,
     extracted_path: Path,
+    store_params: ObjectStoreParams,
     candidates: &mut Vec<ExternalBaseCandidate>,
     seen_base_ids: &mut HashSet<u32>,
 ) {
@@ -584,29 +708,50 @@ fn append_external_base_candidate(
             base_id: base_path.id,
             store_prefix,
             base_path: extracted_path,
+            store_params,
         });
     }
 }
 
+fn write_store_params_for_base(params: &WriteParams, base_path: &str) -> ObjectStoreParams {
+    params
+        .base_store_params
+        .as_ref()
+        .and_then(|base_store_params| base_store_params.get(base_path))
+        .cloned()
+        .unwrap_or_else(|| params.store_params.clone().unwrap_or_default())
+}
+
+fn dataset_store_params_for_base(dataset: &Dataset, base_path: &str) -> ObjectStoreParams {
+    dataset
+        .base_store_params
+        .as_ref()
+        .and_then(|base_store_params| base_store_params.get(base_path))
+        .cloned()
+        .unwrap_or_else(|| dataset.store_params.as_deref().cloned().unwrap_or_default())
+}
+
 async fn append_external_initial_bases(
     initial_bases: Option<&Vec<BasePath>>,
     store_registry: Arc<ObjectStoreRegistry>,
-    store_params: &ObjectStoreParams,
+    params: &WriteParams,
     candidates: &mut Vec<ExternalBaseCandidate>,
     seen_base_ids: &mut HashSet<u32>,
 ) -> Result<()> {
     if let Some(initial_bases) = initial_bases {
         for base_path in initial_bases {
+            let store_params = write_store_params_for_base(params, &base_path.path);
             let (store, extracted_path) = ObjectStore::from_uri_and_params(
                 store_registry.clone(),
                 &base_path.path,
-                store_params,
+                &store_params,
             )
             .await?;
             append_external_base_candidate(
                 base_path,
                 store.store_prefix.clone(),
                 extracted_path,
+                store_params,
                 candidates,
                 seen_base_ids,
             );
@@ -622,13 +767,13 @@ async fn build_external_base_resolver(
     let store_registry = dataset
         .map(|ds| ds.session.store_registry())
         .unwrap_or_else(|| params.store_registry());
-    let store_params = params.store_params.clone().unwrap_or_default();
 
     let mut seen_base_ids = HashSet::new();
     let mut candidates = vec![];
 
     if let Some(dataset) = dataset {
         for base_path in dataset.manifest.base_paths.values() {
+            let store_params = dataset_store_params_for_base(dataset, &base_path.path);
             let (store, extracted_path) = ObjectStore::from_uri_and_params(
                 store_registry.clone(),
                 &base_path.path,
@@ -639,6 +784,7 @@ async fn build_external_base_resolver(
                 base_path,
                 store.store_prefix.clone(),
                 extracted_path,
+                store_params,
                 &mut candidates,
                 &mut seen_base_ids,
             );
@@ -648,17 +794,13 @@ async fn build_external_base_resolver(
     append_external_initial_bases(
         params.initial_bases.as_ref(),
         store_registry.clone(),
-        &store_params,
+        params,
         &mut candidates,
         &mut seen_base_ids,
     )
     .await?;
 
-    Ok(ExternalBaseResolver::new(
-        candidates,
-        store_registry,
-        store_params,
-    ))
+    Ok(ExternalBaseResolver::new(candidates, store_registry))
 }
 
 /// Writes the given data to the dataset and returns fragments.
@@ -696,6 +838,7 @@ pub async fn write_fragments_internal(
 
     // Make sure the max rows per group is not larger than the max rows per file
     params.max_rows_per_group = std::cmp::min(params.max_rows_per_group, params.max_rows_per_file);
+    validate_external_blob_write_params(&params)?;
 
     let (schema, storage_version) = if let Some(dataset) = dataset {
         match params.mode {
@@ -905,6 +1048,10 @@ struct WriterOptions {
     base_id: Option<u32>,
     external_base_resolver: Option<Arc<ExternalBaseResolver>>,
     allow_external_blob_outside_bases: bool,
+    external_blob_mode: ExternalBlobMode,
+    source_store_registry: Arc<ObjectStoreRegistry>,
+    source_store_params: ObjectStoreParams,
+    blob_pack_file_size_threshold: Option<usize>,
 }
 
 async fn open_writer_with_options(
@@ -919,6 +1066,10 @@ async fn open_writer_with_options(
         base_id,
         external_base_resolver,
         allow_external_blob_outside_bases,
+        external_blob_mode,
+        source_store_registry,
+        source_store_params,
+        blob_pack_file_size_threshold,
     } = options;
 
     let data_file_key = generate_random_filename();
@@ -963,6 +1114,10 @@ async fn open_writer_with_options(
                 schema,
                 external_base_resolver,
                 allow_external_blob_outside_bases,
+                external_blob_mode,
+                source_store_registry,
+                source_store_params,
+                blob_pack_file_size_threshold,
             ))
         } else {
             None
@@ -1002,11 +1157,16 @@ struct WriterGenerator {
     target_bases_info: Option<Vec<TargetBaseInfo>>,
     external_base_resolver: Option<Arc<ExternalBaseResolver>>,
     allow_external_blob_outside_bases: bool,
+    external_blob_mode: ExternalBlobMode,
+    source_store_registry: Arc<ObjectStoreRegistry>,
+    source_store_params: ObjectStoreParams,
+    blob_pack_file_size_threshold: Option<usize>,
     /// Counter for round-robin selection
     next_base_index: AtomicUsize,
 }
 
 impl WriterGenerator {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         object_store: Arc<ObjectStore>,
         base_dir: &Path,
@@ -1015,6 +1175,10 @@ impl WriterGenerator {
         target_bases_info: Option<Vec<TargetBaseInfo>>,
         external_base_resolver: Option<Arc<ExternalBaseResolver>>,
         allow_external_blob_outside_bases: bool,
+        external_blob_mode: ExternalBlobMode,
+        source_store_registry: Arc<ObjectStoreRegistry>,
+        source_store_params: ObjectStoreParams,
+        blob_pack_file_size_threshold: Option<usize>,
     ) -> Self {
         Self {
             object_store,
@@ -1024,6 +1188,10 @@ impl WriterGenerator {
             target_bases_info,
             external_base_resolver,
             allow_external_blob_outside_bases,
+            external_blob_mode,
+            source_store_registry,
+            source_store_params,
+            blob_pack_file_size_threshold,
             next_base_index: AtomicUsize::new(0),
         }
     }
@@ -1054,6 +1222,10 @@ impl WriterGenerator {
                     base_id: Some(base_info.base_id),
                     external_base_resolver: self.external_base_resolver.clone(),
                     allow_external_blob_outside_bases: self.allow_external_blob_outside_bases,
+                    external_blob_mode: self.external_blob_mode,
+                    source_store_registry: self.source_store_registry.clone(),
+                    source_store_params: self.source_store_params.clone(),
+                    blob_pack_file_size_threshold: self.blob_pack_file_size_threshold,
                 },
             )
             .await?
@@ -1068,6 +1240,10 @@ impl WriterGenerator {
                     base_id: None,
                     external_base_resolver: self.external_base_resolver.clone(),
                     allow_external_blob_outside_bases: self.allow_external_blob_outside_bases,
+                    external_blob_mode: self.external_blob_mode,
+                    source_store_registry: self.source_store_registry.clone(),
+                    source_store_params: self.source_store_params.clone(),
+                    blob_pack_file_size_threshold: self.blob_pack_file_size_threshold,
                 },
             )
             .await?
@@ -1149,13 +1325,11 @@ async fn new_source_iter(
 
 struct SpillStreamIter {
     receiver: SpillReceiver,
-    #[allow(dead_code)] // Exists to keep the SpillSender alive
-    sender_handle: tokio::task::JoinHandle<SpillSender>,
+    _sender_handle: tokio::task::JoinHandle<SpillSender>,
     // This temp dir is used to store the spilled data. It is kept alive by
     // this struct. When this struct is dropped, the Drop implementation of
     // tempfile::TempDir will delete the temp dir.
-    #[allow(dead_code)] // Exists to keep the temp dir alive
-    tmp_dir: TempDir,
+    _tmp_dir: TempDir,
 }
 
 impl SpillStreamIter {
@@ -1199,8 +1373,8 @@ impl SpillStreamIter {
 
         Ok(Self {
             receiver,
-            tmp_dir,
-            sender_handle,
+            _tmp_dir: tmp_dir,
+            _sender_handle: sender_handle,
         })
     }
 }
@@ -1216,6 +1390,7 @@ impl Iterator for SpillStreamIter {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::collections::HashMap;
 
     use arrow_array::{Int32Array, RecordBatchIterator, RecordBatchReader, StructArray};
     use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema};
@@ -1224,7 +1399,9 @@ mod tests {
     use futures::TryStreamExt;
     use lance_datagen::{BatchCount, RowCount, array, gen_batch};
     use lance_file::previous::reader::FileReader as PreviousFileReader;
+    use lance_io::object_store::StorageOptionsAccessor;
     use lance_io::traits::Reader;
+    use lance_table::format::BasePath;
 
     #[tokio::test]
     async fn test_chunking_large_batches() {
@@ -1647,7 +1824,7 @@ mod tests {
         assert_eq!(fragments.len(), 1);
         let fragment = &fragments[0];
         assert_eq!(fragment.files.len(), 1);
-        assert_eq!(fragment.files[0].fields, vec![0, 1, 3]);
+        assert_eq!(fragment.files[0].fields.as_ref(), &[0, 1, 3]);
 
         let path = base_path
             .child(DATA_DIR)
@@ -1670,6 +1847,65 @@ mod tests {
         assert_eq!(batch, data);
     }
 
+    #[cfg(feature = "azure")]
+    fn azure_store_params(account_name: &str) -> ObjectStoreParams {
+        ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("account_name".to_string(), account_name.to_string()),
+                    ("account_key".to_string(), "dGVzdA==".to_string()),
+                ]),
+            ))),
+            ..Default::default()
+        }
+    }
+
+    #[cfg(feature = "azure")]
+    #[tokio::test]
+    async fn test_validate_and_resolve_target_bases_uses_base_store_params() {
+        let mut params = WriteParams::default()
+            .with_target_bases(vec![1, 2])
+            .with_base_store_params("az://container/path-a", azure_store_params("account-a"))
+            .with_base_store_params("az://container/path-b", azure_store_params("account-b"));
+
+        let existing_base_paths = HashMap::from([
+            (
+                1,
+                BasePath::new(
+                    1,
+                    "az://container/path-a".to_string(),
+                    Some("base-a".to_string()),
+                    false,
+                ),
+            ),
+            (
+                2,
+                BasePath::new(
+                    2,
+                    "az://container/path-b".to_string(),
+                    Some("base-b".to_string()),
+                    false,
+                ),
+            ),
+        ]);
+
+        let target_bases =
+            validate_and_resolve_target_bases(&mut params, Some(&existing_base_paths))
+                .await
+                .unwrap()
+                .unwrap();
+
+        assert_eq!(target_bases.len(), 2);
+        assert_eq!(
+            target_bases[0].object_store.store_prefix,
+            "az$container@account-a"
+        );
+        assert_eq!(
+            target_bases[1].object_store.store_prefix,
+            "az$container@account-b"
+        );
+    }
+
     #[tokio::test]
     async fn test_explicit_data_file_bases_writer_generator() {
         use arrow::datatypes::{DataType, Field as ArrowField, Schema as ArrowSchema};
@@ -1703,6 +1939,10 @@ mod tests {
             Some(target_bases),
             None,
             false,
+            ExternalBlobMode::Reference,
+            Arc::new(ObjectStoreRegistry::default()),
+            ObjectStoreParams::default(),
+            None,
         );
 
         // Create a writer
@@ -1817,6 +2057,10 @@ mod tests {
             Some(target_bases),
             None,
             false,
+            ExternalBlobMode::Reference,
+            Arc::new(ObjectStoreRegistry::default()),
+            ObjectStoreParams::default(),
+            None,
         );
 
         // Create test batch
@@ -1845,6 +2089,11 @@ mod tests {
         let test_cases = vec![
             ("s3://multi-path-test/test1/subBucket2", "test1/subBucket2"),
             ("gs://my-bucket/path/to/data", "path/to/data"),
+            ("az://container/path/to/data", "path/to/data"),
+            (
+                "abfss://filesystem@account.dfs.core.windows.net/path/to/data",
+                "path/to/data",
+            ),
             ("file:///tmp/test/bucket", "tmp/test/bucket"),
         ];
 
@@ -1873,6 +2122,18 @@ mod tests {
                     path: "s3://bucket2/path2".to_string(),
                     is_dataset_root: true,
                 },
+                BasePath {
+                    id: 3,
+                    name: Some("azure-az-base".to_string()),
+                    path: "az://container/path1".to_string(),
+                    is_dataset_root: true,
+                },
+                BasePath {
+                    id: 4,
+                    name: Some("azure-abfss-base".to_string()),
+                    path: "abfss://filesystem@account.dfs.core.windows.net/path1".to_string(),
+                    is_dataset_root: true,
+                },
             ]),
             target_bases: Some(vec![1]), // Use ID 1 which corresponds to bucket1
             ..Default::default()
@@ -1895,6 +2156,8 @@ mod tests {
     }
 
     fn validate_write_params(params: &WriteParams) -> Result<()> {
+        validate_external_blob_write_params(params)?;
+
         // Replicate the validation logic from the main write function
         if matches!(params.mode, WriteMode::Create)
             && let Some(target_bases) = &params.target_bases
@@ -1922,6 +2185,21 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn test_external_blob_mode_validation() {
+        let params = WriteParams {
+            external_blob_mode: ExternalBlobMode::Ingest,
+            allow_external_blob_outside_bases: true,
+            ..Default::default()
+        };
+
+        let err = validate_write_params(&params).unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("allow_external_blob_outside_bases only applies")
+        );
+    }
+
     #[tokio::test]
     async fn test_multi_base_create() {
         use lance_testing::datagen::{BatchGenerator, IncrementingInt32};
diff --git a/rust/lance/src/dataset/write/commit.rs b/rust/lance/src/dataset/write/commit.rs
index 0a85d27ce99..08a4f8f8f02 100644
--- a/rust/lance/src/dataset/write/commit.rs
+++ b/rust/lance/src/dataset/write/commit.rs
@@ -193,9 +193,10 @@ impl<'a> CommitBuilder<'a> {
                 dataset.commit_handler.clone(),
             ),
             WriteDestination::Uri(uri) => {
-                let commit_handler = if self.commit_handler.is_some() && self.object_store.is_some()
+                let commit_handler = if let (Some(_), Some(commit_handler)) =
+                    (&self.object_store, &self.commit_handler)
                 {
-                    self.commit_handler.as_ref().unwrap().clone()
+                    commit_handler.clone()
                 } else {
                     resolve_commit_handler(uri, self.commit_handler.clone(), &self.store_params)
                         .await?
@@ -403,6 +404,7 @@ impl<'a> CommitBuilder<'a> {
                     metadata_cache,
                     file_reader_options: None,
                     store_params: self.store_params.clone().map(Box::new),
+                    base_store_params: None,
                 })
             }
         }
@@ -485,8 +487,8 @@ mod tests {
             id: 0,
             files: vec![DataFile {
                 path: "file.lance".to_string(),
-                fields: vec![0],
-                column_indices: vec![0],
+                fields: Arc::from([0]),
+                column_indices: Arc::from([0]),
                 file_major_version: major_version,
                 file_minor_version: minor_version,
                 file_size_bytes: CachedFileSize::new(100),
diff --git a/rust/lance/src/dataset/write/delete.rs b/rust/lance/src/dataset/write/delete.rs
index 18916b087f5..c8d369dab63 100644
--- a/rust/lance/src/dataset/write/delete.rs
+++ b/rust/lance/src/dataset/write/delete.rs
@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 use crate::dataset::rowids::get_row_id_index;
+use crate::dataset::scanner::ExprFilter;
 use crate::{
     Dataset,
     dataset::transaction::{Operation, Transaction},
@@ -104,17 +105,27 @@ async fn apply_deletions(
 #[derive(Debug, Clone)]
 pub struct DeleteBuilder {
     dataset: Arc<Dataset>,
-    predicate: String,
+    filter: ExprFilter,
     conflict_retries: u32,
     retry_timeout: Duration,
 }
 
 impl DeleteBuilder {
-    /// Create a new DeleteBuilder
+    /// Create a new DeleteBuilder with a SQL predicate string
     pub fn new(dataset: Arc<Dataset>, predicate: impl Into<String>) -> Self {
         Self {
             dataset,
-            predicate: predicate.into(),
+            filter: ExprFilter::Sql(predicate.into()),
+            conflict_retries: 10,
+            retry_timeout: Duration::from_secs(30),
+        }
+    }
+
+    /// Create a new DeleteBuilder with a DataFusion expression filter
+    pub fn from_expr(dataset: Arc<Dataset>, expr: Expr) -> Self {
+        Self {
+            dataset,
+            filter: ExprFilter::Datafusion(expr),
             conflict_retries: 10,
             retry_timeout: Duration::from_secs(30),
         }
@@ -136,7 +147,7 @@ impl DeleteBuilder {
     pub async fn execute(self) -> Result<DeleteResult> {
         let job = DeleteJob {
             dataset: self.dataset.clone(),
-            predicate: self.predicate,
+            filter: self.filter,
         };
 
         let config = RetryConfig {
@@ -152,7 +163,7 @@ impl DeleteBuilder {
 #[derive(Debug, Clone)]
 struct DeleteJob {
     dataset: Arc<Dataset>,
-    predicate: String,
+    filter: ExprFilter,
 }
 
 /// Data returned by delete operation
@@ -170,10 +181,18 @@ impl RetryExecutor for DeleteJob {
     async fn execute_impl(&self) -> Result<Self::Data> {
         // Create a single scanner for the entire dataset
         let mut scanner = self.dataset.scan();
-        scanner
-            .with_row_id()
-            .project(&[ROW_ID])?
-            .filter(&self.predicate)?;
+        scanner.with_row_id().project(&[ROW_ID])?;
+        match &self.filter {
+            ExprFilter::Sql(s) => {
+                scanner.filter(s)?;
+            }
+            ExprFilter::Datafusion(expr) => {
+                scanner.filter_expr(expr.clone());
+            }
+            ExprFilter::Substrait(_) => {
+                unreachable!("Substrait filters are not supported in DeleteBuilder")
+            }
+        }
 
         // Check if the filter optimized to true (delete everything) or false (delete nothing)
         let (updated_fragments, deleted_fragment_ids, affected_rows, num_deleted_rows) =
@@ -247,10 +266,17 @@ impl RetryExecutor for DeleteJob {
 
     async fn commit(&self, dataset: Arc<Dataset>, data: Self::Data) -> Result<Self::Result> {
         let num_deleted_rows = data.num_deleted_rows;
+        let predicate = match &self.filter {
+            ExprFilter::Sql(s) => s.clone(),
+            ExprFilter::Datafusion(expr) => expr.to_string(),
+            ExprFilter::Substrait(_) => {
+                unreachable!("Substrait filters are not supported in DeleteBuilder")
+            }
+        };
         let operation = Operation::Delete {
             updated_fragments: data.updated_fragments,
             deleted_fragment_ids: data.deleted_fragment_ids,
-            predicate: self.predicate.clone(),
+            predicate,
         };
         let transaction = Transaction::new(dataset.manifest.version, operation, None);
 
@@ -288,6 +314,7 @@ mod tests {
     use super::*;
     use crate::dataset::{InsertBuilder, UpdateBuilder};
     use crate::dataset::{WriteMode, WriteParams};
+    use crate::index::DatasetIndexExt;
     use crate::utils::test::TestDatasetGenerator;
     use arrow::array::AsArray;
     use arrow::datatypes::UInt32Type;
@@ -296,7 +323,7 @@ mod tests {
     use futures::TryStreamExt;
     use lance_core::utils::tempfile::TempStrDir;
     use lance_file::version::LanceFileVersion;
-    use lance_index::{DatasetIndexExt, IndexType, scalar::ScalarIndexParams};
+    use lance_index::{IndexType, scalar::ScalarIndexParams};
     use rstest::rstest;
     use std::collections::HashSet;
     use std::ops::Range;
@@ -820,7 +847,7 @@ mod tests {
         let dataset_arc = Arc::new(dataset);
         let delete_job = DeleteJob {
             dataset: dataset_arc.clone(),
-            predicate: "true".to_string(),
+            filter: ExprFilter::Sql("true".to_string()),
         };
         let delete_data = delete_job.execute_impl().await.unwrap();
 
@@ -864,4 +891,37 @@ mod tests {
         // All rows should be deleted, including the updated ones
         assert_eq!(final_result.new_dataset.count_rows(None).await.unwrap(), 0);
     }
+
+    #[tokio::test]
+    async fn test_delete_with_expr_filter() {
+        use datafusion::prelude::{col, lit};
+
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "i",
+            DataType::UInt32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(UInt32Array::from_iter_values(0..100u32))],
+        )
+        .unwrap();
+
+        let mut dataset = InsertBuilder::new("memory://")
+            .execute(vec![batch])
+            .await
+            .unwrap();
+
+        // Delete rows where i < 10 using an Expr filter
+        let expr = col("i").lt(lit(10u32));
+        let result = DeleteBuilder::from_expr(Arc::new(dataset.clone()), expr)
+            .execute()
+            .await
+            .unwrap();
+
+        assert_eq!(result.num_deleted_rows, 10);
+
+        dataset.checkout_latest().await.unwrap();
+        assert_eq!(dataset.count_rows(None).await.unwrap(), 90);
+    }
 }
diff --git a/rust/lance/src/dataset/write/insert.rs b/rust/lance/src/dataset/write/insert.rs
index 3c9d01fe49b..20209ed7f30 100644
--- a/rust/lance/src/dataset/write/insert.rs
+++ b/rust/lance/src/dataset/write/insert.rs
@@ -31,6 +31,8 @@ use super::WriteMode;
 use super::WriteParams;
 use super::commit::CommitBuilder;
 use super::resolve_commit_handler;
+use crate::dataset::progress::{WriteProgressFn, WriteStats};
+
 /// Insert or create a new dataset.
 ///
 /// There are different variants of `execute()` methods. Those with the `_stream`
@@ -46,6 +48,7 @@ pub struct InsertBuilder<'a> {
     dest: WriteDestination<'a>,
     // TODO: make these parameters a part of the builder, and add specific methods.
     params: Option<&'a WriteParams>,
+    write_progress: Option<WriteProgressFn>,
 }
 
 impl<'a> InsertBuilder<'a> {
@@ -53,6 +56,7 @@ impl<'a> InsertBuilder<'a> {
         Self {
             dest: dest.into(),
             params: None,
+            write_progress: None,
         }
     }
 
@@ -61,6 +65,18 @@ impl<'a> InsertBuilder<'a> {
         self
     }
 
+    /// Register a callback that is invoked after each batch of rows is written.
+    ///
+    /// The callback receives cumulative [`WriteStats`] and can be used to drive
+    /// a progress bar or compute throughput. It must be cheap and non-blocking;
+    /// spawn a task if you need async work.
+    ///
+    /// This overrides any `write_progress` set in [`WriteParams`].
+    pub fn progress(mut self, callback: impl Fn(WriteStats) + Send + Sync + 'static) -> Self {
+        self.write_progress = Some(WriteProgressFn::new(callback));
+        self
+    }
+
     /// Execute the insert operation with the given data.
     ///
     /// This writes the data fragments and commits them into the dataset.
@@ -331,7 +347,10 @@ impl<'a> InsertBuilder<'a> {
     }
 
     async fn resolve_context(&self) -> Result<WriteContext<'a>> {
-        let params = self.params.cloned().unwrap_or_default();
+        let mut params = self.params.cloned().unwrap_or_default();
+        if let Some(cb) = self.write_progress.clone() {
+            params.write_progress = Some(cb);
+        }
         let (object_store, base_path, commit_handler) = match &self.dest {
             WriteDestination::Dataset(dataset) => (
                 dataset.object_store.clone(),
@@ -635,4 +654,58 @@ mod test {
             }
         }
     }
+
+    #[tokio::test]
+    async fn test_write_progress_callback() {
+        use std::sync::Mutex;
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        // Three batches of 100 rows each.
+        let batches: Vec<_> = (0..3)
+            .map(|_| {
+                RecordBatch::try_new(
+                    schema.clone(),
+                    vec![Arc::new(Int32Array::from(vec![0i32; 100]))],
+                )
+                .unwrap()
+            })
+            .collect();
+
+        let stats_log: Arc<Mutex<Vec<crate::dataset::WriteStats>>> =
+            Arc::new(Mutex::new(Vec::new()));
+        let log_clone = stats_log.clone();
+
+        InsertBuilder::new("memory://test_write_progress")
+            .progress(move |stats| {
+                log_clone.lock().unwrap().push(stats);
+            })
+            .execute_stream(RecordBatchIterator::new(
+                batches.into_iter().map(Ok),
+                schema,
+            ))
+            .await
+            .unwrap();
+
+        let log = stats_log.lock().unwrap();
+        assert!(
+            !log.is_empty(),
+            "progress callback must be called at least once"
+        );
+        // bytes_written and rows_written must be monotonically non-decreasing.
+        for window in log.windows(2) {
+            assert!(
+                window[1].bytes_written >= window[0].bytes_written,
+                "bytes_written must not decrease: {:?} -> {:?}",
+                window[0].bytes_written,
+                window[1].bytes_written,
+            );
+            assert!(
+                window[1].rows_written >= window[0].rows_written,
+                "rows_written must not decrease",
+            );
+        }
+        let last = log.last().unwrap();
+        assert!(last.bytes_written > 0, "final bytes_written must be > 0");
+        assert_eq!(last.rows_written, 300, "all 300 rows must be reported");
+        assert_eq!(last.files_written, 1, "a single file should be written");
+    }
 }
diff --git a/rust/lance/src/dataset/write/merge_insert.rs b/rust/lance/src/dataset/write/merge_insert.rs
index a0f58b78ae2..8ea25c7285f 100644
--- a/rust/lance/src/dataset/write/merge_insert.rs
+++ b/rust/lance/src/dataset/write/merge_insert.rs
@@ -18,6 +18,23 @@
 
 // Internal column name for the merge action. Using "__action" to avoid collisions with user columns.
 const MERGE_ACTION_COLUMN: &str = "__action";
+// ## NULL-safe source row detection via sentinel column
+//
+// The merge join uses standard SQL equality for ON columns, which means NULL != NULL.
+// After an outer join we need to know for each output row whether it came from the
+// source side, the target side, or both.  The naive approach — checking whether an ON
+// column IS NOT NULL — is wrong: a source row whose ON column is legitimately NULL is
+// indistinguishable from a NULL introduced by the outer join on the target side.
+//
+// Solution: inject a `lit(true)` sentinel into every source row *before* the join.
+// After the join:
+//   - source rows (matched or unmatched)  → sentinel = true   (never NULL)
+//   - target-only rows                    → sentinel = NULL   (outer-join fill)
+//
+// `assign_action` then uses `sentinel IS NOT NULL` instead of key-column IS NOT NULL
+// to determine which side each row came from.  The sentinel is stripped by
+// `prepare_stream_schema` and never written to the dataset.
+pub(super) const MERGE_SOURCE_SENTINEL: &str = "__merge_source_sentinel";
 
 pub mod inserted_rows;
 
@@ -29,6 +46,7 @@ use super::{CommitBuilder, WriteParams, write_fragments_internal};
 use crate::dataset::rowids::get_row_id_index;
 use crate::dataset::transaction::UpdateMode::{RewriteColumns, RewriteRows};
 use crate::dataset::utils::CapturedRowIds;
+use crate::index::DatasetIndexExt;
 use crate::{
     Dataset,
     datafusion::dataframe::SessionContextExt,
@@ -94,8 +112,8 @@ use lance_datafusion::{
     utils::StreamingWriteSource,
 };
 use lance_file::version::LanceFileVersion;
+use lance_index::IndexCriteria;
 use lance_index::mem_wal::MergedGeneration;
-use lance_index::{DatasetIndexExt, IndexCriteria};
 use lance_table::format::{Fragment, IndexMetadata, RowIdMeta};
 use log::info;
 use roaring::RoaringTreemap;
@@ -846,25 +864,28 @@ impl MergeInsertJob {
         &self,
         source: SendableRecordBatchStream,
     ) -> Result<SendableRecordBatchStream> {
-        // We need to do a full index scan if we're deleting source data
-        let can_use_scalar_index = matches!(
-            self.params.delete_not_matched_by_source, // this value marks behavior for rows in target that are not matched by the source. Value assigned earlier.
-            WhenNotMatchedBySource::Keep
-        ) && self.params.use_index;
-
-        if can_use_scalar_index {
+        if self.params.use_index
+            && matches!(
+                self.params.delete_not_matched_by_source,
+                WhenNotMatchedBySource::Keep
+            )
+        {
             // keeping unmatched rows, no deletion
             if let Some(index) = self.join_key_as_scalar_index().await? {
-                self.create_indexed_scan_joined_stream(source, index).await
-            } else {
-                self.create_full_table_joined_stream(source).await
+                return self.create_indexed_scan_joined_stream(source, index).await;
             }
-        } else {
+        }
+
+        if !matches!(
+            self.params.delete_not_matched_by_source,
+            WhenNotMatchedBySource::Keep
+        ) {
             info!(
                 "The merge insert operation is configured to delete rows from the target table, this requires a potentially costly full table scan"
             );
-            self.create_full_table_joined_stream(source).await
         }
+
+        self.create_full_table_joined_stream(source).await
     }
 
     async fn update_fragments(
@@ -1243,12 +1264,19 @@ impl MergeInsertJob {
             let updated_fields = fragment.files.last().unwrap().fields.clone();
             all_fields_updated.extend(updated_fields.iter().map(|&f| f as u32));
             for data_file in &mut fragment.files.iter_mut().rev().skip(1) {
-                for field in &mut data_file.fields {
-                    if updated_fields.contains(field) {
-                        // Tombstone these fields
-                        *field = -2;
-                    }
-                }
+                let new_fields: Arc<[i32]> = data_file
+                    .fields
+                    .iter()
+                    .map(|field| {
+                        if updated_fields.contains(field) {
+                            -2 // Tombstone
+                        } else {
+                            *field
+                        }
+                    })
+                    .collect::<Vec<_>>()
+                    .into();
+                data_file.fields = new_fields;
             }
         }
 
@@ -1299,6 +1327,21 @@ impl MergeInsertJob {
         self.execute_uncommitted_impl(stream).await
     }
 
+    fn create_plan_join_type(&self) -> JoinType {
+        let keep_unmatched_source_rows = self.params.insert_not_matched;
+        let keep_unmatched_target_rows = !matches!(
+            self.params.delete_not_matched_by_source,
+            WhenNotMatchedBySource::Keep
+        );
+
+        match (keep_unmatched_target_rows, keep_unmatched_source_rows) {
+            (false, false) => JoinType::Inner,
+            (false, true) => JoinType::Right,
+            (true, false) => JoinType::Left,
+            (true, true) => JoinType::Full,
+        }
+    }
+
     async fn create_plan(
         self,
         source: SendableRecordBatchStream,
@@ -1320,13 +1363,16 @@ impl MergeInsertJob {
             .collect::<Vec<_>>();
         let on_cols_refs = on_cols.iter().map(|s| s.as_str()).collect::<Vec<_>>();
         let source_df = session_ctx.read_one_shot(source)?;
+        // Inject a sentinel literal column so we can reliably determine, after the join,
+        // whether the source side contributed a row.  This is NULL-safe: even when every
+        // ON column is NULL the sentinel lets us distinguish a source-only row from a
+        // target-only row (where the sentinel is filled with NULL by the outer join).
+        let source_df = source_df
+            .with_column(MERGE_SOURCE_SENTINEL, logical_expr::lit(true))
+            .map_err(crate::Error::from)?;
         let source_df_aliased = source_df.alias("source")?;
         let scan_aliased = scan.alias("target")?;
-        let join_type = if self.params.insert_not_matched {
-            JoinType::Right
-        } else {
-            JoinType::Inner
-        };
+        let join_type = self.create_plan_join_type();
         let dataset_schema: Schema = self.dataset.schema().into();
         let df = scan_aliased
             .join(
@@ -1444,9 +1490,9 @@ impl MergeInsertJob {
     ///
     /// The fast path is only available for specific conditions:
     /// - when_matched is UpdateAll or UpdateIf or Fail
-    /// - Either use_index is false OR there's no scalar index on join key
+    /// - The execution will not use the legacy scalar-index join path
     /// - Source schema matches dataset schema exactly
-    /// - when_not_matched_by_source is Keep
+    /// - when_not_matched_by_source is Keep, Delete, or DeleteIf
     async fn can_use_create_plan(&self, source_schema: &Schema) -> Result<bool> {
         // Convert to lance schema for comparison
         let lance_schema = lance_core::datatypes::Schema::try_from(source_schema)?;
@@ -1463,7 +1509,15 @@ impl MergeInsertJob {
             },
         );
 
-        let has_scalar_index = self.join_key_as_scalar_index().await?.is_some();
+        let would_use_scalar_index = if self.params.use_index
+            && matches!(
+                self.params.delete_not_matched_by_source,
+                WhenNotMatchedBySource::Keep
+            ) {
+            self.join_key_as_scalar_index().await?.is_some()
+        } else {
+            false
+        };
 
         // Check if this is a delete-only operation (no update/insert writes needed from source)
         // For delete-only, we don't need the full source schema, just key columns for matching
@@ -1487,11 +1541,13 @@ impl MergeInsertJob {
                 | WhenMatched::UpdateIf(_)
                 | WhenMatched::Fail
                 | WhenMatched::Delete
-        ) && (!self.params.use_index || !has_scalar_index)
+        ) && !would_use_scalar_index
             && schema_ok
             && matches!(
                 self.params.delete_not_matched_by_source,
                 WhenNotMatchedBySource::Keep
+                    | WhenNotMatchedBySource::Delete
+                    | WhenNotMatchedBySource::DeleteIf(_)
             ))
     }
 
@@ -1733,7 +1789,7 @@ impl MergeInsertJob {
 
         // Check if we can use create_plan
         if !self.can_use_create_plan(&schema).await? {
-            return Err(Error::not_supported_source("This merge insert configuration does not support explain_plan. Only upsert operations with full schema, no scalar index, and keeping unmatched rows are supported.".into()));
+            return Err(Error::not_supported_source("This merge insert configuration does not support explain_plan. Only full-schema merge insert operations without a scalar-index execution path are currently supported.".into()));
         }
 
         // Create an empty batch with the provided schema to pass to create_plan
@@ -1773,7 +1829,7 @@ impl MergeInsertJob {
     pub async fn analyze_plan(&self, source: SendableRecordBatchStream) -> Result<String> {
         // Check if we can use create_plan
         if !self.can_use_create_plan(source.schema().as_ref()).await? {
-            return Err(Error::not_supported_source("This merge insert configuration does not support analyze_plan. Only upsert operations with full schema, no scalar index, and keeping unmatched rows are supported.".into()));
+            return Err(Error::not_supported_source("This merge insert configuration does not support analyze_plan. Only full-schema merge insert operations without a scalar-index execution path are currently supported.".into()));
         }
 
         // Clone self since create_plan consumes the job
@@ -2222,7 +2278,6 @@ mod tests {
             assert_plan_node_equals, assert_string_matches,
         },
     };
-    use arrow_array::RecordBatch;
     use arrow_array::builder::{ListBuilder, StringBuilder};
     use arrow_array::types::Float32Type;
     use arrow_array::{
@@ -2230,6 +2285,7 @@ mod tests {
         RecordBatchIterator, RecordBatchReader, StringArray, StructArray, UInt32Array,
         types::{Int32Type, UInt32Type},
     };
+    use arrow_array::{RecordBatch, record_batch};
     use arrow_buffer::{OffsetBuffer, ScalarBuffer};
     use arrow_schema::{DataType, Field, Schema};
     use arrow_select::concat::concat_batches;
@@ -2716,6 +2772,18 @@ mod tests {
         check_then_refresh_dataset(new_batch.clone(), job, &[], &[4, 5, 6, 7, 8, 9], &[3, 3, 3])
             .await;
 
+        // conditional upsert, with delete all
+        let job = MergeInsertBuilder::try_new(ds.clone(), keys.clone())
+            .unwrap()
+            .when_matched(
+                WhenMatched::update_if(&ds, "source.filterme != target.filterme").unwrap(),
+            )
+            .when_not_matched_by_source(WhenNotMatchedBySource::Delete)
+            .try_build()
+            .unwrap();
+        check_then_refresh_dataset(new_batch.clone(), job, &[4, 5], &[6, 7, 8, 9], &[3, 1, 3])
+            .await;
+
         // update only, with delete all (unusual)
         let job = MergeInsertBuilder::try_new(ds.clone(), keys.clone())
             .unwrap()
@@ -2768,6 +2836,24 @@ mod tests {
         )
         .await;
 
+        // conditional upsert, with delete some
+        let job = MergeInsertBuilder::try_new(ds.clone(), keys.clone())
+            .unwrap()
+            .when_matched(
+                WhenMatched::update_if(&ds, "source.filterme != target.filterme").unwrap(),
+            )
+            .when_not_matched_by_source(WhenNotMatchedBySource::DeleteIf(condition.clone()))
+            .try_build()
+            .unwrap();
+        check_then_refresh_dataset(
+            new_batch.clone(),
+            job,
+            &[1, 4, 5],
+            &[6, 7, 8, 9],
+            &[3, 1, 2],
+        )
+        .await;
+
         // update only, witxh delete some (unusual)
         let job = MergeInsertBuilder::try_new(ds.clone(), keys.clone())
             .unwrap()
@@ -3274,7 +3360,7 @@ mod tests {
 
         // Sample 2048 random indices and then paste on a column of 9999999's
         let some_indices = ds
-            .sample(2048, &(&just_index_col).try_into().unwrap())
+            .sample(2048, &(&just_index_col).try_into().unwrap(), None)
             .await
             .unwrap();
         let some_indices = some_indices.column(0).clone();
@@ -3539,8 +3625,8 @@ mod tests {
                 let data_files = &frag.files;
                 // Updated columns should be only columns in new data files
                 // -2 field ids are tombstoned.
-                assert_eq!(&data_files[0].fields, &[0, -2, -2]);
-                assert_eq!(&data_files[1].fields, &[2, 1]);
+                assert_eq!(data_files[0].fields.as_ref(), &[0, -2, -2]);
+                assert_eq!(data_files[1].fields.as_ref(), &[2, 1]);
             };
             has_added_files(&fragments_after[1]);
             has_added_files(&fragments_after[2]);
@@ -4165,21 +4251,22 @@ mod tests {
         // Assert the plan structure using portable plan matching
         // The optimized plan should have:
         // 1. FullSchemaMergeInsertExec at the top
-        // 2. ProjectionExec that creates action with key validation (source.key IS NOT NULL)
-        // 3. ProjectionExec that creates the common expression for key validation
-        // 4. HashJoin with projection optimization
-        // 5. LanceScan that only reads the key column (projection pushdown working!)
+        // 2. ProjectionExec that creates action based on _rowaddr nullness (sentinel is constant
+        //    true so DataFusion folds `sentinel IS NOT NULL` away from the CASE expression)
+        // 3. HashJoin with projection that includes the sentinel column
+        // 4. LanceScan that only reads the key column (projection pushdown working!)
+        // 5. ProjectionExec on the source side that materializes the sentinel literal
         assert_plan_node_equals(
             plan,
             "MergeInsert: on=[key], when_matched=UpdateAll, when_not_matched=InsertAll, when_not_matched_by_source=Keep
   CoalescePartitionsExec
-    ProjectionExec: expr=[_rowid@1 as _rowid, _rowaddr@2 as _rowaddr, value@3 as value, key@4 as key, CASE WHEN __common_expr_1@0 AND _rowaddr@2 IS NULL THEN 2 WHEN __common_expr_1@0 AND _rowaddr@2 IS NOT NULL THEN 1 ELSE 0 END as __action]
-      ProjectionExec: expr=[key@3 IS NOT NULL as __common_expr_1, _rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key]
-        HashJoinExec: mode=CollectLeft, join_type=Right, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4]
-          CooperativeExec
-            LanceRead: uri=..., projection=[key], num_fragments=1, range_before=None, range_after=None, \
-            row_id=true, row_addr=true, full_filter=--, refine_filter=--
-          RepartitionExec: partitioning=RoundRobinBatch(...), input_partitions=1
+    ProjectionExec: expr=[_rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key, __merge_source_sentinel@4 as __merge_source_sentinel, CASE WHEN _rowaddr@1 IS NULL THEN 2 WHEN _rowaddr@1 IS NOT NULL THEN 1 ELSE 0 END as __action]
+      HashJoinExec: mode=CollectLeft, join_type=Right, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4, __merge_source_sentinel@5]
+        CooperativeExec
+          LanceRead: uri=..., projection=[key], num_fragments=1, range_before=None, range_after=None, \
+          row_id=true, row_addr=true, full_filter=--, refine_filter=--
+        RepartitionExec: partitioning=RoundRobinBatch(...), input_partitions=1
+          ProjectionExec: expr=[value@0 as value, key@1 as key, true as __merge_source_sentinel]
             StreamingTableExec: partition_sizes=1, projection=[value, key]"
         ).await.unwrap();
     }
@@ -4215,18 +4302,20 @@ mod tests {
         // This should use the fast path (execute_uncommitted_v2)
         let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap();
 
-        // The optimized plan should use Inner join instead of Right join
-        // since we're not inserting unmatched rows
+        // The optimized plan should use Inner join instead of Right join since we're not
+        // inserting unmatched rows.  The sentinel IS NOT NULL condition is folded away by
+        // DataFusion because the sentinel is lit(true), so the CASE only checks _rowaddr.
         assert_plan_node_equals(
             plan,
             "MergeInsert: on=[key], when_matched=UpdateAll, when_not_matched=DoNothing, when_not_matched_by_source=Keep
   CoalescePartitionsExec
-    ProjectionExec: expr=[_rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key, CASE WHEN key@3 IS NOT NULL AND _rowaddr@1 IS NOT NULL THEN 1 ELSE 0 END as __action]
-      HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4]
+    ProjectionExec: expr=[_rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key, __merge_source_sentinel@4 as __merge_source_sentinel, CASE WHEN _rowaddr@1 IS NOT NULL THEN 1 ELSE 0 END as __action]
+      HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4, __merge_source_sentinel@5]
         CooperativeExec
           LanceRead: uri=..., projection=[key], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=true, full_filter=--, refine_filter=--
         RepartitionExec...
-          StreamingTableExec: partition_sizes=1, projection=[value, key]"
+          ProjectionExec: expr=[value@0 as value, key@1 as key, true as __merge_source_sentinel]
+            StreamingTableExec: partition_sizes=1, projection=[value, key]"
         ).await.unwrap();
     }
 
@@ -4262,17 +4351,19 @@ mod tests {
 
         let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap();
 
-        // The optimized plan should use Inner join and include the UpdateIf condition
+        // The optimized plan should use Inner join and include the UpdateIf condition.
+        // The sentinel IS NOT NULL condition is folded away (sentinel is lit(true)).
         assert_plan_node_equals(
             plan,
             "MergeInsert: on=[key], when_matched=UpdateIf(source.value > 20), when_not_matched=DoNothing, when_not_matched_by_source=Keep
   CoalescePartitionsExec
-    ProjectionExec: expr=[_rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key, CASE WHEN key@3 IS NOT NULL AND _rowaddr@1 IS NOT NULL AND value@2 > 20 THEN 1 ELSE 0 END as __action]
-      HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4]
+    ProjectionExec: expr=[_rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key, __merge_source_sentinel@4 as __merge_source_sentinel, CASE WHEN _rowaddr@1 IS NOT NULL AND value@2 > 20 THEN 1 ELSE 0 END as __action]
+      HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4, __merge_source_sentinel@5]
         CooperativeExec
           LanceRead: uri=..., projection=[key], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=true, full_filter=--, refine_filter=--
         RepartitionExec...
-          StreamingTableExec: partition_sizes=1, projection=[value, key]"
+          ProjectionExec: expr=[value@0 as value, key@1 as key, true as __merge_source_sentinel]
+            StreamingTableExec: partition_sizes=1, projection=[value, key]"
         ).await.unwrap();
     }
 
@@ -5312,6 +5403,317 @@ MergeInsert: on=[id], when_matched=UpdateAll, when_not_matched=InsertAll, when_n
         assert_string_matches(&verbose_plan, expected_pattern).unwrap();
     }
 
+    #[tokio::test]
+    async fn test_explain_plan_full_schema_delete_by_source_with_fsl() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new(
+                "vec",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+                true,
+            ),
+        ]));
+
+        let dataset_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(
+                    FixedSizeListArray::try_new_from_values(
+                        Float32Array::from(vec![
+                            1.0, 1.1, 1.2, 1.3, 2.0, 2.1, 2.2, 2.3, 3.0, 3.1, 3.2, 3.3,
+                        ]),
+                        4,
+                    )
+                    .unwrap(),
+                ),
+            ],
+        )
+        .unwrap();
+
+        let dataset = Dataset::write(
+            Box::new(RecordBatchIterator::new(
+                [Ok(dataset_batch)],
+                schema.clone(),
+            )),
+            "memory://test_explain_plan_full_schema_delete_by_source_with_fsl",
+            None,
+        )
+        .await
+        .unwrap();
+
+        let merge_insert_job =
+            MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::UpdateAll)
+                .when_not_matched(WhenNotMatched::InsertAll)
+                .when_not_matched_by_source(WhenNotMatchedBySource::Delete)
+                .use_index(false)
+                .try_build()
+                .unwrap();
+
+        let plan = merge_insert_job.explain_plan(None, false).await.unwrap();
+        assert!(plan.contains("HashJoinExec"));
+        assert!(plan.contains("join_type=Full"));
+        assert!(plan.contains("projection=[_rowid"));
+        assert!(
+            plan.contains("LanceRead: uri=") && plan.contains("projection=[id]"),
+            "target-side scan should prune the FSL payload from the join build side: {plan}"
+        );
+        assert!(
+            !plan.contains("LanceRead: uri=test_explain_plan_full_schema_delete_by_source_with_fsl/data, projection=[id, vec]"),
+            "target-side scan should not include the FSL payload in the join build side: {plan}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_explain_plan_full_schema_delete_by_source_with_fsl_and_scalar_index() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new(
+                "vec",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+                true,
+            ),
+        ]));
+
+        let dataset_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(
+                    FixedSizeListArray::try_new_from_values(
+                        Float32Array::from(vec![
+                            1.0, 1.1, 1.2, 1.3, 2.0, 2.1, 2.2, 2.3, 3.0, 3.1, 3.2, 3.3,
+                        ]),
+                        4,
+                    )
+                    .unwrap(),
+                ),
+            ],
+        )
+        .unwrap();
+
+        let mut dataset = Dataset::write(
+            Box::new(RecordBatchIterator::new(
+                [Ok(dataset_batch)],
+                schema.clone(),
+            )),
+            "memory://test_explain_plan_full_schema_delete_by_source_with_fsl_and_scalar_index",
+            None,
+        )
+        .await
+        .unwrap();
+
+        let scalar_params = ScalarIndexParams::default();
+        dataset
+            .create_index(&["id"], IndexType::Scalar, None, &scalar_params, false)
+            .await
+            .unwrap();
+
+        let merge_insert_job =
+            MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::UpdateAll)
+                .when_not_matched(WhenNotMatched::InsertAll)
+                .when_not_matched_by_source(WhenNotMatchedBySource::Delete)
+                .try_build()
+                .unwrap();
+
+        let plan = merge_insert_job.explain_plan(None, false).await.unwrap();
+        assert!(plan.contains("HashJoinExec"));
+        assert!(plan.contains("join_type=Full"));
+        assert!(plan.contains("projection=[_rowid"));
+        assert!(
+            plan.contains("LanceRead: uri=") && plan.contains("projection=[id]"),
+            "target-side scan should prune the FSL payload from the join build side even when a scalar index exists: {plan}"
+        );
+        assert!(
+            !plan.contains(
+                "LanceRead: uri=test_explain_plan_full_schema_delete_by_source_with_fsl_and_scalar_index/data, projection=[id, vec]"
+            ),
+            "target-side scan should not include the FSL payload in the join build side: {plan}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_merge_insert_full_schema_delete_by_source_with_fsl() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new(
+                "vec",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+                true,
+            ),
+        ]));
+
+        let dataset_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(
+                    FixedSizeListArray::try_new_from_values(
+                        Float32Array::from(vec![
+                            1.0, 1.1, 1.2, 1.3, 2.0, 2.1, 2.2, 2.3, 3.0, 3.1, 3.2, 3.3,
+                        ]),
+                        4,
+                    )
+                    .unwrap(),
+                ),
+            ],
+        )
+        .unwrap();
+
+        let dataset = Dataset::write(
+            Box::new(RecordBatchIterator::new(
+                [Ok(dataset_batch)],
+                schema.clone(),
+            )),
+            "memory://test_merge_insert_full_schema_delete_by_source_with_fsl",
+            None,
+        )
+        .await
+        .unwrap();
+
+        let source_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![2, 4])),
+                Arc::new(
+                    FixedSizeListArray::try_new_from_values(
+                        Float32Array::from(vec![20.0, 20.1, 20.2, 20.3, 40.0, 40.1, 40.2, 40.3]),
+                        4,
+                    )
+                    .unwrap(),
+                ),
+            ],
+        )
+        .unwrap();
+
+        let (merged_dataset, stats) =
+            MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::UpdateAll)
+                .when_not_matched(WhenNotMatched::InsertAll)
+                .when_not_matched_by_source(WhenNotMatchedBySource::Delete)
+                .try_build()
+                .unwrap()
+                .execute_reader(Box::new(RecordBatchIterator::new(
+                    [Ok(source_batch)],
+                    schema.clone(),
+                )))
+                .await
+                .unwrap();
+
+        assert_eq!(stats.num_deleted_rows, 2);
+        assert_eq!(stats.num_updated_rows, 1);
+        assert_eq!(stats.num_inserted_rows, 1);
+
+        let merged = merged_dataset.scan().try_into_batch().await.unwrap();
+        let ids = merged["id"].as_primitive::<Int32Type>().values().to_vec();
+        assert_eq!(ids, vec![2, 4]);
+
+        let vecs = merged["vec"].as_fixed_size_list();
+        let actual = vecs
+            .values()
+            .as_primitive::<Float32Type>()
+            .values()
+            .to_vec();
+        assert_eq!(actual, vec![20.0, 20.1, 20.2, 20.3, 40.0, 40.1, 40.2, 40.3]);
+    }
+
+    #[tokio::test]
+    async fn test_merge_insert_full_schema_delete_by_source_with_fsl_and_scalar_index() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new(
+                "vec",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+                true,
+            ),
+        ]));
+
+        let dataset_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(
+                    FixedSizeListArray::try_new_from_values(
+                        Float32Array::from(vec![
+                            1.0, 1.1, 1.2, 1.3, 2.0, 2.1, 2.2, 2.3, 3.0, 3.1, 3.2, 3.3,
+                        ]),
+                        4,
+                    )
+                    .unwrap(),
+                ),
+            ],
+        )
+        .unwrap();
+
+        let mut dataset = Dataset::write(
+            Box::new(RecordBatchIterator::new(
+                [Ok(dataset_batch)],
+                schema.clone(),
+            )),
+            "memory://test_merge_insert_full_schema_delete_by_source_with_fsl_and_scalar_index",
+            None,
+        )
+        .await
+        .unwrap();
+
+        let scalar_params = ScalarIndexParams::default();
+        dataset
+            .create_index(&["id"], IndexType::Scalar, None, &scalar_params, false)
+            .await
+            .unwrap();
+
+        let source_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![2, 4])),
+                Arc::new(
+                    FixedSizeListArray::try_new_from_values(
+                        Float32Array::from(vec![20.0, 20.1, 20.2, 20.3, 40.0, 40.1, 40.2, 40.3]),
+                        4,
+                    )
+                    .unwrap(),
+                ),
+            ],
+        )
+        .unwrap();
+
+        let (merged_dataset, stats) =
+            MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::UpdateAll)
+                .when_not_matched(WhenNotMatched::InsertAll)
+                .when_not_matched_by_source(WhenNotMatchedBySource::Delete)
+                .try_build()
+                .unwrap()
+                .execute_reader(Box::new(RecordBatchIterator::new(
+                    [Ok(source_batch)],
+                    schema.clone(),
+                )))
+                .await
+                .unwrap();
+
+        assert_eq!(stats.num_deleted_rows, 2);
+        assert_eq!(stats.num_updated_rows, 1);
+        assert_eq!(stats.num_inserted_rows, 1);
+
+        let merged = merged_dataset.scan().try_into_batch().await.unwrap();
+        let ids = merged["id"].as_primitive::<Int32Type>().values().to_vec();
+        assert_eq!(ids, vec![2, 4]);
+
+        let vecs = merged["vec"].as_fixed_size_list();
+        let actual = vecs
+            .values()
+            .as_primitive::<Float32Type>()
+            .values()
+            .to_vec();
+        assert_eq!(actual, vec![20.0, 20.1, 20.2, 20.3, 40.0, 40.1, 40.2, 40.3]);
+    }
+
     #[tokio::test]
     async fn test_analyze_plan() {
         // Set up test data using lance_datagen
@@ -6252,6 +6654,194 @@ MergeInsert: on=[id], when_matched=UpdateAll, when_not_matched=InsertAll, when_n
         );
     }
 
+    /// Test case for Issue #4644: merge_insert should NOT skip source rows whose ON
+    /// columns contain NULL.
+    ///
+    /// With standard SQL equality NULL != NULL, so a source row with a NULL key will
+    /// never match any target row.  It must therefore be treated as "not matched" and
+    /// inserted when `when_not_matched = InsertAll`.  The previous implementation
+    /// incorrectly required all ON columns to be non-null before even considering the
+    /// row, causing it to be silently dropped (Action::Nothing).
+    #[tokio::test]
+    async fn test_merge_insert_null_on_column_inserts() {
+        // Initial dataset: one row with a NULL record_type.
+        let initial_data = record_batch!(
+            ("id", Int32, [0]),
+            ("record_type", Utf8, [Option::<&str>::None]),
+            ("value", Int32, [10])
+        )
+        .unwrap();
+
+        let dataset = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(initial_data.clone())], initial_data.schema()),
+            "memory://test_null_on_column",
+            None,
+        )
+        .await
+        .unwrap();
+
+        // New data: a row with a different id AND a NULL record_type.
+        // Because id differs (2 vs 0) no match should be found even with NULL-safe
+        // semantics, so this row must be INSERTED.
+        let new_data = record_batch!(
+            ("id", Int32, [Some(2)]),
+            ("record_type", Utf8, [Option::<&str>::None]),
+            ("value", Int32, [Some(99)])
+        )
+        .unwrap();
+
+        let (merged_dataset, stats) = MergeInsertBuilder::try_new(
+            Arc::new(dataset),
+            vec!["id".to_string(), "record_type".to_string()],
+        )
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .try_build()
+        .unwrap()
+        .execute_reader(Box::new(RecordBatchIterator::new(
+            vec![Ok(new_data.clone())],
+            new_data.schema(),
+        )))
+        .await
+        .unwrap();
+
+        // The source row (id=2, record_type=NULL) must be inserted, NOT silently skipped.
+        assert_eq!(
+            stats.num_inserted_rows, 1,
+            "row with NULL ON column should be inserted"
+        );
+        assert_eq!(stats.num_updated_rows, 0, "no row should be updated");
+
+        let count = merged_dataset.count_rows(None).await.unwrap();
+        assert_eq!(
+            count, 2,
+            "dataset should have the original row plus the newly inserted row"
+        );
+    }
+
+    /// Partial composite key match: the non-null part of the ON key (id) matches an
+    /// existing target row, but the second ON column (record_type) is NULL in the source.
+    /// Standard SQL equality treats NULL != NULL, so the composite key does NOT match
+    /// and the source row must be inserted, not updated and not silently dropped.
+    #[tokio::test]
+    async fn test_merge_insert_partial_composite_key_null() {
+        // Target: one row where id=1 and record_type="A".
+        let initial_data = record_batch!(
+            ("id", Int32, [Some(1)]),
+            ("record_type", Utf8, [Some("A")]),
+            ("value", Int32, [Some(10)])
+        )
+        .unwrap();
+
+        let dataset = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(initial_data.clone())], initial_data.schema()),
+            "memory://test_partial_composite_null",
+            None,
+        )
+        .await
+        .unwrap();
+
+        // Source: one row where id=1 (matches target) but record_type=NULL.
+        // The composite key (1, NULL) does NOT match (1, "A") under standard equality,
+        // so this is a "not matched" row that should be inserted.
+        let new_data = record_batch!(
+            ("id", Int32, [Some(1)]),
+            ("record_type", Utf8, [Option::<&str>::None]),
+            ("value", Int32, [Some(99)])
+        )
+        .unwrap();
+
+        let (merged_dataset, stats) = MergeInsertBuilder::try_new(
+            Arc::new(dataset),
+            vec!["id".to_string(), "record_type".to_string()],
+        )
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .try_build()
+        .unwrap()
+        .execute_reader(Box::new(RecordBatchIterator::new(
+            vec![Ok(new_data.clone())],
+            new_data.schema(),
+        )))
+        .await
+        .unwrap();
+
+        // Source row (id=1, record_type=NULL) must be inserted, not updated and not dropped.
+        assert_eq!(
+            stats.num_inserted_rows, 1,
+            "row with partial NULL composite key should be inserted"
+        );
+        assert_eq!(
+            stats.num_updated_rows, 0,
+            "existing (id=1, record_type=A) row must not be updated"
+        );
+
+        // Dataset: original (1, "A") row + newly inserted (1, NULL) row = 2 rows.
+        let count = merged_dataset.count_rows(None).await.unwrap();
+        assert_eq!(
+            count, 2,
+            "both the original and the new row must be present"
+        );
+    }
+
+    /// Variant of test_merge_insert_null_on_column_inserts with a single ON column
+    /// that is entirely NULL, and a target row that also has a NULL in that column.
+    /// Since standard SQL equality treats NULL != NULL, the source row must not match
+    /// the existing target row and must be inserted separately.
+    #[tokio::test]
+    async fn test_merge_insert_null_single_on_column() {
+        // Dataset with a single row where id is NULL.
+        let initial_data = record_batch!(
+            ("id", Int32, [Option::<i32>::None]),
+            ("value", Int32, [Some(1)])
+        )
+        .unwrap();
+
+        let dataset = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(initial_data.clone())], initial_data.schema()),
+            "memory://test_null_single_on_column",
+            None,
+        )
+        .await
+        .unwrap();
+
+        // Source has two rows: one with id=NULL and one with id=5.
+        // id=NULL should not match the existing id=NULL row (standard equality), so it
+        // gets inserted.  id=5 is a brand-new key and also gets inserted.
+        let new_data = record_batch!(
+            ("id", Int32, [Option::<i32>::None, Some(5)]),
+            ("value", Int32, [Some(99), Some(50)])
+        )
+        .unwrap();
+
+        let (merged_dataset, stats) =
+            MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::UpdateAll)
+                .when_not_matched(WhenNotMatched::InsertAll)
+                .try_build()
+                .unwrap()
+                .execute_reader(Box::new(RecordBatchIterator::new(
+                    vec![Ok(new_data.clone())],
+                    new_data.schema(),
+                )))
+                .await
+                .unwrap();
+
+        // Both source rows must be inserted (not silently dropped).
+        assert_eq!(
+            stats.num_inserted_rows, 2,
+            "both rows with NULL ON column should be inserted"
+        );
+        assert_eq!(stats.num_updated_rows, 0);
+
+        // Dataset now has: original NULL-id row + 2 newly inserted rows = 3 total.
+        let count = merged_dataset.count_rows(None).await.unwrap();
+        assert_eq!(count, 3);
+    }
+
     /// Test case for Issue #3634: merge_insert should provide a helpful error
     /// message when a subschema with a mismatched type is provided.
     #[tokio::test]
diff --git a/rust/lance/src/dataset/write/merge_insert/assign_action.rs b/rust/lance/src/dataset/write/merge_insert/assign_action.rs
index e4ec637c1cf..46bda38873f 100644
--- a/rust/lance/src/dataset/write/merge_insert/assign_action.rs
+++ b/rust/lance/src/dataset/write/merge_insert/assign_action.rs
@@ -1,8 +1,12 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use super::{MergeInsertParams, WhenNotMatchedBySource};
+use super::{MERGE_SOURCE_SENTINEL, MergeInsertParams, WhenNotMatchedBySource};
 use crate::{Result, dataset::WhenMatched};
+use datafusion::common::{
+    Column, TableReference,
+    tree_node::{Transformed, TransformedResult, TreeNode},
+};
 use datafusion::scalar::ScalarValue;
 use datafusion_expr::{Case, Expr, col};
 
@@ -50,48 +54,50 @@ impl Action {
     }
 }
 
+fn qualify_unqualified_columns(expr: Expr, relation: &'static str) -> Result<Expr> {
+    expr.transform(|expr| {
+        Ok(if let Expr::Column(column) = expr {
+            if column.relation.is_none() {
+                let qualified = Column::new_unqualified(column.name)
+                    .with_relation(TableReference::bare(relation));
+                Transformed::yes(Expr::Column(qualified))
+            } else {
+                Transformed::no(Expr::Column(column))
+            }
+        } else {
+            Transformed::no(expr)
+        })
+    })
+    .data()
+    .map_err(crate::Error::from)
+}
+
 /// Transforms merge insert parameters into a logical expression. The output
 /// is a single "action" column, that describes what to do with each row.
 pub fn merge_insert_action(
     params: &MergeInsertParams,
     schema: Option<&arrow_schema::Schema>,
 ) -> Result<Expr> {
-    // Check that at least one key column is non-null in the source
-    // This ensures we only process rows that have valid join keys
-    // Note: Column names are wrapped in double quotes to preserve case
-    // (DataFusion's col() function lowercases unquoted identifiers)
-    let source_has_key: Expr = if params.on.len() == 1 {
-        // Single key column case - check if the source key column is not null
-        // Need to qualify the column to avoid ambiguity between target.key and source.key
-        col(format!("source.\"{}\"", &params.on[0])).is_not_null()
-    } else {
-        // Multiple key columns - require that ALL key columns are non-null
-        // This is a stricter requirement than "at least one" to ensure proper joins
-        let key_conditions: Vec<Expr> = params
-            .on
-            .iter()
-            .map(|key| col(format!("source.\"{}\"", key)).is_not_null())
-            .collect();
-
-        // Use AND to combine all key column checks (all must be non-null)
-        key_conditions
-            .into_iter()
-            .reduce(|acc, expr| acc.and(expr))
-            .unwrap_or_else(|| datafusion_expr::lit(false))
-    };
+    // Use a sentinel column to detect whether the source side contributed a row to the
+    // join output.  This is NULL-safe: the sentinel is `true` for every source row and
+    // is NULL-filled by the outer join for target-only rows, regardless of whether any
+    // ON column contains NULL.  Using ON key columns for this purpose is incorrect
+    // because a key column that is legitimately NULL is indistinguishable from a NULL
+    // introduced by the outer join on the target side.
+    let source_has_row = col(format!("source.\"{}\"", MERGE_SOURCE_SENTINEL)).is_not_null();
 
-    let row_addr_is_not_null = col("target._rowaddr").is_not_null();
-    let matched = source_has_key.clone().and(row_addr_is_not_null);
+    let target_has_row = col("target._rowaddr").is_not_null();
+    let matched = source_has_row.clone().and(target_has_row.clone());
 
-    let row_addr_is_null = col("target._rowaddr").is_null();
-    let not_matched_in_target = source_has_key.and(row_addr_is_null);
+    let source_only = source_has_row.and(col("target._rowaddr").is_null());
 
-    let not_matched_in_source = col("target._rowaddr").is_null().is_not_true();
+    let target_only =
+        target_has_row.and(col(format!("source.\"{}\"", MERGE_SOURCE_SENTINEL)).is_null());
 
     let mut cases = vec![];
 
     if params.insert_not_matched {
-        cases.push((not_matched_in_target, Action::Insert.as_literal_expr()));
+        cases.push((source_only, Action::Insert.as_literal_expr()));
     }
 
     match &params.when_matched {
@@ -130,11 +136,12 @@ pub fn merge_insert_action(
 
     match &params.delete_not_matched_by_source {
         WhenNotMatchedBySource::Delete => {
-            cases.push((not_matched_in_source, Action::Delete.as_literal_expr()));
+            cases.push((target_only, Action::Delete.as_literal_expr()));
         }
         WhenNotMatchedBySource::DeleteIf(condition) => {
+            let target_condition = qualify_unqualified_columns(condition.clone(), "target")?;
             cases.push((
-                not_matched_in_source.and(condition.clone()),
+                target_only.and(target_condition),
                 Action::Delete.as_literal_expr(),
             ));
         }
diff --git a/rust/lance/src/dataset/write/merge_insert/exec/write.rs b/rust/lance/src/dataset/write/merge_insert/exec/write.rs
index c9ac51e7116..703d4e1f6c8 100644
--- a/rust/lance/src/dataset/write/merge_insert/exec/write.rs
+++ b/rust/lance/src/dataset/write/merge_insert/exec/write.rs
@@ -29,7 +29,8 @@ use crate::dataset::write::merge_insert::inserted_rows::{
     KeyExistenceFilter, KeyExistenceFilterBuilder, extract_key_value_from_batch,
 };
 use crate::dataset::write::merge_insert::{
-    SourceDedupeBehavior, create_duplicate_row_error, format_key_values_on_columns,
+    MERGE_SOURCE_SENTINEL, SourceDedupeBehavior, create_duplicate_row_error,
+    format_key_values_on_columns,
 };
 use crate::{
     Dataset,
@@ -418,18 +419,19 @@ impl FullSchemaMergeInsertExec {
         // from the logical join, leaving us with the merged data columns plus special columns
         let total_fields = input_schema.fields().len();
 
-        // Select all columns that are data columns (not _rowaddr or __action)
+        // Select all columns that are data columns (not _rowaddr, __action, or the sentinel)
         // These represent the final merged data values to write
         let data_column_indices: Vec<usize> = (0..total_fields)
             .filter(|&idx| {
                 let field = input_schema.field(idx);
                 let name = field.name();
-                // Skip special columns: _rowaddr and __action
+                // Skip special columns: _rowaddr, __action, and the source-presence sentinel
                 idx != rowaddr_idx
                     && idx != action_idx
                     && name != ROW_ADDR
                     && name != ROW_ID
                     && name != MERGE_ACTION_COLUMN
+                    && name != MERGE_SOURCE_SENTINEL
             })
             .collect();
 
diff --git a/rust/lance/src/dataset/write/merge_insert/logical_plan.rs b/rust/lance/src/dataset/write/merge_insert/logical_plan.rs
index 84b97290dcf..00bae881e8a 100644
--- a/rust/lance/src/dataset/write/merge_insert/logical_plan.rs
+++ b/rust/lance/src/dataset/write/merge_insert/logical_plan.rs
@@ -19,7 +19,7 @@ use crate::dataset::write::merge_insert::exec::{
 };
 use crate::dataset::{WhenMatched, WhenNotMatchedBySource};
 
-use super::{MERGE_ACTION_COLUMN, MergeInsertParams};
+use super::{MERGE_ACTION_COLUMN, MERGE_SOURCE_SENTINEL, MergeInsertParams};
 
 /// Logical plan node for merge insert write.
 ///
@@ -163,11 +163,13 @@ impl UserDefinedLogicalNodeCore for MergeInsertWriteNode {
 
         for (i, (qualifier, field)) in input_schema.iter().enumerate() {
             let should_include = match qualifier {
-                // For delete-only: only include source KEY columns (for matching)
-                // For other ops: include all source columns - they contain the new data to write
+                // For delete-only: only include source KEY columns (for matching) plus the
+                // sentinel column needed for action determination.
+                // For other ops: include all source columns - they contain the new data to write.
                 Some(qualifier) if qualifier.table() == "source" => {
                     if no_upsert {
                         self.params.on.iter().any(|k| k == field.name())
+                            || field.name() == MERGE_SOURCE_SENTINEL
                     } else {
                         true
                     }
diff --git a/rust/lance/src/dataset/write/update.rs b/rust/lance/src/dataset/write/update.rs
index 855d83018f4..ec34000642d 100644
--- a/rust/lance/src/dataset/write/update.rs
+++ b/rust/lance/src/dataset/write/update.rs
@@ -483,6 +483,7 @@ mod tests {
     use super::*;
 
     use crate::dataset::{WriteDestination, WriteMode};
+    use crate::index::DatasetIndexExt;
     use crate::index::vector::VectorIndexParams;
     use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount};
     use arrow::{array::AsArray, datatypes::UInt32Type};
@@ -495,7 +496,6 @@ mod tests {
     use lance_core::utils::tempfile::TempStrDir;
     use lance_datagen::{Dimension, RowCount};
     use lance_file::version::LanceFileVersion;
-    use lance_index::DatasetIndexExt;
     use lance_index::IndexType;
     use lance_index::scalar::ScalarIndexParams;
     use lance_io::object_store::ObjectStoreParams;
diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs
index f4cd31e44b8..40788265172 100644
--- a/rust/lance/src/index.rs
+++ b/rust/lance/src/index.rs
@@ -7,11 +7,10 @@
 use std::collections::{HashMap, HashSet};
 use std::sync::{Arc, OnceLock};
 
-use arrow_schema::{DataType, Schema};
+use arrow_schema::DataType;
 use async_trait::async_trait;
 use datafusion::execution::SendableRecordBatchStream;
-use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-use futures::{FutureExt, stream};
+use futures::FutureExt;
 use itertools::Itertools;
 use lance_core::cache::{CacheKey, UnsizedCacheKey};
 use lance_core::datatypes::Field;
@@ -24,6 +23,7 @@ use lance_core::utils::tracing::{
 };
 use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_file::reader::FileReaderOptions;
+use lance_index::INDEX_METADATA_SCHEMA_KEY;
 pub use lance_index::IndexParams;
 use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseIndex};
 use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndex};
@@ -33,7 +33,7 @@ pub use lance_index::progress::{IndexBuildProgress, NoopIndexBuildProgress};
 use lance_index::scalar::expression::{
     IndexInformationProvider, MultiQueryParser, ScalarQueryParser,
 };
-use lance_index::scalar::inverted::InvertedIndexPlugin;
+use lance_index::scalar::inverted::{InvertedIndex, InvertedIndexPlugin};
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_index::scalar::registry::{TrainingCriteria, TrainingOrdering};
 use lance_index::scalar::{CreatedIndex, ScalarIndex};
@@ -41,9 +41,10 @@ use lance_index::vector::bq::builder::RabitQuantizer;
 use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex, FlatQuantizer};
 use lance_index::vector::hnsw::HNSW;
 use lance_index::vector::pq::ProductQuantizer;
+use lance_index::vector::quantizer::Quantization;
 use lance_index::vector::sq::ScalarQuantizer;
-use lance_index::{DatasetIndexExt, INDEX_METADATA_SCHEMA_KEY, IndexDescription};
-use lance_index::{INDEX_FILE_NAME, Index, IndexType, pb, vector::VectorIndex};
+use lance_index::vector::v3::subindex::IvfSubIndex;
+use lance_index::{INDEX_FILE_NAME, Index, IndexType, PrewarmOptions, pb, vector::VectorIndex};
 use lance_index::{
     IndexCriteria, is_system_index,
     metrics::{MetricsCollector, NoOpMetricsCollector},
@@ -54,17 +55,18 @@ use lance_io::utils::{
     CachedFileSize, read_last_block, read_message, read_message_from_buf, read_metadata_offset,
     read_version,
 };
-use lance_table::format::IndexMetadata;
 use lance_table::format::{Fragment, SelfDescribingFileReader};
+use lance_table::format::{IndexMetadata, list_index_files_with_sizes};
 use lance_table::io::manifest::read_manifest_indexes;
 use roaring::RoaringBitmap;
 use scalar::index_matches_criteria;
 use serde_json::json;
 use tracing::{info, instrument};
 use uuid::Uuid;
-use vector::ivf::v2::IVFIndex;
+use vector::ivf::v2::{IVFIndex, IvfStateEntryBox};
 use vector::utils::get_vector_type;
 
+mod api;
 pub(crate) mod append;
 mod create;
 pub mod frag_reuse;
@@ -78,14 +80,51 @@ use self::vector::remap_vector_index;
 use crate::dataset::index::LanceIndexStoreExt;
 use crate::dataset::optimize::RemappedIndex;
 use crate::dataset::optimize::remapping::RemapResult;
-use crate::dataset::transaction::{Operation, Transaction};
+use crate::dataset::transaction::{Operation, Transaction, TransactionBuilder};
+pub use crate::index::api::DatasetIndexExt;
 use crate::index::frag_reuse::{load_frag_reuse_index_details, open_frag_reuse_index};
 use crate::index::mem_wal::open_mem_wal_index;
 pub use crate::index::prefilter::{FilterLoader, PreFilter};
 use crate::index::scalar::{IndexDetails, fetch_index_details, load_training_data};
+pub use crate::index::vector::{LogicalIvfView, LogicalVectorIndex};
 use crate::session::index_caches::{FragReuseIndexKey, IndexMetadataKey};
 use crate::{Error, Result, dataset::Dataset};
 pub use create::CreateIndexBuilder;
+pub use lance_index::IndexDescription;
+
+fn validate_segment_metadata(index_name: &str, segments: &[IndexMetadata]) -> Result<()> {
+    if segments.is_empty() {
+        return Err(Error::invalid_input(
+            "CreateIndex: at least one index segment is required".to_string(),
+        ));
+    }
+
+    let mut seen_segment_ids = HashSet::with_capacity(segments.len());
+    let mut covered_fragments = RoaringBitmap::new();
+    for segment in segments {
+        if !seen_segment_ids.insert(segment.uuid) {
+            return Err(Error::invalid_input(format!(
+                "CreateIndex: duplicate segment uuid {} for index '{}'",
+                segment.uuid, index_name
+            )));
+        }
+        let fragment_bitmap = segment.fragment_bitmap.as_ref().ok_or_else(|| {
+            Error::invalid_input(format!(
+                "CreateIndex: segment {} is missing fragment coverage",
+                segment.uuid
+            ))
+        })?;
+        if !covered_fragments.is_disjoint(fragment_bitmap) {
+            return Err(Error::invalid_input(format!(
+                "CreateIndex: overlapping fragment coverage in segment set for index '{}'",
+                index_name
+            )));
+        }
+        covered_fragments |= fragment_bitmap.clone();
+    }
+
+    Ok(())
+}
 
 // Cache keys for different index types
 #[derive(Debug, Clone)]
@@ -110,22 +149,63 @@ impl UnsizedCacheKey for ScalarIndexCacheKey<'_> {
             self.uuid.into()
         }
     }
+
+    fn type_name() -> &'static str {
+        "ScalarIndex"
+    }
 }
 
 #[derive(Debug, Clone)]
-pub struct VectorIndexCacheKey<'a> {
-    pub uuid: &'a str,
-    pub fri_uuid: Option<&'a Uuid>,
+pub(crate) struct LegacyVectorIndexCacheKey<'a> {
+    uuid: &'a str,
+    fri_uuid: Option<&'a Uuid>,
 }
 
-impl<'a> VectorIndexCacheKey<'a> {
-    pub fn new(uuid: &'a str, fri_uuid: Option<&'a Uuid>) -> Self {
+impl<'a> LegacyVectorIndexCacheKey<'a> {
+    fn new(uuid: &'a str, fri_uuid: Option<&'a Uuid>) -> Self {
+        Self { uuid, fri_uuid }
+    }
+}
+
+impl CacheKey for LegacyVectorIndexCacheKey<'_> {
+    type ValueType = CachedLegacyVectorIndex;
+
+    fn key(&self) -> std::borrow::Cow<'_, str> {
+        if let Some(fri_uuid) = self.fri_uuid {
+            format!("{}-{}", self.uuid, fri_uuid).into()
+        } else {
+            self.uuid.into()
+        }
+    }
+
+    fn type_name() -> &'static str {
+        "LegacyVectorIndex"
+    }
+}
+
+/// Sized cache key for `IvfIndexState`.
+///
+/// Used for v0.3+ indices that support serialization. This key has a codec,
+/// so custom cache backends can serialize the state to disk/Redis/etc.
+/// Legacy indices use `LegacyVectorIndexCacheKey` instead (in-memory only).
+#[derive(Debug, Clone)]
+pub(crate) struct IvfIndexStateCacheKey<'a> {
+    uuid: &'a str,
+    fri_uuid: Option<&'a Uuid>,
+}
+
+impl<'a> IvfIndexStateCacheKey<'a> {
+    fn new(uuid: &'a str, fri_uuid: Option<&'a Uuid>) -> Self {
         Self { uuid, fri_uuid }
     }
 }
 
-impl UnsizedCacheKey for VectorIndexCacheKey<'_> {
-    type ValueType = dyn VectorIndex;
+impl CacheKey for IvfIndexStateCacheKey<'_> {
+    type ValueType = IvfStateEntryBox;
+
+    fn type_name() -> &'static str {
+        "IvfIndexState"
+    }
 
     fn key(&self) -> std::borrow::Cow<'_, str> {
         if let Some(fri_uuid) = self.fri_uuid {
@@ -134,8 +214,17 @@ impl UnsizedCacheKey for VectorIndexCacheKey<'_> {
             self.uuid.into()
         }
     }
+
+    fn codec() -> Option<lance_core::cache::CacheCodec> {
+        Some(lance_core::cache::CacheCodec::from_impl::<IvfStateEntryBox>())
+    }
 }
 
+/// Wrapper that stores a live VectorIndex in the cache.
+/// Used for v0.1/v0.2 indices that don't support serializable caching.
+#[derive(Debug, deepsize::DeepSizeOf)]
+pub(crate) struct CachedLegacyVectorIndex(Arc<dyn VectorIndex>);
+
 #[derive(Debug, Clone)]
 pub struct FragReuseIndexCacheKey<'a> {
     pub uuid: &'a str,
@@ -158,6 +247,10 @@ impl CacheKey for FragReuseIndexCacheKey<'_> {
             self.uuid.into()
         }
     }
+
+    fn type_name() -> &'static str {
+        "FragReuseIndex"
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -182,6 +275,10 @@ impl CacheKey for MemWalCacheKey<'_> {
             self.uuid.to_string().into()
         }
     }
+
+    fn type_name() -> &'static str {
+        "MemWalIndex"
+    }
 }
 
 // Whether to auto-migrate a dataset when we encounter corruption.
@@ -360,12 +457,18 @@ pub(crate) async fn remap_index(
                 row_id_map,
             )
             .await?;
+
+            // Capture file sizes for the vector index
+            let index_dir = dataset.indices_dir().child(new_id.to_string());
+            let files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?;
+
             CreatedIndex {
                 index_details: prost_types::Any::from_msg(
                     &lance_table::format::pb::VectorIndexDetails::default(),
                 )
                 .unwrap(),
                 index_version,
+                files: Some(files),
             }
         }
         _ => {
@@ -381,6 +484,7 @@ pub(crate) async fn remap_index(
         new_id,
         index_details: created_index.index_details,
         index_version: created_index.index_version,
+        files: created_index.files,
     }))
 }
 
@@ -558,6 +662,18 @@ impl IndexDescription for IndexDescriptionImpl {
             .details_as_json(&self.details.0)
             .map(|v| v.to_string())
     }
+
+    fn total_size_bytes(&self) -> Option<u64> {
+        let mut total = 0u64;
+        for segment in &self.segments {
+            // If any segment is missing file info, return None for backward compatibility
+            let files = segment.files.as_ref()?;
+            for file in files {
+                total += file.size_bytes;
+            }
+        }
+        Some(total)
+    }
 }
 
 #[async_trait]
@@ -574,7 +690,8 @@ impl DatasetIndexExt for Dataset {
     /// Create a scalar BTREE index:
     /// ```
     /// # use lance::{Dataset, Result};
-    /// # use lance_index::{DatasetIndexExt, IndexType, scalar::ScalarIndexParams};
+    /// # use lance::index::DatasetIndexExt;
+    /// # use lance_index::{IndexType, scalar::ScalarIndexParams};
     /// # async fn example(dataset: &mut Dataset) -> Result<()> {
     /// let params = ScalarIndexParams::default();
     /// dataset
@@ -588,7 +705,8 @@ impl DatasetIndexExt for Dataset {
     /// Create an empty index that will be populated later:
     /// ```
     /// # use lance::{Dataset, Result};
-    /// # use lance_index::{DatasetIndexExt, IndexType, scalar::ScalarIndexParams};
+    /// # use lance::index::DatasetIndexExt;
+    /// # use lance_index::{IndexType, scalar::ScalarIndexParams};
     /// # async fn example(dataset: &mut Dataset) -> Result<()> {
     /// let params = ScalarIndexParams::default();
     /// dataset
@@ -654,10 +772,47 @@ impl DatasetIndexExt for Dataset {
             return Err(Error::index_not_found(format!("name={}", name)));
         }
 
-        let index = self
-            .open_generic_index(name, &indices[0].uuid.to_string(), &NoOpMetricsCollector)
-            .await?;
-        index.prewarm().await?;
+        for index_meta in indices {
+            let index = self
+                .open_generic_index(name, &index_meta.uuid.to_string(), &NoOpMetricsCollector)
+                .await?;
+            index.prewarm().await?;
+        }
+
+        Ok(())
+    }
+
+    async fn prewarm_index_with_options(&self, name: &str, options: &PrewarmOptions) -> Result<()> {
+        let indices = self.load_indices_by_name(name).await?;
+        if indices.is_empty() {
+            return Err(Error::index_not_found(format!("name={}", name)));
+        }
+
+        for index_meta in indices {
+            let index = self
+                .open_generic_index(name, &index_meta.uuid.to_string(), &NoOpMetricsCollector)
+                .await?;
+
+            match options {
+                PrewarmOptions::Fts(fts_options) => {
+                    let inverted = index
+                        .as_any()
+                        .downcast_ref::<InvertedIndex>()
+                        .ok_or_else(|| {
+                            Error::invalid_input(format!(
+                                "FTS prewarm options are only supported for inverted indices, got {:?}",
+                                index.index_type()
+                            ))
+                        })?;
+                    inverted.prewarm_with_options(fts_options).await?;
+                }
+                _ => {
+                    return Err(Error::not_supported(
+                        "unsupported prewarm options for this lance version".to_owned(),
+                    ));
+                }
+            }
+        }
 
         Ok(())
     }
@@ -753,11 +908,57 @@ impl DatasetIndexExt for Dataset {
         }
     }
 
-    async fn commit_existing_index(
+    async fn merge_existing_index_segments(
+        &self,
+        source_segments: Vec<IndexMetadata>,
+    ) -> Result<IndexMetadata> {
+        validate_segment_metadata("uncommitted", &source_segments)?;
+        let field_id = *source_segments[0].fields.first().ok_or_else(|| {
+            Error::invalid_input(format!(
+                "CreateIndex: segment {} is missing field ids",
+                source_segments[0].uuid
+            ))
+        })?;
+        if source_segments
+            .iter()
+            .any(|segment| segment.fields != [field_id])
+        {
+            return Err(Error::invalid_input(
+                "merge_existing_index_segments requires segments with identical fields".to_string(),
+            ));
+        }
+        if !source_segments.iter().all(|segment| {
+            segment.index_details.as_ref().map_or_else(
+                || {
+                    segment
+                        .files
+                        .as_ref()
+                        .is_some_and(|files| files.iter().any(|file| file.path == INDEX_FILE_NAME))
+                },
+                |details| details.type_url.ends_with("VectorIndexDetails"),
+            )
+        }) {
+            return Err(Error::invalid_input(
+                "merge_existing_index_segments currently only supports vector segments".to_string(),
+            ));
+        }
+
+        let mut merged_segment = crate::index::vector::ivf::merge_segments(
+            self.object_store(),
+            &self.indices_dir(),
+            source_segments,
+        )
+        .await?;
+        merged_segment.dataset_version = self.manifest.version;
+        merged_segment.fields = vec![field_id];
+        Ok(merged_segment)
+    }
+
+    async fn commit_existing_index_segments(
         &mut self,
         index_name: &str,
         column: &str,
-        index_id: Uuid,
+        segments: Vec<IndexMetadata>,
     ) -> Result<()> {
         let Some(field) = self.schema().field(column) else {
             return Err(Error::index(format!(
@@ -765,25 +966,25 @@ impl DatasetIndexExt for Dataset {
             )));
         };
 
-        // TODO: We will need some way to determine the index details here.  Perhaps
-        // we can load the index itself and get the details that way.
+        validate_segment_metadata(index_name, &segments)?;
 
-        let new_idx = IndexMetadata {
-            uuid: index_id,
-            name: index_name.to_string(),
-            fields: vec![field.id],
-            dataset_version: self.manifest.version,
-            fragment_bitmap: Some(self.get_fragments().iter().map(|f| f.id() as u32).collect()),
-            index_details: None,
-            index_version: 0,
-            created_at: Some(chrono::Utc::now()),
-            base_id: None, // New indices don't have base_id (they're not from shallow clone)
-        };
+        let mut new_indices = Vec::with_capacity(segments.len());
+        for mut segment in segments {
+            if segment.fields != [field.id] {
+                return Err(Error::invalid_input(format!(
+                    "CreateIndex: segment {} was built for fields {:?}, expected [{}]",
+                    segment.uuid, segment.fields, field.id
+                )));
+            }
+            segment.name = index_name.to_string();
+            segment.dataset_version = self.manifest.version;
+            new_indices.push(segment);
+        }
 
         let transaction = Transaction::new(
             self.manifest.version,
             Operation::CreateIndex {
-                new_indices: vec![new_idx],
+                new_indices,
                 removed_indices: vec![],
             },
             None,
@@ -897,16 +1098,10 @@ impl DatasetIndexExt for Dataset {
                 index_details: Some(Arc::new(res.new_index_details)),
                 index_version: res.new_index_version,
                 created_at: Some(chrono::Utc::now()),
-                base_id: None, // Mew merged index file locates in the cloned dataset.
+                base_id: None, // New merged index file locates in the cloned dataset.
+                files: res.files,
             };
             removed_indices.extend(res.removed_indices.iter().map(|&idx| idx.clone()));
-            if deltas.len() > res.removed_indices.len() {
-                new_indices.extend(
-                    deltas[0..(deltas.len() - res.removed_indices.len())]
-                        .iter()
-                        .map(|&idx| idx.clone()),
-                );
-            }
             new_indices.push(new_idx);
         }
 
@@ -914,14 +1109,15 @@ impl DatasetIndexExt for Dataset {
             return Ok(());
         }
 
-        let transaction = Transaction::new(
+        let transaction = TransactionBuilder::new(
             self.manifest.version,
             Operation::CreateIndex {
                 new_indices,
                 removed_indices,
             },
-            None,
-        );
+        )
+        .transaction_properties(options.transaction_properties.clone())
+        .build();
 
         self.apply_commit(transaction, &Default::default(), &Default::default())
             .await?;
@@ -959,34 +1155,13 @@ impl DatasetIndexExt for Dataset {
             return Err(Error::index_not_found(format!("name={}", index_name)));
         }
         let column = self.schema().field_by_id(indices[0].fields[0]).unwrap();
-
-        let mut schema: Option<Arc<Schema>> = None;
-        let mut partition_streams = Vec::with_capacity(indices.len());
-        for index in indices {
-            let index = self
-                .open_vector_index(&column.name, &index.uuid.to_string(), &NoOpMetricsCollector)
-                .await?;
-
-            let stream = index
-                .partition_reader(partition_id, with_vector, &NoOpMetricsCollector)
-                .await?;
-            if schema.is_none() {
-                schema = Some(stream.schema());
-            }
-            partition_streams.push(stream);
-        }
-
-        match schema {
-            Some(schema) => {
-                let merged = stream::select_all(partition_streams);
-                let stream = RecordBatchStreamAdapter::new(schema, merged);
-                Ok(Box::pin(stream))
-            }
-            None => Ok(Box::pin(RecordBatchStreamAdapter::new(
-                Arc::new(Schema::empty()),
-                stream::empty(),
-            ))),
-        }
+        let logical_index = self
+            .open_logical_vector_index(&column.name, index_name)
+            .await?;
+        logical_index
+            .as_ivf()?
+            .read_partition(partition_id, with_vector)
+            .await
     }
 }
 
@@ -1089,7 +1264,9 @@ async fn index_statistics_scalar(
         "index_type": index_type,
         "name": index_name,
         "num_indices": num_indices,
-        "indices": indices_stats,
+        "num_segments": num_indices,
+        "indices": indices_stats.clone(),
+        "segments": indices_stats,
         "num_indexed_fragments": num_indexed_fragments,
         "num_indexed_rows": num_indexed_rows,
         "num_unindexed_fragments": num_unindexed_fragments,
@@ -1244,6 +1421,12 @@ pub trait DatasetIndexInternalExt: DatasetIndexExt {
         uuid: &str,
         metrics: &dyn MetricsCollector,
     ) -> Result<Arc<dyn VectorIndex>>;
+    /// Opens all segments for one logical vector index and returns a materialized snapshot.
+    async fn open_logical_vector_index(
+        &self,
+        column: &str,
+        name: &str,
+    ) -> Result<LogicalVectorIndex>;
 
     /// Opens the fragment reuse index
     async fn open_frag_reuse_index(
@@ -1292,15 +1475,20 @@ impl DatasetIndexInternalExt for Dataset {
             return Ok(index.as_index());
         }
 
-        let vector_cache_key = VectorIndexCacheKey::new(uuid, frag_reuse_uuid.as_ref());
-        if let Some(index) = self
-            .index_cache
-            .get_unsized_with_key(&vector_cache_key)
-            .await
-        {
+        // Check sized cache for IvfIndexState (v2+ indices).
+        let state_key = IvfIndexStateCacheKey::new(uuid, frag_reuse_uuid.as_ref());
+        if self.index_cache.get_with_key(&state_key).await.is_some() {
+            // Reconstruct via open_vector_index which will hit the same sized key.
+            let index = self.open_vector_index(column, uuid, metrics).await?;
             return Ok(index.as_index());
         }
 
+        // Fallback: in-memory cache for legacy indices.
+        let vector_cache_key = LegacyVectorIndexCacheKey::new(uuid, frag_reuse_uuid.as_ref());
+        if let Some(cached) = self.index_cache.get_with_key(&vector_cache_key).await {
+            return Ok(cached.0.clone().as_index());
+        }
+
         let frag_reuse_cache_key = FragReuseIndexCacheKey::new(uuid, frag_reuse_uuid.as_ref());
         if let Some(index) = self.index_cache.get_with_key(&frag_reuse_cache_key).await {
             return Ok(index.as_index());
@@ -1309,18 +1497,26 @@ impl DatasetIndexInternalExt for Dataset {
         // Sometimes we want to open an index and we don't care if it is a scalar or vector index.
         // For example, we might want to get statistics for an index, regardless of type.
         //
-        // Currently, we solve this problem by checking for the existence of INDEX_FILE_NAME since
-        // only vector indices have this file.  In the future, once we support multiple kinds of
-        // scalar indices, we may start having this file with scalar indices too.  Once that happens
-        // we can just read this file and look at the `implementation` or `index_type` fields to
-        // determine what kind of index it is.
+        // We determine if this is a vector index by checking if INDEX_FILE_NAME exists in the
+        // file list (available since file sizes tracking was added). If the file list is not
+        // available (older indices), we fall back to checking file existence via HEAD request.
         let index_meta = self
             .load_index(uuid)
             .await?
             .ok_or_else(|| Error::index(format!("Index with id {} does not exist", uuid)))?;
-        let index_dir = self.indice_files_dir(&index_meta)?;
-        let index_file = index_dir.child(uuid).child(INDEX_FILE_NAME);
-        if self.object_store.exists(&index_file).await? {
+
+        // Check if this is a vector index by looking at the files list
+        let is_vector_index = if let Some(files) = &index_meta.files {
+            // If we have file metadata, check if INDEX_FILE_NAME is in the list
+            files.iter().any(|f| f.path == INDEX_FILE_NAME)
+        } else {
+            // Fall back to file existence check for older indices without file metadata
+            let index_dir = self.indice_files_dir(&index_meta)?;
+            let index_file = index_dir.child(uuid).child(INDEX_FILE_NAME);
+            self.object_store.exists(&index_file).await?
+        };
+
+        if is_vector_index {
             let index = self.open_vector_index(column, uuid, metrics).await?;
             Ok(index.as_index())
         } else {
@@ -1365,11 +1561,26 @@ impl DatasetIndexInternalExt for Dataset {
         metrics: &dyn MetricsCollector,
     ) -> Result<Arc<dyn VectorIndex>> {
         let frag_reuse_uuid = self.frag_reuse_index_uuid().await;
-        let cache_key = VectorIndexCacheKey::new(uuid, frag_reuse_uuid.as_ref());
 
-        if let Some(index) = self.index_cache.get_unsized_with_key(&cache_key).await {
-            log::debug!("Found vector index in cache uuid: {}", uuid);
-            return Ok(index);
+        // Check sized cache first (v2+ indices with serializable state).
+        let state_key = IvfIndexStateCacheKey::new(uuid, frag_reuse_uuid.as_ref());
+        if let Some(entry) = self.index_cache.get_with_key(&state_key).await {
+            log::debug!("Found IvfIndexState in cache uuid: {}", uuid);
+            let partition_cache = self.index_cache.with_key_prefix(&state_key.key());
+            return entry
+                .0
+                .reconstruct(
+                    self.object_store.clone(),
+                    self.metadata_cache.as_ref(),
+                    partition_cache,
+                )
+                .await;
+        }
+
+        // Fallback: in-memory cache for legacy indices.
+        let cache_key = LegacyVectorIndexCacheKey::new(uuid, frag_reuse_uuid.as_ref());
+        if let Some(cached) = self.index_cache.get_with_key(&cache_key).await {
+            return Ok(cached.0.clone());
         }
 
         let frag_reuse_index = self.open_frag_reuse_index(metrics).await?;
@@ -1387,23 +1598,35 @@ impl DatasetIndexInternalExt for Dataset {
         // Namespace the index cache by the UUID of the index.
         let index_cache = self.index_cache.with_key_prefix(&cache_key.key());
 
+        // Extract the cacheable state before type-erasing to Arc<dyn VectorIndex>.
+        fn wrap_ivf<S: IvfSubIndex + 'static, Q: Quantization + 'static>(
+            ivf: IVFIndex<S, Q>,
+        ) -> (Arc<dyn VectorIndex>, Option<IvfStateEntryBox>) {
+            let entry = ivf.to_state_entry();
+            (Arc::new(ivf), Some(entry))
+        }
+
         // the index file is in lance format since version (0,2)
         // TODO: we need to change the legacy IVF_PQ to be in lance format
-        let index = match (major_version, minor_version) {
+        let result: Result<(Arc<dyn VectorIndex>, Option<IvfStateEntryBox>)> = match (
+            major_version,
+            minor_version,
+        ) {
             (0, 1) | (0, 0) => {
                 info!(target: TRACE_IO_EVENTS, index_uuid=uuid, r#type=IO_TYPE_OPEN_VECTOR, version="0.1", index_type="IVF_PQ");
                 let proto = open_index_proto(reader.as_ref()).await?;
                 match &proto.implementation {
                     Some(Implementation::VectorIndex(vector_index)) => {
                         let dataset = Arc::new(self.clone());
-                        vector::open_vector_index(
+                        let idx = vector::open_vector_index(
                             dataset,
                             uuid,
                             vector_index,
                             reader,
                             frag_reuse_index,
                         )
-                        .await
+                        .await?;
+                        Ok((idx, None::<IvfStateEntryBox>))
                     }
                     None => Err(Error::internal(
                         "Index proto was missing implementation field",
@@ -1418,14 +1641,15 @@ impl DatasetIndexInternalExt for Dataset {
                     Some(&self.metadata_cache.file_metadata_cache(&index_file)),
                 )
                 .await?;
-                vector::open_vector_index_v2(
+                let idx = vector::open_vector_index_v2(
                     Arc::new(self.clone()),
                     column,
                     uuid,
                     reader,
                     frag_reuse_index,
                 )
-                .await
+                .await?;
+                Ok((idx, None::<IvfStateEntryBox>))
             }
 
             (0, 3) | (2, _) => {
@@ -1433,9 +1657,12 @@ impl DatasetIndexInternalExt for Dataset {
                     self.object_store.clone(),
                     SchedulerConfig::max_bandwidth(&self.object_store),
                 );
-                let file = scheduler
-                    .open_file(&index_file, &CachedFileSize::unknown())
-                    .await?;
+                let file_sizes = index_meta.file_size_map();
+                let cached_size = file_sizes
+                    .get(INDEX_FILE_NAME)
+                    .map(|&size| CachedFileSize::new(size))
+                    .unwrap_or_else(CachedFileSize::unknown);
+                let file = scheduler.open_file(&index_file, &cached_size).await?;
                 let reader = lance_file::reader::FileReader::try_open(
                     file,
                     None,
@@ -1469,9 +1696,10 @@ impl DatasetIndexInternalExt for Dataset {
                                 frag_reuse_index,
                                 self.metadata_cache.as_ref(),
                                 index_cache,
+                                file_sizes,
                             )
                             .await?;
-                            Ok(Arc::new(ivf) as Arc<dyn VectorIndex>)
+                            Ok(wrap_ivf(ivf))
                         }
                         DataType::UInt8 => {
                             let ivf = IVFIndex::<FlatIndex, FlatBinQuantizer>::try_new(
@@ -1481,9 +1709,10 @@ impl DatasetIndexInternalExt for Dataset {
                                 frag_reuse_index,
                                 self.metadata_cache.as_ref(),
                                 index_cache,
+                                file_sizes,
                             )
                             .await?;
-                            Ok(Arc::new(ivf) as Arc<dyn VectorIndex>)
+                            Ok(wrap_ivf(ivf))
                         }
                         _ => Err(Error::index(format!(
                             "the field type {} is not supported for FLAT index",
@@ -1499,9 +1728,10 @@ impl DatasetIndexInternalExt for Dataset {
                             frag_reuse_index,
                             self.metadata_cache.as_ref(),
                             index_cache,
+                            file_sizes,
                         )
                         .await?;
-                        Ok(Arc::new(ivf) as Arc<dyn VectorIndex>)
+                        Ok(wrap_ivf(ivf))
                     }
 
                     "IVF_SQ" => {
@@ -1512,9 +1742,10 @@ impl DatasetIndexInternalExt for Dataset {
                             frag_reuse_index,
                             self.metadata_cache.as_ref(),
                             index_cache,
+                            file_sizes,
                         )
                         .await?;
-                        Ok(Arc::new(ivf) as Arc<dyn VectorIndex>)
+                        Ok(wrap_ivf(ivf))
                     }
 
                     "IVF_RQ" => {
@@ -1525,26 +1756,40 @@ impl DatasetIndexInternalExt for Dataset {
                             frag_reuse_index,
                             self.metadata_cache.as_ref(),
                             index_cache,
+                            file_sizes,
                         )
                         .await?;
-                        Ok(Arc::new(ivf) as Arc<dyn VectorIndex>)
+                        Ok(wrap_ivf(ivf))
                     }
 
-                    "IVF_HNSW_FLAT" => {
-                        let uri = index_dir.child(uuid).child("index.pb");
-                        let file_metadata_cache =
-                            self.session.metadata_cache.file_metadata_cache(&uri);
-                        let ivf = IVFIndex::<HNSW, FlatQuantizer>::try_new(
-                            self.object_store.clone(),
-                            index_dir,
-                            uuid.to_owned(),
-                            frag_reuse_index,
-                            &file_metadata_cache,
-                            index_cache,
-                        )
-                        .await?;
-                        Ok(Arc::new(ivf) as Arc<dyn VectorIndex>)
-                    }
+                    "IVF_HNSW_FLAT" => match element_type {
+                        DataType::UInt8 => {
+                            let ivf = IVFIndex::<HNSW, FlatBinQuantizer>::try_new(
+                                self.object_store.clone(),
+                                index_dir,
+                                uuid.to_owned(),
+                                frag_reuse_index,
+                                self.metadata_cache.as_ref(),
+                                index_cache,
+                                file_sizes,
+                            )
+                            .await?;
+                            Ok(wrap_ivf(ivf))
+                        }
+                        _ => {
+                            let ivf = IVFIndex::<HNSW, FlatQuantizer>::try_new(
+                                self.object_store.clone(),
+                                index_dir,
+                                uuid.to_owned(),
+                                frag_reuse_index,
+                                self.metadata_cache.as_ref(),
+                                index_cache,
+                                file_sizes,
+                            )
+                            .await?;
+                            Ok(wrap_ivf(ivf))
+                        }
+                    },
 
                     "IVF_HNSW_SQ" => {
                         let ivf = IVFIndex::<HNSW, ScalarQuantizer>::try_new(
@@ -1554,9 +1799,10 @@ impl DatasetIndexInternalExt for Dataset {
                             frag_reuse_index,
                             self.metadata_cache.as_ref(),
                             index_cache,
+                            file_sizes,
                         )
                         .await?;
-                        Ok(Arc::new(ivf) as Arc<dyn VectorIndex>)
+                        Ok(wrap_ivf(ivf))
                     }
 
                     "IVF_HNSW_PQ" => {
@@ -1567,9 +1813,10 @@ impl DatasetIndexInternalExt for Dataset {
                             frag_reuse_index,
                             self.metadata_cache.as_ref(),
                             index_cache,
+                            file_sizes,
                         )
                         .await?;
-                        Ok(Arc::new(ivf) as Arc<dyn VectorIndex>)
+                        Ok(wrap_ivf(ivf))
                     }
 
                     _ => Err(Error::index(format!(
@@ -1583,14 +1830,53 @@ impl DatasetIndexInternalExt for Dataset {
                 "unsupported index version (maybe need to upgrade your lance version)".to_owned(),
             )),
         };
-        let index = index?;
+        let (index, ivf_entry) = result?;
         metrics.record_index_load();
-        self.index_cache
-            .insert_unsized_with_key(&cache_key, index.clone())
-            .await;
+        if let Some(ivf_entry) = ivf_entry {
+            let state_key = IvfIndexStateCacheKey::new(uuid, frag_reuse_uuid.as_ref());
+            self.index_cache
+                .insert_with_key(&state_key, Arc::new(ivf_entry))
+                .await;
+        } else {
+            self.index_cache
+                .insert_with_key(&cache_key, Arc::new(CachedLegacyVectorIndex(index.clone())))
+                .await;
+        }
         Ok(index)
     }
 
+    async fn open_logical_vector_index(
+        &self,
+        column: &str,
+        name: &str,
+    ) -> Result<LogicalVectorIndex> {
+        let metadatas = self.load_indices_by_name(name).await?;
+        if metadatas.is_empty() {
+            return Err(Error::index_not_found(format!("name={name}")));
+        }
+
+        let field_id = self.schema().field_id(column)?;
+        if let Some(invalid_metadata) = metadatas
+            .iter()
+            .find(|metadata| !metadata.fields.contains(&field_id))
+        {
+            return Err(Error::invalid_input(format!(
+                "Logical vector index '{}' contains segment {} that does not belong to column '{}'",
+                name, invalid_metadata.uuid, column
+            )));
+        }
+
+        let mut segments = Vec::with_capacity(metadatas.len());
+        for metadata in metadatas {
+            let index = self
+                .open_vector_index(column, &metadata.uuid.to_string(), &NoOpMetricsCollector)
+                .await?;
+            segments.push((metadata, index));
+        }
+
+        LogicalVectorIndex::try_new(name.to_string(), column.to_string(), segments)
+    }
+
     async fn open_frag_reuse_index(
         &self,
         metrics: &dyn MetricsCollector,
@@ -1966,55 +2252,466 @@ mod tests {
     use lance_arrow::*;
     use lance_core::utils::tempfile::TempStrDir;
     use lance_datagen::gen_batch;
-    use lance_datagen::{BatchCount, Dimension, RowCount, array};
+    use lance_datagen::{BatchCount, ByteCount, Dimension, RowCount, array};
     use lance_index::scalar::bitmap::BITMAP_LOOKUP_NAME;
     use lance_index::scalar::{
         BuiltinIndexType, FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams,
     };
     use lance_index::vector::{
-        hnsw::builder::HnswBuildParams, ivf::IvfBuildParams, sq::builder::SQBuildParams,
+        hnsw::builder::HnswBuildParams,
+        ivf::IvfBuildParams,
+        kmeans::{KMeansParams, train_kmeans},
+        sq::builder::SQBuildParams,
     };
     use lance_io::{assert_io_eq, assert_io_lt};
     use lance_linalg::distance::{DistanceType, MetricType};
     use lance_testing::datagen::generate_random_array;
     use rstest::rstest;
-    use std::collections::HashSet;
+    use std::collections::{HashMap, HashSet};
 
-    #[tokio::test]
-    async fn test_recreate_index() {
-        const DIM: i32 = 8;
+    async fn write_vector_segment_metadata(
+        dataset: &Dataset,
+        index_name: &str,
+        field_id: i32,
+        uuid: Uuid,
+        fragment_bitmap: impl IntoIterator<Item = u32>,
+        payload: &[u8],
+    ) -> IndexMetadata {
+        let index_path = dataset
+            .indices_dir()
+            .child(uuid.to_string())
+            .child(INDEX_FILE_NAME);
+        dataset
+            .object_store()
+            .put(&index_path, payload)
+            .await
+            .unwrap();
+        IndexMetadata {
+            uuid,
+            name: index_name.to_string(),
+            fields: vec![field_id],
+            dataset_version: dataset.manifest.version,
+            fragment_bitmap: Some(fragment_bitmap.into_iter().collect()),
+            index_details: Some(Arc::new(vector_index_details())),
+            index_version: IndexType::Vector.version(),
+            created_at: Some(chrono::Utc::now()),
+            base_id: None,
+            files: Some(vec![lance_table::format::IndexFile {
+                path: INDEX_FILE_NAME.to_string(),
+                size_bytes: payload.len() as u64,
+            }]),
+        }
+    }
+
+    async fn write_fragmented_vector_dataset(uri: &str, dimension: i32) -> Dataset {
         let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
             Field::new(
-                "v",
-                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
-                true,
+                "vector",
+                DataType::FixedSizeList(
+                    Arc::new(Field::new("item", DataType::Float32, true)),
+                    dimension,
+                ),
+                false,
             ),
+        ]));
+        let batches = (0..5)
+            .map(|i| {
+                let vector_values: Float32Array = (0..dimension * 80)
+                    .map(|value| value as f32 + (i * 1000) as f32)
+                    .collect();
+                let vectors =
+                    FixedSizeListArray::try_new_from_values(vector_values, dimension).unwrap();
+                RecordBatch::try_new(
+                    schema.clone(),
+                    vec![
+                        Arc::new(Int32Array::from_iter_values(i * 80..(i + 1) * 80)),
+                        Arc::new(vectors),
+                    ],
+                )
+            })
+            .collect::<std::result::Result<Vec<_>, arrow_schema::ArrowError>>()
+            .unwrap();
+        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
+        Dataset::write(
+            reader,
+            uri,
+            Some(WriteParams {
+                max_rows_per_group: 10,
+                max_rows_per_file: 80,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap()
+    }
+
+    async fn write_skewed_fragmented_vector_dataset(uri: &str, dimension: i32) -> Dataset {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
             Field::new(
-                "o",
-                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
-                true,
+                "vector",
+                DataType::FixedSizeList(
+                    Arc::new(Field::new("item", DataType::Float32, true)),
+                    dimension,
+                ),
+                false,
             ),
         ]));
-        let data = generate_random_array(2048 * DIM as usize);
-        let batches: Vec<RecordBatch> = vec![
+        let first_fragment_rows = 20_000;
+        let second_fragment_rows = 100;
+        let first_values = vec![0.0f32; first_fragment_rows * dimension as usize];
+        let second_values = vec![100.0f32; second_fragment_rows * dimension as usize];
+        let first_vectors =
+            FixedSizeListArray::try_new_from_values(Float32Array::from(first_values), dimension)
+                .unwrap();
+        let second_vectors =
+            FixedSizeListArray::try_new_from_values(Float32Array::from(second_values), dimension)
+                .unwrap();
+        let batches = vec![
             RecordBatch::try_new(
                 schema.clone(),
                 vec![
-                    Arc::new(FixedSizeListArray::try_new_from_values(data.clone(), DIM).unwrap()),
-                    Arc::new(FixedSizeListArray::try_new_from_values(data, DIM).unwrap()),
+                    Arc::new(Int32Array::from_iter_values(0..first_fragment_rows as i32)),
+                    Arc::new(first_vectors),
+                ],
+            )
+            .unwrap(),
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(
+                        first_fragment_rows as i32
+                            ..(first_fragment_rows + second_fragment_rows) as i32,
+                    )),
+                    Arc::new(second_vectors),
                 ],
             )
             .unwrap(),
         ];
+        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
+        Dataset::write(
+            reader,
+            uri,
+            Some(WriteParams {
+                max_rows_per_group: 1024,
+                max_rows_per_file: first_fragment_rows,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap()
+    }
 
-        let test_dir = TempStrDir::default();
-        let test_uri = &test_dir;
-        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
-        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
-
-        let params = VectorIndexParams::ivf_pq(2, 8, 2, MetricType::L2, 2);
-        dataset
-            .create_index(&["v"], IndexType::Vector, None, &params, true)
+    async fn create_segmented_vector_index(
+        dataset: &mut Dataset,
+        index_name: &str,
+        column: &str,
+        dimension: i32,
+    ) -> Vec<Uuid> {
+        let batch = dataset
+            .scan()
+            .project(&[column])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let vectors = batch
+            .column_by_name(column)
+            .expect("vector column should exist")
+            .as_fixed_size_list();
+        let values = vectors.values().as_primitive::<Float32Type>();
+        let centroids = train_kmeans::<Float32Type>(
+            values,
+            KMeansParams::new(None, 10, 1, DistanceType::L2),
+            dimension as usize,
+            2,
+            2,
+        )
+        .unwrap()
+        .centroids
+        .as_primitive::<Float32Type>()
+        .clone();
+        let centroids =
+            Arc::new(FixedSizeListArray::try_new_from_values(centroids, dimension).unwrap());
+        let params = VectorIndexParams::with_ivf_flat_params(
+            DistanceType::L2,
+            IvfBuildParams::try_with_centroids(2, centroids).unwrap(),
+        );
+        let fragment_ids = dataset
+            .get_fragments()
+            .iter()
+            .map(|fragment| fragment.id() as u32)
+            .collect::<Vec<_>>();
+        let columns = [column];
+
+        let mut segments = Vec::with_capacity(fragment_ids.len());
+        for fragment_id in fragment_ids {
+            let mut builder = dataset.create_index_builder(&columns, IndexType::Vector, &params);
+            builder = builder
+                .name(index_name.to_string())
+                .fragments(vec![fragment_id]);
+            segments.push(builder.execute_uncommitted().await.unwrap());
+        }
+
+        let segment_ids = segments
+            .iter()
+            .map(|segment| segment.uuid)
+            .collect::<Vec<_>>();
+        dataset
+            .commit_existing_index_segments(index_name, column, segments)
+            .await
+            .unwrap();
+        segment_ids
+    }
+
+    #[tokio::test]
+    async fn test_open_logical_vector_index_single_segment_quality_apis() {
+        const DIMENSION: i32 = 8;
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+        let mut dataset = write_fragmented_vector_dataset(test_uri, DIMENSION).await;
+        let params =
+            VectorIndexParams::with_ivf_flat_params(DistanceType::L2, IvfBuildParams::new(2));
+
+        dataset
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some("vector_idx".to_string()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        let logical_index = dataset
+            .open_logical_vector_index("vector", "vector_idx")
+            .await
+            .unwrap();
+
+        assert_eq!(logical_index.name(), "vector_idx");
+        assert_eq!(logical_index.column(), "vector");
+        assert_eq!(logical_index.num_segments(), 1);
+        assert_eq!(logical_index.metadatas().len(), 1);
+
+        let rows_per_segment = logical_index.num_rows_per_segment();
+        assert_eq!(rows_per_segment.len(), 1);
+        assert_eq!(rows_per_segment[0].1, 400);
+
+        let ivf_view = logical_index.as_ivf().unwrap();
+        let partitions_per_segment = ivf_view.num_partitions_per_segment();
+        assert_eq!(partitions_per_segment, vec![(rows_per_segment[0].0, 2)]);
+
+        let partition_sizes = ivf_view.partition_sizes();
+        assert_eq!(partition_sizes.len(), 1);
+        assert_eq!(partition_sizes[0].1.len(), 2);
+        assert_eq!(
+            partition_sizes[0].1.iter().sum::<usize>(),
+            rows_per_segment[0].1 as usize
+        );
+    }
+
+    #[tokio::test]
+    async fn test_open_logical_vector_index_segmented_quality_apis() {
+        const DIMENSION: i32 = 8;
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+        let mut dataset = write_fragmented_vector_dataset(test_uri, DIMENSION).await;
+        let segment_ids =
+            create_segmented_vector_index(&mut dataset, "vector_idx", "vector", DIMENSION).await;
+
+        let logical_index = dataset
+            .open_logical_vector_index("vector", "vector_idx")
+            .await
+            .unwrap();
+
+        assert_eq!(logical_index.name(), "vector_idx");
+        assert_eq!(logical_index.column(), "vector");
+        assert_eq!(logical_index.num_segments(), segment_ids.len());
+
+        let metadata_ids = logical_index
+            .metadatas()
+            .map(|metadata| metadata.uuid)
+            .collect::<HashSet<_>>();
+        assert_eq!(
+            metadata_ids,
+            segment_ids.into_iter().collect::<HashSet<_>>()
+        );
+
+        let rows_per_segment = logical_index.num_rows_per_segment();
+        assert_eq!(rows_per_segment.len(), logical_index.num_segments());
+        assert_eq!(
+            rows_per_segment
+                .iter()
+                .map(|(_, num_rows)| *num_rows)
+                .sum::<u64>(),
+            400
+        );
+        assert!(
+            rows_per_segment.iter().all(|(_, num_rows)| *num_rows > 0),
+            "each segment should contain indexed rows"
+        );
+
+        let ivf_view = logical_index.as_ivf().unwrap();
+        let partitions_per_segment = ivf_view.num_partitions_per_segment();
+        assert!(
+            partitions_per_segment
+                .iter()
+                .all(|(_, num_partitions)| *num_partitions == 2)
+        );
+
+        let row_count_by_segment = rows_per_segment.into_iter().collect::<HashMap<_, _>>();
+        let partition_sizes = ivf_view.partition_sizes();
+        assert_eq!(partition_sizes.len(), logical_index.num_segments());
+        for (segment_id, sizes) in partition_sizes {
+            assert_eq!(sizes.len(), 2);
+            assert_eq!(
+                sizes.iter().sum::<usize>(),
+                row_count_by_segment[&segment_id] as usize
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn test_open_logical_vector_index_rejects_wrong_column() {
+        const DIMENSION: i32 = 8;
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+        let mut dataset = write_fragmented_vector_dataset(test_uri, DIMENSION).await;
+        let params =
+            VectorIndexParams::with_ivf_flat_params(DistanceType::L2, IvfBuildParams::new(2));
+
+        dataset
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some("vector_idx".to_string()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        let err = dataset
+            .open_logical_vector_index("id", "vector_idx")
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string().contains("does not belong to column 'id'"),
+            "unexpected error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_segmented_optimize_rebalances_only_one_segment() {
+        const DIMENSION: i32 = 8;
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+        let mut dataset = write_skewed_fragmented_vector_dataset(test_uri, DIMENSION).await;
+        create_segmented_vector_index(&mut dataset, "vector_idx", "vector", DIMENSION).await;
+
+        let before_segments = dataset.load_indices_by_name("vector_idx").await.unwrap();
+        assert_eq!(before_segments.len(), 2);
+        let before_by_fragment = before_segments
+            .iter()
+            .map(|metadata| {
+                let fragments = metadata
+                    .fragment_bitmap
+                    .as_ref()
+                    .unwrap()
+                    .iter()
+                    .collect::<Vec<_>>();
+                (fragments, metadata.uuid)
+            })
+            .collect::<HashMap<_, _>>();
+        assert_eq!(before_by_fragment.len(), 2);
+
+        dataset
+            .optimize_indices(&OptimizeOptions::default())
+            .await
+            .unwrap();
+
+        let after_segments = dataset.load_indices_by_name("vector_idx").await.unwrap();
+        assert_eq!(after_segments.len(), 2);
+        let after_by_fragment = after_segments
+            .iter()
+            .map(|metadata| {
+                let fragments = metadata
+                    .fragment_bitmap
+                    .as_ref()
+                    .unwrap()
+                    .iter()
+                    .collect::<Vec<_>>();
+                (fragments, metadata.uuid)
+            })
+            .collect::<HashMap<_, _>>();
+        assert_eq!(after_by_fragment.len(), 2);
+
+        assert_ne!(
+            before_by_fragment[&vec![0]],
+            after_by_fragment[&vec![0]],
+            "expected optimize to replace the oversized segment"
+        );
+        assert_eq!(
+            before_by_fragment[&vec![1]],
+            after_by_fragment[&vec![1]],
+            "expected optimize to leave the smaller segment untouched"
+        );
+
+        let logical_index = dataset
+            .open_logical_vector_index("vector", "vector_idx")
+            .await
+            .unwrap();
+        let partitions_per_segment = logical_index
+            .as_ivf()
+            .unwrap()
+            .num_partitions_per_segment()
+            .into_iter()
+            .collect::<HashMap<_, _>>();
+        assert_eq!(partitions_per_segment[&after_by_fragment[&vec![0]]], 3);
+        assert_eq!(partitions_per_segment[&after_by_fragment[&vec![1]]], 2);
+    }
+
+    #[tokio::test]
+    async fn test_recreate_index() {
+        const DIM: i32 = 8;
+        let schema = Arc::new(Schema::new(vec![
+            Field::new(
+                "v",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
+                true,
+            ),
+            Field::new(
+                "o",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
+                true,
+            ),
+        ]));
+        let data = generate_random_array(2048 * DIM as usize);
+        let batches: Vec<RecordBatch> = vec![
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(FixedSizeListArray::try_new_from_values(data.clone(), DIM).unwrap()),
+                    Arc::new(FixedSizeListArray::try_new_from_values(data, DIM).unwrap()),
+                ],
+            )
+            .unwrap(),
+        ];
+
+        let test_dir = TempStrDir::default();
+        let test_uri = &test_dir;
+        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
+
+        let params = VectorIndexParams::ivf_pq(2, 8, 2, MetricType::L2, 2);
+        dataset
+            .create_index(&["v"], IndexType::Vector, None, &params, true)
             .await
             .unwrap();
         dataset
@@ -2318,8 +3015,13 @@ mod tests {
         fn get_bitmap(meta: &IndexMetadata) -> Vec<u32> {
             meta.fragment_bitmap.as_ref().unwrap().iter().collect()
         }
+        fn assert_segment_aliases(stats: &serde_json::Value) {
+            assert_eq!(stats["num_segments"], stats["num_indices"]);
+            assert_eq!(stats["segments"], stats["indices"]);
+        }
 
         let stats = get_stats(&dataset, "vec_idx").await;
+        assert_segment_aliases(&stats);
         assert_eq!(stats["num_unindexed_rows"], 0);
         assert_eq!(stats["num_indexed_rows"], 512);
         assert_eq!(stats["num_indexed_fragments"], 1);
@@ -2332,6 +3034,7 @@ mod tests {
             RecordBatchIterator::new(vec![record_batch].into_iter().map(Ok), schema.clone());
         dataset.append(reader, None).await.unwrap();
         let stats = get_stats(&dataset, "vec_idx").await;
+        assert_segment_aliases(&stats);
         assert_eq!(stats["num_unindexed_rows"], 512);
         assert_eq!(stats["num_indexed_rows"], 512);
         assert_eq!(stats["num_indexed_fragments"], 1);
@@ -2346,6 +3049,7 @@ mod tests {
             .await
             .unwrap();
         let stats = get_stats(&dataset, "vec_idx").await;
+        assert_segment_aliases(&stats);
         assert_eq!(stats["num_unindexed_rows"], 512);
         assert_eq!(stats["num_indexed_rows"], 512);
         assert_eq!(stats["num_indexed_fragments"], 1);
@@ -2363,6 +3067,7 @@ mod tests {
             .await
             .unwrap();
         let stats = get_stats(&dataset, "vec_idx").await;
+        assert_segment_aliases(&stats);
         assert_eq!(stats["num_unindexed_rows"], 512);
         assert_eq!(stats["num_indexed_rows"], 512);
         assert_eq!(stats["num_indexed_fragments"], 1);
@@ -2373,6 +3078,7 @@ mod tests {
         assert_eq!(get_bitmap(&meta[0]), vec![0]);
 
         let stats = get_stats(&dataset, "other_vec_idx").await;
+        assert_segment_aliases(&stats);
         assert_eq!(stats["num_unindexed_rows"], 0);
         assert_eq!(stats["num_indexed_rows"], 1024);
         assert_eq!(stats["num_indexed_fragments"], 2);
@@ -2389,6 +3095,7 @@ mod tests {
             .unwrap();
 
         let stats = get_stats(&dataset, "vec_idx").await;
+        assert_segment_aliases(&stats);
         assert_eq!(stats["num_unindexed_rows"], 0);
         assert_eq!(stats["num_indexed_rows"], 1024);
         assert_eq!(stats["num_indexed_fragments"], 2);
@@ -2403,6 +3110,7 @@ mod tests {
             .await
             .unwrap();
         let stats = get_stats(&dataset, "other_vec_idx").await;
+        assert_segment_aliases(&stats);
         assert_eq!(stats["num_unindexed_rows"], 0);
         assert_eq!(stats["num_indexed_rows"], 1024);
         assert_eq!(stats["num_indexed_fragments"], 2);
@@ -5123,10 +5831,9 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_resolve_index_column_error_cases() {
+    async fn test_commit_existing_index_segments_commits_multiple_segments() {
         use lance_datagen::{BatchCount, RowCount, array};
 
-        // Create a test dataset
         let test_dir = tempfile::tempdir().unwrap();
         let test_uri = test_dir.path().to_str().unwrap();
 
@@ -5134,94 +5841,293 @@ mod tests {
             .col("id", array::step::<arrow_array::types::Int32Type>())
             .col(
                 "vector",
-                array::rand_vec::<arrow_array::types::Float32Type>(32.into()),
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
             )
-            .into_reader_rows(RowCount::from(100), BatchCount::from(1));
+            .into_reader_rows(RowCount::from(20), BatchCount::from(2));
+
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 10,
+                max_rows_per_group: 10,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
 
-        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
+        let field_id = dataset.schema().field("vector").unwrap().id;
+        let seg0 = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [0_u32],
+            b"seg0",
+        )
+        .await;
+        let seg1 = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [1_u32],
+            b"seg1",
+        )
+        .await;
 
-        // Create an index
-        let params = crate::index::vector::VectorIndexParams::ivf_flat(
-            4,
-            lance_linalg::distance::MetricType::L2,
-        );
         dataset
-            .create_index(
-                &["vector"],
-                IndexType::Vector,
-                Some("my_index".to_string()),
-                &params,
-                false,
+            .commit_existing_index_segments(
+                "vector_idx",
+                "vector",
+                vec![seg0.clone(), seg1.clone()],
             )
             .await
             .unwrap();
 
-        // Reload dataset
-        let dataset = Dataset::open(test_uri).await.unwrap();
-        let indices = dataset.load_indices().await.unwrap();
-        let index_meta = &indices[0];
-
-        // Test: Pass a column that doesn't exist and is not the index name
-        let result = resolve_index_column(dataset.schema(), index_meta, "nonexistent_column");
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
+        let committed = dataset.load_indices_by_name("vector_idx").await.unwrap();
+        assert_eq!(committed.len(), 2);
+        let committed_uuids = committed.iter().map(|idx| idx.uuid).collect::<HashSet<_>>();
+        assert_eq!(
+            committed_uuids,
+            HashSet::from([seg0.uuid, seg1.uuid]),
+            "all committed segment uuids should be preserved"
+        );
+        assert_eq!(
+            committed
+                .iter()
+                .map(|idx| idx
+                    .fragment_bitmap
+                    .as_ref()
+                    .unwrap()
+                    .iter()
+                    .collect::<Vec<_>>())
+                .collect::<HashSet<_>>(),
+            HashSet::from([vec![0], vec![1]]),
+            "each committed segment should preserve its fragment coverage"
+        );
         assert!(
-            err_msg.contains("does not exist in the schema"),
-            "Error message should mention column doesn't exist, got: {}",
-            err_msg
+            committed
+                .iter()
+                .all(|idx| idx.files.as_ref().is_some_and(|files| !files.is_empty())),
+            "committed segment metadata should capture on-disk file info"
         );
     }
 
     #[tokio::test]
-    async fn test_resolve_index_column_nested_field() {
-        use arrow_array::{RecordBatch, StructArray};
-        use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
+    async fn test_commit_existing_index_segments_rejects_duplicate_segment_ids() {
+        use lance_datagen::{BatchCount, RowCount, array};
 
-        // Create a test dataset with nested struct manually
         let test_dir = tempfile::tempdir().unwrap();
         let test_uri = test_dir.path().to_str().unwrap();
 
-        // Create schema with nested structure: data.vector
-        let vector_field = ArrowField::new(
-            "vector",
-            DataType::FixedSizeList(
-                Arc::new(ArrowField::new("item", DataType::Float32, true)),
-                8,
-            ),
-            false,
-        );
-        let struct_field = ArrowField::new(
-            "data",
-            DataType::Struct(vec![vector_field.clone()].into()),
-            false,
-        );
-        let schema = Arc::new(ArrowSchema::new(vec![
-            ArrowField::new("id", DataType::Int32, false),
-            struct_field,
-        ]));
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(10), BatchCount::from(1));
 
-        // Create data
-        let id_array = arrow_array::Int32Array::from(vec![1, 2, 3, 4, 5]);
+        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
 
-        // Create nested vector data
-        let mut vector_values = Vec::new();
-        for _ in 0..5 {
-            for _ in 0..8 {
-                vector_values.push(rand::random::<f32>());
-            }
-        }
-        let vector_array = arrow_array::FixedSizeListArray::try_new_from_values(
-            arrow_array::Float32Array::from(vector_values),
-            8,
+        let field_id = dataset.schema().field("vector").unwrap().id;
+        let base = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [0_u32],
+            b"base",
         )
-        .unwrap();
-
-        let struct_array = StructArray::from(vec![(
-            Arc::new(vector_field),
-            Arc::new(vector_array) as arrow_array::ArrayRef,
-        )]);
+        .await;
 
-        let batch = RecordBatch::try_new(
+        let err = dataset
+            .commit_existing_index_segments(
+                "vector_idx",
+                "vector",
+                vec![
+                    base.clone(),
+                    IndexMetadata {
+                        fragment_bitmap: Some(std::iter::once(1_u32).collect()),
+                        ..base
+                    },
+                ],
+            )
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("duplicate segment uuid"));
+    }
+
+    #[tokio::test]
+    async fn test_commit_existing_index_segments_rejects_empty_segments() {
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(10), BatchCount::from(1));
+
+        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
+
+        let err = dataset
+            .commit_existing_index_segments("vector_idx", "vector", vec![])
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("at least one index segment"));
+    }
+
+    #[tokio::test]
+    async fn test_commit_existing_index_segments_rejects_overlapping_fragment_coverage() {
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(20), BatchCount::from(2));
+
+        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
+
+        let field_id = dataset.schema().field("vector").unwrap().id;
+        let seg0 = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [0_u32, 1_u32],
+            b"seg0",
+        )
+        .await;
+        let seg1 = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [1_u32],
+            b"seg1",
+        )
+        .await;
+
+        let err = dataset
+            .commit_existing_index_segments("vector_idx", "vector", vec![seg0, seg1])
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("overlapping fragment coverage"));
+    }
+
+    #[tokio::test]
+    async fn test_resolve_index_column_error_cases() {
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        // Create a test dataset
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(32.into()),
+            )
+            .into_reader_rows(RowCount::from(100), BatchCount::from(1));
+
+        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
+
+        // Create an index
+        let params = crate::index::vector::VectorIndexParams::ivf_flat(
+            4,
+            lance_linalg::distance::MetricType::L2,
+        );
+        dataset
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some("my_index".to_string()),
+                &params,
+                false,
+            )
+            .await
+            .unwrap();
+
+        // Reload dataset
+        let dataset = Dataset::open(test_uri).await.unwrap();
+        let indices = dataset.load_indices().await.unwrap();
+        let index_meta = &indices[0];
+
+        // Test: Pass a column that doesn't exist and is not the index name
+        let result = resolve_index_column(dataset.schema(), index_meta, "nonexistent_column");
+        assert!(result.is_err());
+        let err_msg = result.unwrap_err().to_string();
+        assert!(
+            err_msg.contains("does not exist in the schema"),
+            "Error message should mention column doesn't exist, got: {}",
+            err_msg
+        );
+    }
+
+    #[tokio::test]
+    async fn test_resolve_index_column_nested_field() {
+        use arrow_array::{RecordBatch, StructArray};
+        use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
+
+        // Create a test dataset with nested struct manually
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+
+        // Create schema with nested structure: data.vector
+        let vector_field = ArrowField::new(
+            "vector",
+            DataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                8,
+            ),
+            false,
+        );
+        let struct_field = ArrowField::new(
+            "data",
+            DataType::Struct(vec![vector_field.clone()].into()),
+            false,
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int32, false),
+            struct_field,
+        ]));
+
+        // Create data
+        let id_array = arrow_array::Int32Array::from(vec![1, 2, 3, 4, 5]);
+
+        // Create nested vector data
+        let mut vector_values = Vec::new();
+        for _ in 0..5 {
+            for _ in 0..8 {
+                vector_values.push(rand::random::<f32>());
+            }
+        }
+        let vector_array = arrow_array::FixedSizeListArray::try_new_from_values(
+            arrow_array::Float32Array::from(vector_values),
+            8,
+        )
+        .unwrap();
+
+        let struct_array = StructArray::from(vec![(
+            Arc::new(vector_field),
+            Arc::new(vector_array) as arrow_array::ArrayRef,
+        )]);
+
+        let batch = RecordBatch::try_new(
             schema.clone(),
             vec![Arc::new(id_array), Arc::new(struct_array)],
         )
@@ -5275,6 +6181,637 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_scalar_index_file_sizes_captured() {
+        // Test that file sizes are captured when creating a scalar index
+        let reader = gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col("values", array::rand_utf8(ByteCount::from(10), false))
+            .into_reader_rows(RowCount::from(4), BatchCount::from(1));
+
+        let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap();
+
+        // Create a scalar index
+        dataset
+            .create_index(
+                &["values"],
+                IndexType::Scalar,
+                Some("test_idx".to_string()),
+                &ScalarIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+
+        // Get index metadata and verify files are populated
+        let indices = dataset.load_indices().await.unwrap();
+        let test_index = indices.iter().find(|idx| idx.name == "test_idx").unwrap();
+
+        assert!(
+            test_index.files.is_some(),
+            "Index should have files populated"
+        );
+        let files = test_index.files.as_ref().unwrap();
+        assert!(!files.is_empty(), "Index should have at least one file");
+
+        // Verify each file has a positive size
+        for file in files {
+            assert!(
+                file.size_bytes > 0,
+                "File {} should have positive size",
+                file.path
+            );
+        }
+
+        // Verify total_size_bytes works
+        let total_size = test_index.total_size_bytes();
+        assert!(total_size.is_some(), "total_size_bytes should return Some");
+        assert!(total_size.unwrap() > 0, "Total size should be positive");
+    }
+
+    #[tokio::test]
+    async fn test_vector_index_file_sizes_captured() {
+        // Test that file sizes are captured when creating a vector index
+        let reader = gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(4.into()),
+            )
+            .into_reader_rows(RowCount::from(300), BatchCount::from(1));
+
+        let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap();
+
+        // Create vector index
+        let params = VectorIndexParams::ivf_pq(1, 8, 2, MetricType::L2, 2);
+        dataset
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some("test_vec_idx".to_string()),
+                &params,
+                false,
+            )
+            .await
+            .unwrap();
+
+        // Get index metadata and verify files are populated
+        let indices = dataset.load_indices().await.unwrap();
+        let test_index = indices
+            .iter()
+            .find(|idx| idx.name == "test_vec_idx")
+            .unwrap();
+
+        assert!(
+            test_index.files.is_some(),
+            "Index should have files populated"
+        );
+        let files = test_index.files.as_ref().unwrap();
+        assert!(!files.is_empty(), "Index should have at least one file");
+
+        // Verify each file has a positive size
+        for file in files {
+            assert!(
+                file.size_bytes > 0,
+                "File {} should have positive size",
+                file.path
+            );
+        }
+
+        // Verify total_size_bytes works
+        let total_size = test_index.total_size_bytes();
+        assert!(total_size.is_some(), "total_size_bytes should return Some");
+        assert!(total_size.unwrap() > 0, "Total size should be positive");
+    }
+
+    #[tokio::test]
+    async fn test_describe_indices_total_size() {
+        // Test that describe_indices returns total_size_bytes
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("values", DataType::Utf8, false),
+        ]));
+
+        let values = StringArray::from_iter_values(["hello", "world", "foo", "bar"]);
+        let record_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..4)),
+                Arc::new(values),
+            ],
+        )
+        .unwrap();
+
+        let reader =
+            RecordBatchIterator::new(vec![record_batch].into_iter().map(Ok), schema.clone());
+
+        let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap();
+
+        // Create a scalar index
+        dataset
+            .create_index(
+                &["values"],
+                IndexType::Scalar,
+                Some("test_idx".to_string()),
+                &ScalarIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+
+        // Use describe_indices to get index info
+        let descriptions = dataset.describe_indices(None).await.unwrap();
+        assert_eq!(descriptions.len(), 1);
+
+        let desc = &descriptions[0];
+        assert_eq!(desc.name(), "test_idx");
+
+        // Verify total_size_bytes is available
+        let total_size = desc.total_size_bytes();
+        assert!(total_size.is_some(), "total_size_bytes should be Some");
+        assert!(total_size.unwrap() > 0, "Total size should be positive");
+    }
+
+    /// Helper to assert that all indices have file sizes populated
+    async fn assert_all_indices_have_files(dataset: &Dataset, context: &str) {
+        let indices = dataset.load_indices().await.unwrap();
+        for index in indices.iter() {
+            // Skip system indices (mem_wal, frag_reuse) which don't have files
+            if index.name == lance_index::mem_wal::MEM_WAL_INDEX_NAME
+                || index.name == lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME
+            {
+                continue;
+            }
+            assert!(
+                index.files.is_some(),
+                "{}: Index '{}' should have files field populated",
+                context,
+                index.name
+            );
+            let files = index.files.as_ref().unwrap();
+            assert!(
+                !files.is_empty(),
+                "{}: Index '{}' should have at least one file",
+                context,
+                index.name
+            );
+            for file in files {
+                assert!(
+                    file.size_bytes > 0,
+                    "{}: Index '{}' file '{}' should have positive size",
+                    context,
+                    index.name,
+                    file.path
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn test_index_file_sizes_through_lifecycle() {
+        use crate::dataset::WriteDestination;
+        use crate::dataset::optimize::{CompactionOptions, compact_files, remapping};
+        use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME;
+
+        // Create initial dataset with columns for different index types
+        let data = gen_batch()
+            .col("int_col", array::step::<Int32Type>())
+            .col("str_col", array::rand_utf8(8.into(), false))
+            .col(
+                "vec_col",
+                array::rand_vec::<Float32Type>(Dimension::from(32)),
+            )
+            .into_reader_rows(RowCount::from(1000), BatchCount::from(1));
+
+        let test_dir = TempStrDir::default();
+        let mut dataset = Dataset::write(
+            data,
+            test_dir.as_str(),
+            Some(WriteParams {
+                max_rows_per_file: 200, // Multiple fragments for compaction
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // Create BTree index
+        dataset
+            .create_index(
+                &["int_col"],
+                IndexType::BTree,
+                Some("btree_idx".to_string()),
+                &ScalarIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+
+        // Create Bitmap index
+        dataset
+            .create_index(
+                &["int_col"],
+                IndexType::Bitmap,
+                Some("bitmap_idx".to_string()),
+                &ScalarIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+
+        // Create Inverted index for text search
+        dataset
+            .create_index(
+                &["str_col"],
+                IndexType::Inverted,
+                Some("inverted_idx".to_string()),
+                &InvertedIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+
+        // Validate files are populated after creation
+        assert_all_indices_have_files(&dataset, "after initial creation").await;
+
+        // Append more data
+        let more_data = gen_batch()
+            .col("int_col", array::step::<Int32Type>())
+            .col("str_col", array::rand_utf8(8.into(), false))
+            .col(
+                "vec_col",
+                array::rand_vec::<Float32Type>(Dimension::from(32)),
+            )
+            .into_reader_rows(RowCount::from(500), BatchCount::from(1));
+
+        Dataset::write(
+            more_data,
+            WriteDestination::Dataset(Arc::new(dataset.clone())),
+            Some(WriteParams {
+                max_rows_per_file: 200,
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        dataset = DatasetBuilder::from_uri(test_dir.as_str())
+            .load()
+            .await
+            .unwrap();
+
+        // Optimize indices (triggers update/merge)
+        dataset
+            .optimize_indices(&OptimizeOptions::default())
+            .await
+            .unwrap();
+
+        // Validate files are still populated after optimize
+        assert_all_indices_have_files(&dataset, "after optimize_indices").await;
+
+        // Run compaction with deferred remap
+        let options = CompactionOptions {
+            target_rows_per_fragment: 500,
+            defer_index_remap: true,
+            ..Default::default()
+        };
+
+        compact_files(&mut dataset, options.clone(), None)
+            .await
+            .unwrap();
+
+        // Check if frag reuse index exists (indicates remap is needed)
+        if dataset
+            .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+            .await
+            .unwrap()
+            .is_some()
+        {
+            // Remap each index
+            remapping::remap_column_index(
+                &mut dataset,
+                &["int_col"],
+                Some("btree_idx".to_string()),
+            )
+            .await
+            .unwrap();
+
+            remapping::remap_column_index(
+                &mut dataset,
+                &["int_col"],
+                Some("bitmap_idx".to_string()),
+            )
+            .await
+            .unwrap();
+
+            remapping::remap_column_index(
+                &mut dataset,
+                &["str_col"],
+                Some("inverted_idx".to_string()),
+            )
+            .await
+            .unwrap();
+
+            // Validate files are populated after remap
+            assert_all_indices_have_files(&dataset, "after remap").await;
+        }
+    }
+
+    #[tokio::test]
+    async fn test_btree_index_iops() {
+        // Test that querying a BTree index uses minimal IOPs (no HEAD requests)
+        let test_dir = TempStrDir::default();
+
+        // Create dataset with a column suitable for BTree index
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("value", DataType::Int32, false),
+        ]));
+
+        let num_rows = 1000;
+        let ids = Int32Array::from_iter_values(0..num_rows);
+        let values = Int32Array::from_iter_values((0..num_rows).map(|i| i % 100));
+
+        let batch =
+            RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(values)]).unwrap();
+
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(reader, test_dir.as_str(), None)
+            .await
+            .unwrap();
+
+        // Create BTree index
+        dataset
+            .create_index(
+                &["value"],
+                IndexType::BTree,
+                Some("btree_idx".to_string()),
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Re-open dataset fresh to avoid cached state
+        let dataset = DatasetBuilder::from_uri(test_dir.as_str())
+            .load()
+            .await
+            .unwrap();
+
+        // Reset IO stats before query
+        let _ = dataset.object_store().io_stats_incremental();
+
+        // Query using the BTree index
+        let results = dataset
+            .scan()
+            .filter("value = 50")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert!(results.num_rows() > 0);
+
+        // Verify IOPs - should be minimal (no HEAD requests)
+        let stats = dataset.object_store().io_stats_incremental();
+        // We expect reads for: index metadata + index pages + data files
+        // The key assertion is that we don't have extra HEAD requests
+        assert_io_lt!(
+            stats,
+            read_iops,
+            10,
+            "BTree index query should use minimal IOPs"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_bitmap_index_iops() {
+        // Test that querying a Bitmap index uses minimal IOPs (no HEAD requests)
+        let test_dir = TempStrDir::default();
+
+        // Create dataset with low-cardinality column for Bitmap index
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("category", DataType::Int32, false),
+        ]));
+
+        let num_rows = 1000;
+        let ids = Int32Array::from_iter_values(0..num_rows);
+        // Low cardinality - only 10 unique values
+        let categories = Int32Array::from_iter_values((0..num_rows).map(|i| i % 10));
+
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(categories)])
+            .unwrap();
+
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(reader, test_dir.as_str(), None)
+            .await
+            .unwrap();
+
+        // Create Bitmap index
+        dataset
+            .create_index(
+                &["category"],
+                IndexType::Bitmap,
+                Some("bitmap_idx".to_string()),
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Re-open dataset fresh
+        let dataset = DatasetBuilder::from_uri(test_dir.as_str())
+            .load()
+            .await
+            .unwrap();
+
+        // Reset IO stats before query
+        let _ = dataset.object_store().io_stats_incremental();
+
+        // Query using the Bitmap index
+        let results = dataset
+            .scan()
+            .filter("category = 5")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert!(results.num_rows() > 0);
+
+        // Verify IOPs
+        let stats = dataset.object_store().io_stats_incremental();
+        assert_io_lt!(
+            stats,
+            read_iops,
+            10,
+            "Bitmap index query should use minimal IOPs"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_inverted_index_iops() {
+        // Test that querying an Inverted (FTS) index uses minimal IOPs
+        let test_dir = TempStrDir::default();
+
+        // Create dataset with text column for Inverted index
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("text", DataType::Utf8, false),
+        ]));
+
+        let num_rows = 100;
+        let ids = Int32Array::from_iter_values(0..num_rows);
+        let texts = StringArray::from_iter_values((0..num_rows).map(|i| {
+            if i % 3 == 0 {
+                format!("hello world document {}", i)
+            } else if i % 3 == 1 {
+                format!("goodbye universe text {}", i)
+            } else {
+                format!("random content item {}", i)
+            }
+        }));
+
+        let batch =
+            RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(texts)]).unwrap();
+
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(reader, test_dir.as_str(), None)
+            .await
+            .unwrap();
+
+        // Create Inverted index
+        let params = InvertedIndexParams::default();
+        dataset
+            .create_index(
+                &["text"],
+                IndexType::Inverted,
+                Some("inverted_idx".to_string()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Re-open dataset fresh
+        let dataset = DatasetBuilder::from_uri(test_dir.as_str())
+            .load()
+            .await
+            .unwrap();
+
+        // Reset IO stats before query
+        let _ = dataset.object_store().io_stats_incremental();
+
+        // Query using the Inverted index (full-text search)
+        let results = dataset
+            .scan()
+            .full_text_search(FullTextSearchQuery::new("hello".to_string()))
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert!(results.num_rows() > 0);
+
+        // Verify IOPs
+        let stats = dataset.object_store().io_stats_incremental();
+        assert_io_lt!(
+            stats,
+            read_iops,
+            15,
+            "Inverted index query should use minimal IOPs"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_ivf_pq_index_iops() {
+        // Test that querying an IVF_PQ vector index uses minimal IOPs
+        let test_dir = TempStrDir::default();
+
+        // Create dataset with vector column
+        let dimension = 32;
+        let num_rows = 1000;
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(
+                    Arc::new(Field::new("item", DataType::Float32, true)),
+                    dimension,
+                ),
+                false,
+            ),
+        ]));
+
+        let ids = Int32Array::from_iter_values(0..num_rows);
+        let vectors: Vec<Option<Vec<Option<f32>>>> = (0..num_rows)
+            .map(|i| {
+                Some(
+                    (0..dimension)
+                        .map(|j| Some((i * dimension + j) as f32 / 1000.0))
+                        .collect(),
+                )
+            })
+            .collect();
+        let vector_array =
+            FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(vectors, dimension);
+
+        let batch =
+            RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(vector_array)])
+                .unwrap();
+
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(reader, test_dir.as_str(), None)
+            .await
+            .unwrap();
+
+        // Create IVF_PQ index
+        let params = VectorIndexParams::ivf_pq(4, 8, 4, MetricType::L2, 50);
+        dataset
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some("ivf_pq_idx".to_string()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Re-open dataset fresh
+        let dataset = DatasetBuilder::from_uri(test_dir.as_str())
+            .load()
+            .await
+            .unwrap();
+
+        // Do a full scan to warm up data file metadata
+        let _ = dataset.scan().try_into_batch().await.unwrap();
+
+        // Reset IO stats before query
+        let _ = dataset.object_store().io_stats_incremental();
+
+        // Query using the IVF_PQ index (KNN search)
+        let query_vector: Vec<f32> = (0..dimension).map(|i| i as f32 / 1000.0).collect();
+        let results = dataset
+            .scan()
+            .nearest("vector", &Float32Array::from(query_vector), 10)
+            .unwrap()
+            .nprobes(2)
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert!(results.num_rows() > 0);
+
+        // Verify IOPs
+        let stats = dataset.object_store().io_stats_incremental();
+        assert_io_lt!(
+            stats,
+            read_iops,
+            17,
+            "IVF_PQ index query should use minimal IOPs"
+        );
+    }
+
     #[tokio::test]
     async fn test_describe_indices_returns_correct_vector_index_type() {
         const DIM: i32 = 8;
diff --git a/rust/lance/src/index/api.rs b/rust/lance/src/index/api.rs
new file mode 100644
index 00000000000..2b61cdfe12b
--- /dev/null
+++ b/rust/lance/src/index/api.rs
@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use datafusion::execution::SendableRecordBatchStream;
+use lance_index::{IndexParams, IndexType, PrewarmOptions, optimize::OptimizeOptions};
+use lance_table::format::IndexMetadata;
+
+use crate::{Error, Result};
+
+/// Extends [`crate::Dataset`] with secondary index APIs.
+#[async_trait]
+pub trait DatasetIndexExt {
+    type IndexBuilder<'a>
+    where
+        Self: 'a;
+
+    /// Create a builder for creating an index on columns.
+    ///
+    /// This returns a builder that can be configured with additional options
+    /// like `name()`, `replace()`, and `train()` before awaiting to execute.
+    fn create_index_builder<'a>(
+        &'a mut self,
+        columns: &'a [&'a str],
+        index_type: IndexType,
+        params: &'a dyn IndexParams,
+    ) -> Self::IndexBuilder<'a>;
+
+    /// Create indices on columns.
+    ///
+    /// Upon finish, a new dataset version is generated.
+    async fn create_index(
+        &mut self,
+        columns: &[&str],
+        index_type: IndexType,
+        name: Option<String>,
+        params: &dyn IndexParams,
+        replace: bool,
+    ) -> Result<IndexMetadata>;
+
+    /// Drop indices by name.
+    ///
+    /// Upon finish, a new dataset version is generated.
+    async fn drop_index(&mut self, name: &str) -> Result<()>;
+
+    /// Prewarm an index by name.
+    ///
+    /// This will load the index into memory and cache it.
+    async fn prewarm_index(&self, name: &str) -> Result<()>;
+
+    /// Prewarm an index by name with additional options.
+    async fn prewarm_index_with_options(
+        &self,
+        _name: &str,
+        _options: &PrewarmOptions,
+    ) -> Result<()> {
+        Err(Error::not_supported(
+            "prewarm options are not supported by this dataset implementation".to_owned(),
+        ))
+    }
+
+    /// Read all indices of this Dataset version.
+    ///
+    /// The indices are lazy loaded and cached in memory within the `Dataset` instance.
+    /// The cache is invalidated when the dataset version (Manifest) is changed.
+    async fn load_indices(&self) -> Result<Arc<Vec<IndexMetadata>>>;
+
+    /// Loads all the indices of a given UUID.
+    ///
+    /// Note that it is possible to have multiple indices with the same UUID,
+    /// as they are the deltas of the same index.
+    async fn load_index(&self, uuid: &str) -> Result<Option<IndexMetadata>> {
+        self.load_indices().await.map(|indices| {
+            indices
+                .iter()
+                .find(|idx| idx.uuid.to_string() == uuid)
+                .cloned()
+        })
+    }
+
+    /// Loads a specific index with the given index name.
+    ///
+    /// Returns `Ok(vec![])` if the index does not exist.
+    async fn load_indices_by_name(&self, name: &str) -> Result<Vec<IndexMetadata>> {
+        self.load_indices().await.map(|indices| {
+            indices
+                .iter()
+                .filter(|idx| idx.name == name)
+                .cloned()
+                .collect()
+        })
+    }
+
+    /// Loads a specific index with the given index name.
+    /// This function only works for indices that are unique.
+    /// If there are multiple indices sharing the same name, please use [`Self::load_indices_by_name`].
+    async fn load_index_by_name(&self, name: &str) -> Result<Option<IndexMetadata>> {
+        let indices = self.load_indices_by_name(name).await?;
+        if indices.is_empty() {
+            Ok(None)
+        } else if indices.len() == 1 {
+            Ok(Some(indices[0].clone()))
+        } else {
+            Err(Error::index(format!(
+                "Found multiple indices of the same name: {:?}, please use load_indices_by_name",
+                indices.iter().map(|idx| &idx.name).collect::<Vec<_>>()
+            )))
+        }
+    }
+
+    /// Describes indexes in a dataset.
+    ///
+    /// This method should only access the index metadata and should not load the index into memory.
+    async fn describe_indices<'a, 'b>(
+        &'a self,
+        criteria: Option<lance_index::IndexCriteria<'b>>,
+    ) -> Result<Vec<Arc<dyn lance_index::IndexDescription>>>;
+
+    /// Loads a specific scalar index using the provided criteria.
+    async fn load_scalar_index<'a, 'b>(
+        &'a self,
+        criteria: lance_index::IndexCriteria<'b>,
+    ) -> Result<Option<IndexMetadata>>;
+
+    /// Optimize indices.
+    async fn optimize_indices(&mut self, options: &OptimizeOptions) -> Result<()>;
+
+    /// Find an index with the given name and return its serialized statistics.
+    async fn index_statistics(&self, index_name: &str) -> Result<String>;
+
+    /// Merge one caller-defined group of existing uncommitted index segments into a
+    /// single segment.
+    async fn merge_existing_index_segments(
+        &self,
+        segments: Vec<IndexMetadata>,
+    ) -> Result<IndexMetadata>;
+
+    /// Commit one or more existing physical index segments as a logical index.
+    async fn commit_existing_index_segments(
+        &mut self,
+        index_name: &str,
+        column: &str,
+        segments: Vec<IndexMetadata>,
+    ) -> Result<()>;
+
+    async fn read_index_partition(
+        &self,
+        index_name: &str,
+        partition_id: usize,
+        with_vector: bool,
+    ) -> Result<SendableRecordBatchStream>;
+}
diff --git a/rust/lance/src/index/append.rs b/rust/lance/src/index/append.rs
index e0aa6438f4a..b93e325d472 100644
--- a/rust/lance/src/index/append.rs
+++ b/rust/lance/src/index/append.rs
@@ -9,17 +9,19 @@ use lance_core::{
     utils::mask::{RowAddrTreeMap, RowSetOps},
 };
 use lance_index::{
+    INDEX_FILE_NAME,
     metrics::NoOpMetricsCollector,
     optimize::OptimizeOptions,
     progress::NoopIndexBuildProgress,
     scalar::{CreatedIndex, OldIndexDataFilter, lance_format::LanceIndexStore},
 };
-use lance_table::format::{Fragment, IndexMetadata};
+use lance_table::format::{Fragment, IndexMetadata, list_index_files_with_sizes};
 use roaring::RoaringBitmap;
 use uuid::Uuid;
 
 use super::DatasetIndexInternalExt;
-use super::vector::ivf::optimize_vector_indices;
+use super::vector::LogicalVectorIndex;
+use super::vector::ivf::{optimize_vector_indices, select_segment_for_single_rebalance};
 use crate::dataset::Dataset;
 use crate::dataset::index::LanceIndexStoreExt;
 use crate::dataset::rowids::load_row_id_sequences;
@@ -33,6 +35,8 @@ pub struct IndexMergeResults<'a> {
     pub new_fragment_bitmap: RoaringBitmap,
     pub new_index_version: i32,
     pub new_index_details: prost_types::Any,
+    /// List of files and their sizes for the merged index
+    pub files: Option<Vec<lance_table::format::IndexFile>>,
 }
 
 async fn build_stable_row_id_filter(
@@ -70,6 +74,18 @@ async fn build_stable_row_id_filter(
     Ok(<RowAddrTreeMap as RowSetOps>::union_all(&row_id_map_refs))
 }
 
+async fn metadata_is_vector_index(dataset: &Dataset, index: &IndexMetadata) -> Result<bool> {
+    if let Some(files) = &index.files {
+        return Ok(files.iter().any(|file| file.path == INDEX_FILE_NAME));
+    }
+
+    let index_dir = dataset.indice_files_dir(index)?;
+    let index_file = index_dir
+        .child(index.uuid.to_string())
+        .child(INDEX_FILE_NAME);
+    dataset.object_store.exists(&index_file).await
+}
+
 /// Merge in-inflight unindexed data, with a specific number of previous indices
 /// into a new index, to improve the query performance.
 ///
@@ -92,7 +108,13 @@ pub async fn merge_indices<'a>(
     };
 
     let unindexed = dataset.unindexed_fragments(&old_indices[0].name).await?;
-    merge_indices_with_unindexed_frags(dataset, old_indices, &unindexed, options).await
+    Box::pin(merge_indices_with_unindexed_frags(
+        dataset,
+        old_indices,
+        &unindexed,
+        options,
+    ))
+    .await
 }
 
 /// Merge a list of provided unindexed data, with a specific number of previous indices
@@ -118,120 +140,123 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
         )))?;
 
     let field_path = dataset.schema().field_path(old_indices[0].fields[0])?;
-    let mut indices = Vec::with_capacity(old_indices.len());
-    for idx in old_indices {
-        match dataset
-            .open_generic_index(&field_path, &idx.uuid.to_string(), &NoOpMetricsCollector)
-            .await
-        {
-            Ok(index) => indices.push(index),
-            Err(e) => {
-                log::warn!(
-                    "Cannot open index on column '{}': {}. \
-                     Skipping index merge for this column.",
-                    field_path,
-                    e
-                );
-                return Ok(None);
-            }
+    let first_is_vector_index = metadata_is_vector_index(dataset.as_ref(), old_indices[0]).await?;
+    for idx in old_indices.iter().skip(1) {
+        let is_vector_index = metadata_is_vector_index(dataset.as_ref(), idx).await?;
+        if is_vector_index != first_is_vector_index {
+            return Err(Error::index(format!(
+                "Append index: invalid mixed index deltas: {:?}",
+                old_indices
+            )));
         }
     }
 
-    if indices
-        .windows(2)
-        .any(|w| w[0].index_type() != w[1].index_type())
-    {
-        return Err(Error::index(format!(
-            "Append index: invalid index deltas: {:?}",
-            old_indices
-        )));
-    }
-
-    let mut frag_bitmap = RoaringBitmap::new();
+    let mut base_unindexed_bitmap = RoaringBitmap::new();
     unindexed.iter().for_each(|frag| {
-        frag_bitmap.insert(frag.id as u32);
+        base_unindexed_bitmap.insert(frag.id as u32);
     });
 
-    let index_type = indices[0].index_type();
-    let (new_uuid, indices_merged, created_index) = match index_type {
-        it if it.is_scalar() => {
-            // Use effective bitmap (intersected with existing dataset fragments)
-            // to avoid carrying stale data from pruned indices.
-            let effective_old_frags: RoaringBitmap = old_indices
-                .iter()
-                .filter_map(|idx| idx.effective_fragment_bitmap(&dataset.fragment_bitmap))
-                .fold(RoaringBitmap::new(), |mut acc, b| {
-                    acc |= &b;
-                    acc
-                });
-            frag_bitmap |= &effective_old_frags;
-
-            let index = dataset
-                .open_scalar_index(
-                    &field_path,
-                    &old_indices[0].uuid.to_string(),
-                    &NoOpMetricsCollector,
-                )
-                .await?;
-
-            let update_criteria = index.update_criteria();
-
-            let fragments = if update_criteria.requires_old_data {
-                None
-            } else {
-                Some(unindexed.to_vec())
+    let (new_uuid, removed_indices, new_fragment_bitmap, created_index) = if first_is_vector_index {
+        let full_logical_index = dataset
+            .open_logical_vector_index(&field_path, &old_indices[0].name)
+            .await?;
+        let mut opened_indices_by_uuid = full_logical_index
+            .iter()
+            .map(|(metadata, index)| (metadata.uuid, (metadata.clone(), index.clone())))
+            .collect::<std::collections::HashMap<_, _>>();
+        let mut selected_metadatas = Vec::with_capacity(old_indices.len());
+        let mut selected_indices = Vec::with_capacity(old_indices.len());
+        for metadata in old_indices {
+            let (selected_metadata, selected_index) = opened_indices_by_uuid.remove(&metadata.uuid).ok_or_else(|| {
+                Error::index(format!(
+                    "Append index: logical vector index '{}' does not contain requested segment {}",
+                    old_indices[0].name, metadata.uuid
+                ))
+            })?;
+            selected_metadatas.push(selected_metadata);
+            selected_indices.push(selected_index);
+        }
+        let logical_index = LogicalVectorIndex::try_new(
+            old_indices[0].name.clone(),
+            field_path.clone(),
+            selected_metadatas
+                .into_iter()
+                .zip(selected_indices)
+                .collect(),
+        )?;
+        let ivf_view = logical_index.as_ivf()?;
+
+        let use_single_segment_rebalance = logical_index.num_segments() > 1
+            && options.num_indices_to_merge.is_none()
+            && !options.retrain
+            && unindexed.is_empty();
+
+        if use_single_segment_rebalance {
+            let Some(selected_segment_id) = select_segment_for_single_rebalance(&ivf_view)? else {
+                return Ok(None);
             };
-            let new_data_stream = load_training_data(
-                dataset.as_ref(),
+            let removed_segment = old_indices
+                .iter()
+                .copied()
+                .find(|metadata| metadata.uuid == selected_segment_id)
+                .ok_or_else(|| {
+                    Error::index(format!(
+                        "Append index: logical vector index '{}' does not contain selected segment {}",
+                        old_indices[0].name, selected_segment_id
+                    ))
+                })?;
+            let (selected_metadata, selected_index) = logical_index
+                .iter()
+                .find(|(metadata, _)| metadata.uuid == selected_segment_id)
+                .map(|(metadata, index)| (metadata.clone(), index.clone()))
+                .ok_or_else(|| {
+                    Error::index(format!(
+                        "Append index: failed to materialize selected segment {} from logical vector index '{}'",
+                        selected_segment_id, old_indices[0].name
+                    ))
+                })?;
+            let selected_logical_index = LogicalVectorIndex::try_new(
+                old_indices[0].name.clone(),
+                field_path.clone(),
+                vec![(selected_metadata, selected_index)],
+            )?;
+            let selected_ivf_view = selected_logical_index.as_ivf()?;
+            let (new_uuid, indices_merged) = Box::pin(optimize_vector_indices(
+                dataset.as_ref().clone(),
+                Option::<
+                    lance_io::stream::RecordBatchStreamAdapter<
+                        futures::stream::Empty<lance_core::Result<arrow_array::RecordBatch>>,
+                    >,
+                >::None,
                 &field_path,
-                &update_criteria.data_criteria,
-                fragments,
-                true,
-                None,
-            )
+                &selected_ivf_view,
+                options,
+            ))
             .await?;
+            if indices_merged == 0 {
+                return Ok(None);
+            }
 
-            let new_uuid = Uuid::new_v4();
+            let index_dir = dataset.indices_dir().child(new_uuid.to_string());
+            let files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?;
+            let new_fragment_bitmap = removed_segment
+                .effective_fragment_bitmap(&dataset.fragment_bitmap)
+                .or_else(|| removed_segment.fragment_bitmap.clone())
+                .unwrap_or_default();
 
-            let created_index = if effective_old_frags.is_empty() {
-                // Old data is fully stale (bitmap pruned to empty). Rebuild
-                // from scratch instead of merging stale entries.
-                let params = index.derive_index_params()?;
-                super::scalar::build_scalar_index(
-                    dataset.as_ref(),
-                    column.name.as_str(),
-                    &new_uuid.to_string(),
-                    &params,
-                    true,
-                    None,
-                    Some(new_data_stream),
-                    Arc::new(NoopIndexBuildProgress),
-                )
-                .await?
-            } else {
-                let new_store =
-                    LanceIndexStore::from_dataset_for_new(&dataset, &new_uuid.to_string())?;
-                let old_data_filter = if dataset.manifest.uses_stable_row_ids() {
-                    // Stable row IDs are opaque IDs, so fragment-bit filtering on
-                    // (row_id >> 32) is invalid. Build an exact allow-list from retained
-                    // fragments' row-id sequences and use precise filtering.
-                    let valid_old_row_ids =
-                        build_stable_row_id_filter(dataset.as_ref(), &effective_old_frags).await?;
-                    Some(OldIndexDataFilter::RowIds(valid_old_row_ids))
-                } else {
-                    // Address-style row IDs encode fragment_id in high 32 bits.
-                    // Fragment bitmap filtering is valid and cheaper in this mode.
-                    Some(OldIndexDataFilter::Fragments(effective_old_frags))
-                };
-                index
-                    .update(new_data_stream, &new_store, old_data_filter)
-                    .await?
-            };
+            Ok((
+                new_uuid,
+                vec![removed_segment],
+                new_fragment_bitmap,
+                CreatedIndex {
+                    index_details: vector_index_details(),
+                    index_version: lance_index::IndexType::Vector.version() as u32,
+                    files: Some(files),
+                },
+            ))
+        } else {
+            let mut frag_bitmap = base_unindexed_bitmap.clone();
 
-            // TODO: don't hard-code index version
-            Ok((new_uuid, 1, created_index))
-        }
-        it if it.is_vector() => {
             let new_data_stream = if unindexed.is_empty() {
                 None
             } else {
@@ -252,49 +277,177 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                 dataset.as_ref().clone(),
                 new_data_stream,
                 &field_path,
-                &indices,
+                &ivf_view,
                 options,
             )
             .boxed()
             .await?;
 
-            old_indices[old_indices.len() - indices_merged..]
-                .iter()
-                .for_each(|idx| {
-                    frag_bitmap.extend(idx.fragment_bitmap.as_ref().unwrap().iter());
-                });
+            let removed_indices = old_indices[old_indices.len() - indices_merged..].to_vec();
+            removed_indices.iter().for_each(|idx| {
+                frag_bitmap.extend(idx.fragment_bitmap.as_ref().unwrap().iter());
+            });
+            for removed in removed_indices.iter() {
+                if let Some(effective) = removed.effective_fragment_bitmap(&dataset.fragment_bitmap)
+                {
+                    frag_bitmap |= &effective;
+                }
+            }
+
+            let index_dir = dataset.indices_dir().child(new_uuid.to_string());
+            let files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?;
 
             Ok((
                 new_uuid,
-                indices_merged,
+                removed_indices,
+                frag_bitmap,
                 CreatedIndex {
                     index_details: vector_index_details(),
-                    // retain_supported_indices guarantees all old_indices have
-                    // index_version <= our max supported version, so we can safely
-                    // write the current library's version for this index type.
-                    index_version: it.version() as u32,
+                    index_version: lance_index::IndexType::Vector.version() as u32,
+                    files: Some(files),
                 },
             ))
         }
-        _ => Err(Error::index(format!(
-            "Append index: invalid index type: {:?}",
-            indices[0].index_type()
-        ))),
-    }?;
+    } else {
+        let mut frag_bitmap = base_unindexed_bitmap;
+        let mut indices = Vec::with_capacity(old_indices.len());
+        for idx in old_indices {
+            match dataset
+                .open_generic_index(&field_path, &idx.uuid.to_string(), &NoOpMetricsCollector)
+                .await
+            {
+                Ok(index) => indices.push(index),
+                Err(e) => {
+                    log::warn!(
+                        "Cannot open index on column '{}': {}. \
+                         Skipping index merge for this column.",
+                        field_path,
+                        e
+                    );
+                    return Ok(None);
+                }
+            }
+        }
 
-    let removed_indices = old_indices[old_indices.len() - indices_merged..].to_vec();
-    for removed in removed_indices.iter() {
-        if let Some(effective) = removed.effective_fragment_bitmap(&dataset.fragment_bitmap) {
-            frag_bitmap |= &effective;
+        if indices
+            .windows(2)
+            .any(|w| w[0].index_type() != w[1].index_type())
+        {
+            return Err(Error::index(format!(
+                "Append index: invalid index deltas: {:?}",
+                old_indices
+            )));
         }
-    }
+
+        let index_type = indices[0].index_type();
+        match index_type {
+            it if it.is_scalar() => {
+                // Use effective bitmap (intersected with existing dataset fragments)
+                // to avoid carrying stale data from pruned indices.
+                let effective_old_frags: RoaringBitmap = old_indices
+                    .iter()
+                    .filter_map(|idx| idx.effective_fragment_bitmap(&dataset.fragment_bitmap))
+                    .fold(RoaringBitmap::new(), |mut acc, b| {
+                        acc |= &b;
+                        acc
+                    });
+                let deleted_old_frags: RoaringBitmap = old_indices
+                    .iter()
+                    .filter_map(|idx| idx.deleted_fragment_bitmap(&dataset.fragment_bitmap))
+                    .fold(RoaringBitmap::new(), |mut acc, b| {
+                        acc |= &b;
+                        acc
+                    });
+                frag_bitmap |= &effective_old_frags;
+
+                let index = dataset
+                    .open_scalar_index(
+                        &field_path,
+                        &old_indices[0].uuid.to_string(),
+                        &NoOpMetricsCollector,
+                    )
+                    .await?;
+
+                let update_criteria = index.update_criteria();
+
+                let fragments = if update_criteria.requires_old_data {
+                    None
+                } else {
+                    Some(unindexed.to_vec())
+                };
+                let new_data_stream = load_training_data(
+                    dataset.as_ref(),
+                    &field_path,
+                    &update_criteria.data_criteria,
+                    fragments,
+                    true,
+                    None,
+                )
+                .await?;
+
+                let new_uuid = Uuid::new_v4();
+
+                let created_index = if effective_old_frags.is_empty() {
+                    // Old data is fully stale (bitmap pruned to empty). Rebuild
+                    // from scratch instead of merging stale entries.
+                    let params = index.derive_index_params()?;
+                    super::scalar::build_scalar_index(
+                        dataset.as_ref(),
+                        column.name.as_str(),
+                        &new_uuid.to_string(),
+                        &params,
+                        true,
+                        None,
+                        Some(new_data_stream),
+                        Arc::new(NoopIndexBuildProgress),
+                    )
+                    .await?
+                } else {
+                    let new_store =
+                        LanceIndexStore::from_dataset_for_new(&dataset, &new_uuid.to_string())?;
+                    let old_data_filter = if dataset.manifest.uses_stable_row_ids() {
+                        // Stable row IDs are opaque IDs, so fragment-bit filtering on
+                        // (row_id >> 32) is invalid. Build an exact allow-list from retained
+                        // fragments' row-id sequences and use precise filtering.
+                        let valid_old_row_ids =
+                            build_stable_row_id_filter(dataset.as_ref(), &effective_old_frags)
+                                .await?;
+                        Some(OldIndexDataFilter::RowIds(valid_old_row_ids))
+                    } else {
+                        // Address-style row IDs encode fragment_id in high 32 bits.
+                        // Fragment bitmap filtering is valid and cheaper in this mode.
+                        Some(OldIndexDataFilter::Fragments {
+                            to_keep: effective_old_frags,
+                            to_remove: deleted_old_frags,
+                        })
+                    };
+                    index
+                        .update(new_data_stream, &new_store, old_data_filter)
+                        .await?
+                };
+
+                // TODO: don't hard-code index version
+                Ok((
+                    new_uuid,
+                    vec![old_indices[old_indices.len() - 1]],
+                    frag_bitmap,
+                    created_index,
+                ))
+            }
+            _ => Err(Error::index(format!(
+                "Append index: invalid index type: {:?}",
+                indices[0].index_type()
+            ))),
+        }
+    }?;
 
     Ok(Some(IndexMergeResults {
         new_uuid,
         removed_indices,
-        new_fragment_bitmap: frag_bitmap,
+        new_fragment_bitmap,
         new_index_version: created_index.index_version as i32,
         new_index_details: created_index.index_details,
+        files: created_index.files,
     }))
 }
 
@@ -302,6 +455,8 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
 mod tests {
     use super::*;
 
+    use crate::index::DatasetIndexExt;
+    use crate::index::DatasetIndexInternalExt;
     use arrow::datatypes::{Float32Type, UInt32Type};
     use arrow_array::cast::AsArray;
     use arrow_array::{
@@ -316,7 +471,7 @@ mod tests {
     use lance_index::vector::hnsw::builder::HnswBuildParams;
     use lance_index::vector::sq::builder::SQBuildParams;
     use lance_index::{
-        DatasetIndexExt, IndexType,
+        IndexType,
         scalar::ScalarIndexParams,
         vector::{ivf::IvfBuildParams, pq::PQBuildParams},
     };
@@ -535,6 +690,19 @@ mod tests {
         assert_eq!(stats["num_indices"], 2);
         assert_eq!(stats["num_indexed_fragments"], 2);
         assert_eq!(stats["num_unindexed_fragments"], 0);
+        let logical_index = dataset
+            .open_logical_vector_index("vector", "vector_idx")
+            .await
+            .unwrap();
+        assert_eq!(logical_index.num_segments(), 2);
+        assert_eq!(
+            logical_index
+                .num_rows_per_segment()
+                .into_iter()
+                .map(|(_, num_rows)| num_rows)
+                .sum::<u64>(),
+            2000
+        );
 
         let results = dataset
             .scan()
@@ -696,6 +864,83 @@ mod tests {
         assert_eq!(results[0].num_rows(), 10);
     }
 
+    #[tokio::test]
+    async fn test_merge_indices_with_unindexed_frags_vector_subset() {
+        const DIM: usize = 64;
+        const TOTAL: usize = 1000;
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        let vectors = generate_random_array(TOTAL * DIM);
+        let schema = Arc::new(Schema::new(vec![
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(
+                    Arc::new(Field::new("item", DataType::Float32, true)),
+                    DIM as i32,
+                ),
+                true,
+            ),
+            Field::new("id", DataType::UInt32, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(FixedSizeListArray::try_new_from_values(vectors, DIM as i32).unwrap()),
+                Arc::new(UInt32Array::from_iter_values(0..TOTAL as u32)),
+            ],
+        )
+        .unwrap();
+        let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(batches, test_uri, None).await.unwrap();
+        let index_params = VectorIndexParams::ivf_pq(2, 8, 4, MetricType::L2, 2);
+        dataset
+            .create_index(&["vector"], IndexType::Vector, None, &index_params, true)
+            .await
+            .unwrap();
+
+        let next_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(
+                    FixedSizeListArray::try_new_from_values(
+                        generate_random_array(TOTAL * DIM),
+                        DIM as i32,
+                    )
+                    .unwrap(),
+                ),
+                Arc::new(UInt32Array::from_iter_values(
+                    TOTAL as u32..(TOTAL * 2) as u32,
+                )),
+            ],
+        )
+        .unwrap();
+        let batches = RecordBatchIterator::new(vec![next_batch].into_iter().map(Ok), schema);
+        dataset.append(batches, None).await.unwrap();
+        dataset
+            .optimize_indices(&OptimizeOptions::append())
+            .await
+            .unwrap();
+
+        let indices = dataset.load_indices_by_name("vector_idx").await.unwrap();
+        assert_eq!(indices.len(), 2);
+        let subset = vec![&indices[1]];
+        let merge_result = merge_indices_with_unindexed_frags(
+            Arc::new(dataset),
+            &subset,
+            &[],
+            &OptimizeOptions::merge(1),
+        )
+        .await
+        .unwrap();
+
+        assert!(
+            merge_result.is_some(),
+            "subset merges should respect the caller-provided indices"
+        );
+    }
+
     #[tokio::test]
     async fn test_optimize_btree_keeps_rows_with_stable_row_ids_after_compaction() {
         async fn query_id_count(dataset: &Dataset, id: &str) -> usize {
diff --git a/rust/lance/src/index/create.rs b/rust/lance/src/index/create.rs
index 50027833bfc..8aed939787a 100644
--- a/rust/lance/src/index/create.rs
+++ b/rust/lance/src/index/create.rs
@@ -5,7 +5,7 @@ use crate::{
     Error, Result,
     dataset::{
         Dataset,
-        transaction::{Operation, Transaction},
+        transaction::{Operation, TransactionBuilder},
     },
     index::{
         DatasetIndexExt, DatasetIndexInternalExt,
@@ -25,13 +25,12 @@ use lance_index::{
     metrics::NoOpMetricsCollector,
     scalar::{LANCE_SCALAR_INDEX, ScalarIndexParams, inverted::tokenizer::InvertedIndexParams},
 };
-use lance_table::format::IndexMetadata;
-use std::{future::IntoFuture, sync::Arc};
+use lance_table::format::{IndexMetadata, list_index_files_with_sizes};
+use std::{collections::HashMap, future::IntoFuture, sync::Arc};
 use tracing::instrument;
 use uuid::Uuid;
 
 use arrow_array::RecordBatchReader;
-
 /// Generate default index name from field path.
 ///
 /// Joins field names with `.` to create the base index name.
@@ -56,6 +55,8 @@ pub struct CreateIndexBuilder<'a> {
     index_uuid: Option<String>,
     preprocessed_data: Option<Box<dyn RecordBatchReader + Send + 'static>>,
     progress: Arc<dyn IndexBuildProgress>,
+    /// Transaction properties to store with this commit.
+    transaction_properties: Option<Arc<HashMap<String, String>>>,
 }
 
 impl<'a> CreateIndexBuilder<'a> {
@@ -77,6 +78,7 @@ impl<'a> CreateIndexBuilder<'a> {
             index_uuid: None,
             preprocessed_data: None,
             progress: Arc::new(NoopIndexBuildProgress),
+            transaction_properties: None,
         }
     }
 
@@ -118,6 +120,16 @@ impl<'a> CreateIndexBuilder<'a> {
         self
     }
 
+    /// Set transaction properties to store with this commit.
+    ///
+    /// These key-value pairs are stored in the transaction file
+    /// and can be read later to identify the source of the commit
+    /// (e.g., job_id for tracking completed index jobs).
+    pub fn transaction_properties(mut self, properties: HashMap<String, String>) -> Self {
+        self.transaction_properties = Some(Arc::new(properties));
+        self
+    }
+
     #[instrument(skip_all)]
     pub async fn execute_uncommitted(&mut self) -> Result<IndexMetadata> {
         if self.columns.len() != 1 {
@@ -171,19 +183,24 @@ impl<'a> CreateIndexBuilder<'a> {
             }
             candidate
         };
-        if let Some(idx) = indices.iter().find(|i| i.name == index_name) {
-            if idx.fields == [field.id] && !self.replace {
-                return Err(Error::index(format!(
-                    "Index name '{index_name} already exists, \
-                    please specify a different name or use replace=True"
-                )));
-            };
-            if idx.fields != [field.id] {
-                return Err(Error::index(format!(
-                    "Index name '{index_name} already exists with different fields, \
-                    please specify a different name"
-                )));
-            }
+        let existing_named_indices = indices
+            .iter()
+            .filter(|idx| idx.name == index_name)
+            .collect::<Vec<_>>();
+        if existing_named_indices
+            .iter()
+            .any(|idx| idx.fields != [field.id])
+        {
+            return Err(Error::index(format!(
+                "Index name '{index_name}' already exists with different fields, \
+                please specify a different name"
+            )));
+        }
+        if !existing_named_indices.is_empty() && !self.replace {
+            return Err(Error::index(format!(
+                "Index name '{index_name}' already exists, \
+                please specify a different name or use replace=True"
+            )));
         }
 
         let index_id = match &self.index_uuid {
@@ -191,6 +208,7 @@ impl<'a> CreateIndexBuilder<'a> {
                 .map_err(|e| Error::index(format!("Invalid UUID string provided: {}", e)))?,
             None => Uuid::new_v4(),
         };
+        let mut output_index_uuid = index_id;
         let created_index = match (self.index_type, self.params.index_name()) {
             (
                 IndexType::Bitmap
@@ -278,8 +296,8 @@ impl<'a> CreateIndexBuilder<'a> {
                         )
                     })?;
 
-                let params =
-                    ScalarIndexParams::new("inverted".to_string()).with_params(inverted_params);
+                let params = ScalarIndexParams::new("inverted".to_string())
+                    .with_params(&inverted_params.to_training_json()?);
                 build_scalar_index(
                     self.dataset,
                     column,
@@ -315,20 +333,21 @@ impl<'a> CreateIndexBuilder<'a> {
 
                 if train {
                     // Check if this is distributed indexing (fragment-level)
-                    if self.fragments.is_some() {
+                    if let Some(fragments) = &self.fragments {
                         // For distributed indexing, build only on specified fragments
                         // This creates temporary index metadata without committing
-                        Box::pin(build_distributed_vector_index(
+                        let segment_uuid = Box::pin(build_distributed_vector_index(
                             self.dataset,
                             column,
                             &index_name,
                             &index_id.to_string(),
                             vec_params,
                             fri,
-                            self.fragments.as_ref().unwrap(),
+                            fragments,
                             self.progress.clone(),
                         ))
                         .await?;
+                        output_index_uuid = segment_uuid;
                     } else {
                         // Standard full dataset indexing
                         Box::pin(build_vector_index(
@@ -353,9 +372,17 @@ impl<'a> CreateIndexBuilder<'a> {
                     )
                     .await?;
                 }
+                // Capture file sizes after vector index creation
+                let index_dir = self
+                    .dataset
+                    .indices_dir()
+                    .child(output_index_uuid.to_string());
+                let files =
+                    list_index_files_with_sizes(&self.dataset.object_store, &index_dir).await?;
                 CreatedIndex {
                     index_details: vector_index_details(),
                     index_version,
+                    files: Some(files),
                 }
             }
             // Can't use if let Some(...) here because it's not stable yet.
@@ -387,9 +414,14 @@ impl<'a> CreateIndexBuilder<'a> {
                 } else {
                     todo!("create empty vector index when train=false");
                 }
+                // Capture file sizes after vector index creation
+                let index_dir = self.dataset.indices_dir().child(index_id.to_string());
+                let files =
+                    list_index_files_with_sizes(&self.dataset.object_store, &index_dir).await?;
                 CreatedIndex {
                     index_details: vector_index_details(),
                     index_version: self.index_type.version() as u32,
+                    files: Some(files),
                 }
             }
             (IndexType::FragmentReuse, _) => {
@@ -405,20 +437,14 @@ impl<'a> CreateIndexBuilder<'a> {
         };
 
         Ok(IndexMetadata {
-            uuid: index_id,
+            uuid: output_index_uuid,
             name: index_name,
             fields: vec![field.id],
             dataset_version: self.dataset.manifest.version,
             fragment_bitmap: if train {
                 match &self.fragments {
                     Some(fragment_ids) => Some(fragment_ids.iter().collect()),
-                    None => Some(
-                        self.dataset
-                            .get_fragments()
-                            .iter()
-                            .map(|f| f.id() as u32)
-                            .collect(),
-                    ),
+                    None => Some(self.dataset.fragment_bitmap.as_ref().clone()),
                 }
             } else {
                 // Empty bitmap for untrained indices
@@ -428,6 +454,7 @@ impl<'a> CreateIndexBuilder<'a> {
             index_version: created_index.index_version as i32,
             created_at: Some(chrono::Utc::now()),
             base_id: None,
+            files: created_index.files,
         })
     }
 
@@ -435,14 +462,26 @@ impl<'a> CreateIndexBuilder<'a> {
     async fn execute(mut self) -> Result<IndexMetadata> {
         let new_idx = self.execute_uncommitted().await?;
         let index_uuid = new_idx.uuid;
-        let transaction = Transaction::new(
+        let removed_indices = if self.replace {
+            self.dataset
+                .load_indices()
+                .await?
+                .iter()
+                .filter(|idx| idx.name == new_idx.name)
+                .cloned()
+                .collect()
+        } else {
+            vec![]
+        };
+        let transaction = TransactionBuilder::new(
             new_idx.dataset_version,
             Operation::CreateIndex {
                 new_indices: vec![new_idx],
-                removed_indices: vec![],
+                removed_indices,
             },
-            None,
-        );
+        )
+        .transaction_properties(self.transaction_properties.clone())
+        .build();
 
         self.dataset
             .apply_commit(transaction, &Default::default(), &Default::default())
@@ -477,17 +516,44 @@ impl<'a> IntoFuture for CreateIndexBuilder<'a> {
 mod tests {
     use super::*;
     use crate::dataset::{WriteMode, WriteParams};
+    use crate::index::DatasetIndexExt;
     use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount};
     use arrow::datatypes::{Float32Type, Int32Type};
-    use arrow_array::RecordBatchIterator;
-    use arrow_array::{Int32Array, RecordBatch, StringArray};
+    use arrow_array::cast::AsArray;
+    use arrow_array::{
+        FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator, StringArray,
+    };
     use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
+    use lance_arrow::FixedSizeListArrayExt;
     use lance_core::utils::tempfile::TempStrDir;
     use lance_datagen::{self, gen_batch};
     use lance_index::optimize::OptimizeOptions;
+    use lance_index::progress::IndexBuildProgress;
     use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
-    use lance_linalg::distance::MetricType;
+    use lance_index::vector::hnsw::builder::HnswBuildParams;
+    use lance_index::vector::ivf::IvfBuildParams;
+    use lance_index::vector::kmeans::{KMeansParams, train_kmeans};
+    use lance_linalg::distance::{DistanceType, MetricType};
     use std::sync::Arc;
+    use uuid::Uuid;
+
+    lance_testing::define_stage_event_progress!(RecordingProgress, IndexBuildProgress, Result<()>);
+
+    #[test]
+    fn test_inverted_training_params_include_build_only_fields() {
+        let params = InvertedIndexParams::default()
+            .memory_limit_mb(4096)
+            .num_workers(7);
+        let scalar_params = ScalarIndexParams::new("inverted".to_string())
+            .with_params(&params.to_training_json().unwrap());
+        let json: serde_json::Value =
+            serde_json::from_str(scalar_params.params.as_ref().unwrap()).unwrap();
+        assert_eq!(
+            json.get("memory_limit"),
+            Some(&serde_json::Value::from(4096))
+        );
+        assert_eq!(json.get("num_workers"), Some(&serde_json::Value::from(7)));
+    }
 
     #[test]
     fn test_default_index_name() {
@@ -612,6 +678,92 @@ mod tests {
         assert!(err.to_string().contains("already exists"));
     }
 
+    #[tokio::test]
+    async fn test_concurrent_create_index_same_name_returns_retryable_conflict() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let reader = gen_batch()
+            .col("a", lance_datagen::array::step::<Int32Type>())
+            .into_reader_rows(
+                lance_datagen::RowCount::from(100),
+                lance_datagen::BatchCount::from(1),
+            );
+        let dataset = Dataset::write(reader, &dataset_uri, None).await.unwrap();
+
+        let params = ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::BTree);
+        let read_version = dataset.manifest.version;
+        let mut reader1 = dataset.checkout_version(read_version).await.unwrap();
+        let mut reader2 = dataset.checkout_version(read_version).await.unwrap();
+
+        let first = CreateIndexBuilder::new(&mut reader1, &["a"], IndexType::BTree, &params)
+            .name("a_idx".to_string())
+            .execute()
+            .await;
+        assert!(
+            first.is_ok(),
+            "first create_index should succeed: {first:?}"
+        );
+
+        let second = CreateIndexBuilder::new(&mut reader2, &["a"], IndexType::BTree, &params)
+            .name("a_idx".to_string())
+            .execute()
+            .await;
+        assert!(
+            matches!(second, Err(Error::RetryableCommitConflict { .. })),
+            "second concurrent create_index should be retryable, got {second:?}"
+        );
+
+        let latest_indices = reader1.load_indices_by_name("a_idx").await.unwrap();
+        assert_eq!(latest_indices.len(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_concurrent_replace_index_same_name_returns_retryable_conflict() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let reader = gen_batch()
+            .col("a", lance_datagen::array::step::<Int32Type>())
+            .into_reader_rows(
+                lance_datagen::RowCount::from(100),
+                lance_datagen::BatchCount::from(1),
+            );
+        let mut dataset = Dataset::write(reader, &dataset_uri, None).await.unwrap();
+
+        let params = ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::BTree);
+        let original = CreateIndexBuilder::new(&mut dataset, &["a"], IndexType::BTree, &params)
+            .name("a_idx".to_string())
+            .execute()
+            .await
+            .unwrap();
+
+        let read_version = dataset.manifest.version;
+        let mut reader1 = dataset.checkout_version(read_version).await.unwrap();
+        let mut reader2 = dataset.checkout_version(read_version).await.unwrap();
+
+        let replacement = CreateIndexBuilder::new(&mut reader1, &["a"], IndexType::BTree, &params)
+            .name("a_idx".to_string())
+            .replace(true)
+            .execute()
+            .await
+            .unwrap();
+        assert_ne!(replacement.uuid, original.uuid);
+
+        let second = CreateIndexBuilder::new(&mut reader2, &["a"], IndexType::BTree, &params)
+            .name("a_idx".to_string())
+            .replace(true)
+            .execute()
+            .await;
+        assert!(
+            matches!(second, Err(Error::RetryableCommitConflict { .. })),
+            "second concurrent replace should be retryable, got {second:?}"
+        );
+
+        let latest_indices = reader1.load_indices_by_name("a_idx").await.unwrap();
+        assert_eq!(latest_indices.len(), 1);
+        assert_eq!(latest_indices[0].uuid, replacement.uuid);
+        assert_ne!(latest_indices[0].uuid, original.uuid);
+    }
+
     // Helper function to create test data with text field suitable for inverted index
     fn create_text_batch(start: i32, end: i32) -> RecordBatch {
         let schema = Arc::new(ArrowSchema::new(vec![
@@ -636,6 +788,39 @@ mod tests {
         .unwrap()
     }
 
+    async fn prepare_vector_ivf(dataset: &Dataset, vector_column: &str) -> IvfBuildParams {
+        let batch = dataset
+            .scan()
+            .project(&[vector_column.to_string()])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let vectors = batch
+            .column_by_name(vector_column)
+            .expect("vector column should exist")
+            .as_fixed_size_list();
+        let dim = vectors.value_length() as usize;
+        let values = vectors.values().as_primitive::<Float32Type>();
+
+        let kmeans = train_kmeans::<Float32Type>(
+            values,
+            KMeansParams::new(None, 10, 1, DistanceType::L2),
+            dim,
+            4,
+            3,
+        )
+        .unwrap();
+        let centroids = Arc::new(
+            FixedSizeListArray::try_new_from_values(
+                kmeans.centroids.as_primitive::<Float32Type>().clone(),
+                dim as i32,
+            )
+            .unwrap(),
+        );
+        IvfBuildParams::try_with_centroids(4, centroids).unwrap()
+    }
+
     #[tokio::test]
     async fn test_execute_uncommitted() {
         // Test the complete workflow that covers the user's specified code pattern:
@@ -711,18 +896,20 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_merge_index_metadata() {
-        // Test the complete workflow for merge_index_metadata:
-        // 1. Create multiple fragment indexes using execute_uncommitted
-        // 2. Use merge_index_metadata to merge temporary metadata files
-        // 3. Commit the index using the standard commit process
-        // 4. Verify the final index is properly created and accessible
-
-        // Create temporary directory for dataset
+    async fn test_merge_index_metadata_inverted_reports_progress() {
+        // This exercises the public distributed inverted-index workflow end to end:
+        // 1. build one uncommitted shard per fragment with CreateIndexBuilder.progress(...)
+        // 2. merge those shards with Dataset::merge_index_metadata(...)
+        //
+        // Expected outcomes:
+        // - the build callback should surface public build stages such as load_data,
+        //   tokenize_docs, copy_partitions, and write_metadata
+        // - the merge callback should surface public merge stages such as
+        //   read_partition_metadata, remap_partition_files, and write_merged_metadata
+        // - merge stages should be reported in execution order
         let tmpdir = TempStrDir::default();
         let dataset_uri = format!("file://{}", tmpdir.as_str());
 
-        // Create test data with multiple fragments
         let batch1 = create_text_batch(0, 15);
         let batch2 = create_text_batch(15, 30);
         let batch3 = create_text_batch(30, 45);
@@ -745,58 +932,579 @@ mod tests {
         let params = InvertedIndexParams::default();
         let fragments = dataset.get_fragments();
         let fragment_ids: Vec<u32> = fragments.iter().map(|f| f.id() as u32).collect();
-
-        // Use a shared UUID for distributed indexing
         let shared_uuid = Uuid::new_v4().to_string();
+        let build_progress = Arc::new(RecordingProgress::default());
 
-        // Step 1: Create indexes for each fragment using execute_uncommitted
-        let mut index_metadatas = Vec::new();
         for &fragment_id in &fragment_ids {
             let mut builder =
                 CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::Inverted, &params)
                     .name("distributed_index".to_string())
                     .fragments(vec![fragment_id])
-                    .index_uuid(shared_uuid.clone());
+                    .index_uuid(shared_uuid.clone())
+                    .progress(build_progress.clone());
 
             let index_metadata = builder.execute_uncommitted().await.unwrap();
-
-            // Verify each fragment's index metadata
             assert_eq!(index_metadata.uuid.to_string(), shared_uuid);
             assert_eq!(index_metadata.name, "distributed_index");
 
             let fragment_bitmap = index_metadata.fragment_bitmap.as_ref().unwrap();
             let indexed_fragments: Vec<u32> = fragment_bitmap.iter().collect();
             assert_eq!(indexed_fragments, vec![fragment_id]);
+        }
+
+        let merge_progress = Arc::new(RecordingProgress::default());
+        dataset
+            .merge_index_metadata(
+                &shared_uuid,
+                IndexType::Inverted,
+                None,
+                merge_progress.clone(),
+            )
+            .await
+            .unwrap();
+
+        let build_tags = build_progress
+            .recorded_events()
+            .iter()
+            .map(|(kind, stage, _)| format!("{kind}:{stage}"))
+            .collect::<Vec<_>>();
+        assert!(
+            build_tags.iter().any(|e| e == "start:load_data"),
+            "expected load_data progress during public distributed build"
+        );
+        assert!(
+            build_tags.iter().any(|e| e == "start:tokenize_docs"),
+            "expected tokenize_docs progress during public distributed build"
+        );
+        assert!(
+            build_tags.iter().any(|e| e == "start:copy_partitions"),
+            "expected copy_partitions progress during public distributed build"
+        );
+        assert!(
+            build_tags.iter().any(|e| e == "start:write_metadata"),
+            "expected write_metadata progress during public distributed build"
+        );
+
+        let merge_events = merge_progress.recorded_events();
+        let merge_tags = merge_events
+            .iter()
+            .map(|(kind, stage, _)| format!("{kind}:{stage}"))
+            .collect::<Vec<_>>();
+        let read_start = merge_tags
+            .iter()
+            .position(|e| e == "start:read_partition_metadata")
+            .expect("missing read_partition_metadata start");
+        let read_complete = merge_tags
+            .iter()
+            .position(|e| e == "complete:read_partition_metadata")
+            .expect("missing read_partition_metadata complete");
+        let remap_start = merge_tags
+            .iter()
+            .position(|e| e == "start:remap_partition_files")
+            .expect("missing remap_partition_files start");
+        let remap_complete = merge_tags
+            .iter()
+            .position(|e| e == "complete:remap_partition_files")
+            .expect("missing remap_partition_files complete");
+        let metadata_start = merge_tags
+            .iter()
+            .position(|e| e == "start:write_merged_metadata")
+            .expect("missing write_merged_metadata start");
+        let metadata_complete = merge_tags
+            .iter()
+            .position(|e| e == "complete:write_merged_metadata")
+            .expect("missing write_merged_metadata complete");
+        assert!(read_start < read_complete);
+        assert!(read_complete < remap_start);
+        assert!(remap_start < remap_complete);
+        assert!(remap_complete < metadata_start);
+        assert!(metadata_start < metadata_complete);
+        assert!(
+            merge_tags
+                .iter()
+                .any(|e| e == "progress:read_partition_metadata"),
+            "expected read_partition_metadata progress during public merge"
+        );
+        assert!(
+            merge_tags
+                .iter()
+                .any(|e| e == "progress:remap_partition_files"),
+            "expected remap_partition_files progress during public merge"
+        );
+        assert!(
+            merge_tags
+                .iter()
+                .any(|e| e == "progress:write_merged_metadata"),
+            "expected write_merged_metadata progress during public merge"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_merge_index_metadata_btree_reports_progress() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+
+        let reader = gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .into_reader_rows(
+                lance_datagen::RowCount::from(256),
+                lance_datagen::BatchCount::from(4),
+            );
+        let mut dataset = Dataset::write(
+            reader,
+            &dataset_uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                mode: WriteMode::Overwrite,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let params = ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::BTree);
+        let fragments = dataset.get_fragments();
+        let fragment_ids: Vec<u32> = fragments.iter().map(|f| f.id() as u32).collect();
+        let shared_uuid = Uuid::new_v4().to_string();
+        let build_progress = Arc::new(RecordingProgress::default());
+
+        for &fragment_id in &fragment_ids {
+            CreateIndexBuilder::new(&mut dataset, &["id"], IndexType::BTree, &params)
+                .name("distributed_btree".to_string())
+                .fragments(vec![fragment_id])
+                .index_uuid(shared_uuid.clone())
+                .progress(build_progress.clone())
+                .execute_uncommitted()
+                .await
+                .unwrap();
+        }
+
+        let merge_progress = Arc::new(RecordingProgress::default());
+        dataset
+            .merge_index_metadata(
+                &shared_uuid,
+                IndexType::BTree,
+                Some(1),
+                merge_progress.clone(),
+            )
+            .await
+            .unwrap();
+
+        let build_tags = build_progress
+            .recorded_events()
+            .iter()
+            .map(|(kind, stage, _)| format!("{kind}:{stage}"))
+            .collect::<Vec<_>>();
+        assert!(
+            build_tags.iter().any(|e| e == "start:load_data"),
+            "expected load_data progress during public distributed build"
+        );
+
+        let merge_tags = merge_progress
+            .recorded_events()
+            .iter()
+            .map(|(kind, stage, _)| format!("{kind}:{stage}"))
+            .collect::<Vec<_>>();
+        let pages_start = merge_tags
+            .iter()
+            .position(|e| e == "start:merge_pages")
+            .expect("missing merge_pages start");
+        let pages_complete = merge_tags
+            .iter()
+            .position(|e| e == "complete:merge_pages")
+            .expect("missing merge_pages complete");
+        let write_start = merge_tags
+            .iter()
+            .position(|e| e == "start:write_lookup_file")
+            .expect("missing write_lookup_file start");
+        let write_complete = merge_tags
+            .iter()
+            .position(|e| e == "complete:write_lookup_file")
+            .expect("missing write_lookup_file complete");
+        assert!(pages_start < pages_complete);
+        assert!(pages_complete < write_start);
+        assert!(write_start < write_complete);
+        assert!(
+            merge_tags.iter().any(|e| e == "progress:merge_pages"),
+            "expected merge_pages progress during public merge"
+        );
+        assert!(
+            merge_tags.iter().any(|e| e == "progress:write_lookup_file"),
+            "expected write_lookup_file progress during public merge"
+        );
+        assert!(
+            !merge_tags.iter().any(|e| e == "start:merge_lookups"),
+            "fragment-based distributed BTREE merge should not use merge_lookups"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_execute_uncommitted_segments_commit_without_staging() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+
+        let reader = gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .col(
+                "vector",
+                lance_datagen::array::rand_vec::<Float32Type>(lance_datagen::Dimension::from(16)),
+            )
+            .into_reader_rows(
+                lance_datagen::RowCount::from(256),
+                lance_datagen::BatchCount::from(4),
+            );
+        let mut dataset = Dataset::write(
+            reader,
+            &dataset_uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                mode: WriteMode::Overwrite,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
 
-            index_metadatas.push(index_metadata);
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+        let params = VectorIndexParams::with_ivf_flat_params(
+            DistanceType::L2,
+            prepare_vector_ivf(&dataset, "vector").await,
+        );
+        let mut input_segments = Vec::new();
+
+        for fragment in &fragments {
+            let segment =
+                CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                    .name("vector_idx".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap();
+            let segment_index = dataset
+                .indices_dir()
+                .child(segment.uuid.to_string())
+                .child(crate::index::INDEX_FILE_NAME);
+            assert!(dataset.object_store().exists(&segment_index).await.unwrap());
+            input_segments.push(segment);
         }
 
-        // Step 2: Merge inverted index metadata
-        // Note: This step would typically be done by calling dataset.merge_index_metadata()
-        // but for this test, we verify that the execute_uncommitted workflow produces correct metadata
+        dataset
+            .commit_existing_index_segments("vector_idx", "vector", input_segments)
+            .await
+            .unwrap();
+
+        let indices = dataset.load_indices_by_name("vector_idx").await.unwrap();
+        assert_eq!(indices.len(), fragments.len());
+
+        let query_batch = dataset
+            .scan()
+            .project(&["vector"] as &[&str])
+            .unwrap()
+            .limit(Some(4), None)
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let q = query_batch["vector"].as_fixed_size_list().value(0);
+        let result = dataset
+            .scan()
+            .project(&["_rowid"] as &[&str])
+            .unwrap()
+            .nearest("vector", q.as_ref(), 5)
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert!(result.num_rows() > 0);
+    }
+
+    #[tokio::test]
+    async fn test_merge_existing_index_segments_vector_commits_single_logical_index() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
 
-        // Step 3: Verify the metadata from execute_uncommitted contains all necessary information
-        assert_eq!(index_metadatas.len(), fragment_ids.len());
+        let reader = gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .col(
+                "vector",
+                lance_datagen::array::rand_vec::<Float32Type>(lance_datagen::Dimension::from(16)),
+            )
+            .into_reader_rows(
+                lance_datagen::RowCount::from(256),
+                lance_datagen::BatchCount::from(4),
+            );
+        let mut dataset = Dataset::write(
+            reader,
+            &dataset_uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                mode: WriteMode::Overwrite,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
 
-        // Verify all metadata have the same UUID (shared UUID for distributed indexing)
-        for metadata in &index_metadatas {
-            assert_eq!(metadata.uuid.to_string(), shared_uuid);
-            assert_eq!(metadata.name, "distributed_index");
-            assert!(metadata.fragment_bitmap.is_some());
-            assert!(metadata.created_at.is_some());
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+        let params = VectorIndexParams::with_ivf_flat_params(
+            DistanceType::L2,
+            prepare_vector_ivf(&dataset, "vector").await,
+        );
+        let mut input_segments = Vec::new();
+
+        for fragment in fragments.iter().take(2) {
+            let segment =
+                CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                    .name("vector_idx".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap();
+            input_segments.push(segment);
         }
 
-        // Verify that each fragment is covered by exactly one metadata
-        let mut all_covered_fragments = Vec::new();
-        for metadata in &index_metadatas {
-            let fragment_bitmap = metadata.fragment_bitmap.as_ref().unwrap();
-            let covered_fragments: Vec<u32> = fragment_bitmap.iter().collect();
-            all_covered_fragments.extend(covered_fragments);
+        let segment = dataset
+            .merge_existing_index_segments(input_segments)
+            .await
+            .unwrap();
+
+        dataset
+            .commit_existing_index_segments("vector_idx", "vector", vec![segment])
+            .await
+            .unwrap();
+
+        let indices = dataset.load_indices_by_name("vector_idx").await.unwrap();
+        assert_eq!(indices.len(), 1);
+        let mut committed_fragment_sets = indices
+            .iter()
+            .map(|metadata| {
+                metadata
+                    .fragment_bitmap
+                    .as_ref()
+                    .unwrap()
+                    .iter()
+                    .collect::<Vec<_>>()
+            })
+            .collect::<Vec<_>>();
+        committed_fragment_sets.sort();
+        assert_eq!(committed_fragment_sets, vec![vec![0, 1]]);
+
+        let query_batch = dataset
+            .scan()
+            .project(&["vector"] as &[&str])
+            .unwrap()
+            .limit(Some(4), None)
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let q = query_batch["vector"].as_fixed_size_list().value(0);
+        let result = dataset
+            .scan()
+            .project(&["_rowid"] as &[&str])
+            .unwrap()
+            .nearest("vector", q.as_ref(), 5)
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert!(result.num_rows() > 0);
+    }
+
+    #[tokio::test]
+    async fn test_merge_existing_index_segments_accepts_python_round_tripped_metadata() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+
+        let reader = gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .col(
+                "vector",
+                lance_datagen::array::rand_vec::<Float32Type>(lance_datagen::Dimension::from(16)),
+            )
+            .into_reader_rows(
+                lance_datagen::RowCount::from(256),
+                lance_datagen::BatchCount::from(4),
+            );
+        let mut dataset = Dataset::write(
+            reader,
+            &dataset_uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                mode: WriteMode::Overwrite,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+        let params = VectorIndexParams::with_ivf_flat_params(
+            DistanceType::L2,
+            prepare_vector_ivf(&dataset, "vector").await,
+        );
+        let mut input_segments = Vec::new();
+
+        for fragment in fragments.iter().take(2) {
+            let mut segment =
+                CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                    .name("vector_idx".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap();
+            segment.index_details = None;
+            input_segments.push(segment);
         }
-        all_covered_fragments.sort();
-        let mut expected_fragments = fragment_ids.clone();
-        expected_fragments.sort();
-        assert_eq!(all_covered_fragments, expected_fragments);
+
+        let merged_segment = dataset
+            .merge_existing_index_segments(input_segments)
+            .await
+            .unwrap();
+        assert!(
+            merged_segment
+                .fragment_bitmap
+                .as_ref()
+                .is_some_and(|bitmap| bitmap.iter().collect::<Vec<_>>() == vec![0, 1])
+        );
+    }
+
+    #[tokio::test]
+    async fn test_commit_existing_index_supports_local_hnsw_segments() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+
+        let reader = gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .col(
+                "vector",
+                lance_datagen::array::rand_vec::<Float32Type>(lance_datagen::Dimension::from(16)),
+            )
+            .into_reader_rows(
+                lance_datagen::RowCount::from(128),
+                lance_datagen::BatchCount::from(2),
+            );
+        let mut dataset = Dataset::write(
+            reader,
+            &dataset_uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                mode: WriteMode::Overwrite,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let uuid = Uuid::new_v4();
+        let params = VectorIndexParams::ivf_hnsw(
+            DistanceType::L2,
+            prepare_vector_ivf(&dataset, "vector").await,
+            HnswBuildParams::default(),
+        );
+
+        let segment =
+            CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                .name("vector_idx".to_string())
+                .index_uuid(uuid.to_string())
+                .execute_uncommitted()
+                .await
+                .unwrap();
+        assert_eq!(segment.uuid, uuid);
+
+        dataset
+            .commit_existing_index_segments("vector_idx", "vector", vec![segment])
+            .await
+            .unwrap();
+
+        let indices = dataset.load_indices_by_name("vector_idx").await.unwrap();
+        assert_eq!(indices.len(), 1);
+        assert_eq!(indices[0].uuid, uuid);
+        assert_eq!(
+            indices[0].fragment_bitmap.as_ref().unwrap(),
+            dataset.fragment_bitmap.as_ref()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_create_index_vector_commits_with_segment_metadata() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+
+        let reader = gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .col(
+                "vector",
+                lance_datagen::array::rand_vec::<Float32Type>(lance_datagen::Dimension::from(16)),
+            )
+            .into_reader_rows(
+                lance_datagen::RowCount::from(128),
+                lance_datagen::BatchCount::from(2),
+            );
+        let mut dataset = Dataset::write(reader, &dataset_uri, None).await.unwrap();
+
+        let params = VectorIndexParams::with_ivf_flat_params(
+            DistanceType::L2,
+            prepare_vector_ivf(&dataset, "vector").await,
+        );
+
+        let committed = dataset
+            .create_index(&["vector"], IndexType::Vector, None, &params, false)
+            .await
+            .unwrap();
+
+        assert!(
+            committed
+                .files
+                .as_ref()
+                .is_some_and(|files| !files.is_empty()),
+            "single-machine vector create_index should preserve committed file info"
+        );
+
+        let loaded = dataset.load_indices_by_name(&committed.name).await.unwrap();
+        assert_eq!(loaded.len(), 1);
+        assert_eq!(loaded[0].uuid, committed.uuid);
+        assert!(
+            loaded[0]
+                .files
+                .as_ref()
+                .is_some_and(|files| !files.is_empty()),
+            "committed metadata loaded from the manifest should include file info"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_create_index_ivf_rq_preserves_index_version_on_segment_commit_path() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+
+        let reader = gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .col(
+                "vector",
+                lance_datagen::array::rand_vec::<Float32Type>(lance_datagen::Dimension::from(16)),
+            )
+            .into_reader_rows(
+                lance_datagen::RowCount::from(128),
+                lance_datagen::BatchCount::from(2),
+            );
+        let mut dataset = Dataset::write(reader, &dataset_uri, None).await.unwrap();
+
+        let params = VectorIndexParams::ivf_rq(4, 1, DistanceType::L2);
+
+        let committed = dataset
+            .create_index(&["vector"], IndexType::IvfRq, None, &params, false)
+            .await
+            .unwrap();
+
+        assert_eq!(committed.index_version, IndexType::IvfRq.version());
+
+        let loaded = dataset.load_indices_by_name(&committed.name).await.unwrap();
+        assert_eq!(loaded.len(), 1);
+        assert_eq!(loaded[0].index_version, IndexType::IvfRq.version());
     }
 
     #[tokio::test]
diff --git a/rust/lance/src/index/frag_reuse.rs b/rust/lance/src/index/frag_reuse.rs
index 76d225d2c27..e5f63514d86 100644
--- a/rust/lance/src/index/frag_reuse.rs
+++ b/rust/lance/src/index/frag_reuse.rs
@@ -3,8 +3,8 @@
 
 use crate::Dataset;
 use crate::dataset::optimize::remapping::transpose_row_ids_from_digest;
+use crate::index::DatasetIndexExt;
 use lance_core::Error;
-use lance_index::DatasetIndexExt;
 use lance_index::frag_reuse::{
     FRAG_REUSE_DETAILS_FILE_NAME, FRAG_REUSE_INDEX_NAME, FragReuseGroup, FragReuseIndex,
     FragReuseIndexDetails, FragReuseVersion,
@@ -172,5 +172,7 @@ pub(crate) async fn build_frag_reuse_index_metadata(
         index_version: index_meta.map_or(0, |index_meta| index_meta.index_version),
         created_at: Some(chrono::Utc::now()),
         base_id: None,
+        // Fragment reuse index is inline (no files)
+        files: None,
     })
 }
diff --git a/rust/lance/src/index/mem_wal.rs b/rust/lance/src/index/mem_wal.rs
index 7407fd467de..a9a966f4ee9 100644
--- a/rust/lance/src/index/mem_wal.rs
+++ b/rust/lance/src/index/mem_wal.rs
@@ -4,12 +4,12 @@
 //! MemWAL Index operations.
 //!
 //! The MemWAL Index stores:
-//! - Configuration (region_specs, maintained_indexes)
-//! - Merge progress (merged_generations per region)
-//! - Region state snapshots (eventually consistent)
+//! - Configuration (shard_specs, maintained_indexes)
+//! - Merge progress (merged_generations per shard)
+//! - Shard state snapshots (eventually consistent)
 //!
 //! Writers no longer update the index on every write. Instead, they update
-//! region manifests directly. This module provides functions to:
+//! shard manifests directly. This module provides functions to:
 //! - Load the MemWAL index
 //! - Update merged generations (called during merge-insert commits)
 
@@ -65,12 +65,12 @@ pub(crate) fn update_mem_wal_index_merged_generations(
         let current_meta = indices.remove(pos);
         let mut details = load_mem_wal_index_details(current_meta)?;
 
-        // Update merged_generations - for each region, keep the higher generation
+        // Update merged_generations - for each shard, keep the higher generation
         for new_mg in new_merged_generations {
             if let Some(existing) = details
                 .merged_generations
                 .iter_mut()
-                .find(|mg| mg.region_id == new_mg.region_id)
+                .find(|mg| mg.shard_id == new_mg.shard_id)
             {
                 if new_mg.generation > existing.generation {
                     existing.generation = new_mg.generation;
@@ -111,6 +111,8 @@ pub(crate) fn new_mem_wal_index_meta(
         index_version: 0,
         created_at: Some(chrono::Utc::now()),
         base_id: None,
+        // Memory WAL index is inline (no files)
+        files: None,
     })
 }
 
@@ -120,9 +122,9 @@ mod tests {
 
     use std::sync::Arc;
 
+    use crate::index::DatasetIndexExt;
     use arrow_array::{Int32Array, RecordBatch};
     use arrow_schema::{DataType, Field, Schema};
-    use lance_index::DatasetIndexExt;
 
     use crate::dataset::transaction::{Operation, Transaction};
     use crate::dataset::{CommitBuilder, InsertBuilder, WriteParams};
@@ -155,13 +157,13 @@ mod tests {
     #[tokio::test]
     async fn test_update_mem_wal_state_conflict_lower_generation_no_retry() {
         let dataset = test_dataset().await;
-        let region = Uuid::new_v4();
+        let shard = Uuid::new_v4();
 
         // First commit UpdateMemWalState with generation 10
         let txn1 = Transaction::new(
             dataset.manifest.version,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 10)],
+                merged_generations: vec![MergedGeneration::new(shard, 10)],
             },
             None,
         );
@@ -175,7 +177,7 @@ mod tests {
         let txn2 = Transaction::new(
             dataset.manifest.version - 1, // Based on old version
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 5)],
+                merged_generations: vec![MergedGeneration::new(shard, 5)],
             },
             None,
         );
@@ -192,13 +194,13 @@ mod tests {
     #[tokio::test]
     async fn test_update_mem_wal_state_conflict_equal_generation_no_retry() {
         let dataset = test_dataset().await;
-        let region = Uuid::new_v4();
+        let shard = Uuid::new_v4();
 
         // First commit UpdateMemWalState with generation 10
         let txn1 = Transaction::new(
             dataset.manifest.version,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 10)],
+                merged_generations: vec![MergedGeneration::new(shard, 10)],
             },
             None,
         );
@@ -211,7 +213,7 @@ mod tests {
         let txn2 = Transaction::new(
             dataset.manifest.version - 1, // Based on old version
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 10)],
+                merged_generations: vec![MergedGeneration::new(shard, 10)],
             },
             None,
         );
@@ -229,13 +231,13 @@ mod tests {
     #[tokio::test]
     async fn test_update_mem_wal_state_conflict_higher_generation_retryable() {
         let dataset = test_dataset().await;
-        let region = Uuid::new_v4();
+        let shard = Uuid::new_v4();
 
         // First commit UpdateMemWalState with generation 5
         let txn1 = Transaction::new(
             dataset.manifest.version,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 5)],
+                merged_generations: vec![MergedGeneration::new(shard, 5)],
             },
             None,
         );
@@ -249,7 +251,7 @@ mod tests {
         let txn2 = Transaction::new(
             dataset.manifest.version - 1, // Based on old version
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 10)],
+                merged_generations: vec![MergedGeneration::new(shard, 10)],
             },
             None,
         );
@@ -262,18 +264,18 @@ mod tests {
         );
     }
 
-    /// Test that UpdateMemWalState on different regions don't conflict.
+    /// Test that UpdateMemWalState on different shards don't conflict.
     #[tokio::test]
-    async fn test_update_mem_wal_state_different_regions_no_conflict() {
+    async fn test_update_mem_wal_state_different_shards_no_conflict() {
         let dataset = test_dataset().await;
-        let region1 = Uuid::new_v4();
-        let region2 = Uuid::new_v4();
+        let shard1 = Uuid::new_v4();
+        let shard2 = Uuid::new_v4();
 
-        // First commit UpdateMemWalState for region1
+        // First commit UpdateMemWalState for shard1
         let txn1 = Transaction::new(
             dataset.manifest.version,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region1, 10)],
+                merged_generations: vec![MergedGeneration::new(shard1, 10)],
             },
             None,
         );
@@ -282,12 +284,12 @@ mod tests {
             .await
             .unwrap();
 
-        // Commit UpdateMemWalState for region2 based on old version
-        // This should succeed because different regions don't conflict
+        // Commit UpdateMemWalState for shard2 based on old version
+        // This should succeed because different shards don't conflict
         let txn2 = Transaction::new(
             dataset.manifest.version - 1, // Based on old version
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region2, 5)],
+                merged_generations: vec![MergedGeneration::new(shard2, 5)],
             },
             None,
         );
@@ -295,11 +297,11 @@ mod tests {
 
         assert!(
             result.is_ok(),
-            "Expected success for different regions, got {:?}",
+            "Expected success for different shards, got {:?}",
             result
         );
 
-        // Verify both regions are in the index
+        // Verify both shards are in the index
         let dataset = result.unwrap();
         let mem_wal_idx = dataset
             .load_indices()
@@ -318,13 +320,13 @@ mod tests {
     #[tokio::test]
     async fn test_create_index_rebase_against_update_mem_wal_state() {
         let dataset = test_dataset().await;
-        let region = Uuid::new_v4();
+        let shard = Uuid::new_v4();
 
         // First commit UpdateMemWalState with generation 10
         let txn1 = Transaction::new(
             dataset.manifest.version,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 10)],
+                merged_generations: vec![MergedGeneration::new(shard, 10)],
             },
             None,
         );
@@ -336,7 +338,7 @@ mod tests {
         // CreateIndex of MemWalIndex based on old version (before UpdateMemWalState)
         // This should succeed and merge the generations
         let details = MemWalIndexDetails {
-            num_regions: 1,
+            num_shards: 1,
             ..Default::default()
         };
         let mem_wal_index = new_mem_wal_index_meta(dataset.manifest.version - 1, details).unwrap();
@@ -369,20 +371,20 @@ mod tests {
             .clone();
         let details = load_mem_wal_index_details(mem_wal_idx).unwrap();
         assert_eq!(details.merged_generations.len(), 1);
-        assert_eq!(details.merged_generations[0].region_id, region);
+        assert_eq!(details.merged_generations[0].shard_id, shard);
         assert_eq!(details.merged_generations[0].generation, 10);
-        assert_eq!(details.num_regions, 1); // Config from CreateIndex preserved
+        assert_eq!(details.num_shards, 1); // Config from CreateIndex preserved
     }
 
     /// Test that UpdateMemWalState against CreateIndex of MemWalIndex checks generations.
     #[tokio::test]
     async fn test_update_mem_wal_state_against_create_index_lower_generation() {
         let dataset = test_dataset().await;
-        let region = Uuid::new_v4();
+        let shard = Uuid::new_v4();
 
         // First commit CreateIndex of MemWalIndex with merged_generations
         let details = MemWalIndexDetails {
-            merged_generations: vec![MergedGeneration::new(region, 10)],
+            merged_generations: vec![MergedGeneration::new(shard, 10)],
             ..Default::default()
         };
         let mem_wal_index = new_mem_wal_index_meta(dataset.manifest.version, details).unwrap();
@@ -404,7 +406,7 @@ mod tests {
         let txn2 = Transaction::new(
             dataset.manifest.version - 1, // Based on old version
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 5)],
+                merged_generations: vec![MergedGeneration::new(shard, 5)],
             },
             None,
         );
@@ -420,28 +422,28 @@ mod tests {
     #[test]
     fn test_update_merged_generations() {
         let mut indices = Vec::new();
-        let region1 = Uuid::new_v4();
-        let region2 = Uuid::new_v4();
+        let shard1 = Uuid::new_v4();
+        let shard2 = Uuid::new_v4();
 
         // First update - creates new index
         update_mem_wal_index_merged_generations(
             &mut indices,
             1,
-            vec![MergedGeneration::new(region1, 5)],
+            vec![MergedGeneration::new(shard1, 5)],
         )
         .unwrap();
 
         assert_eq!(indices.len(), 1);
         let details = load_mem_wal_index_details(indices[0].clone()).unwrap();
         assert_eq!(details.merged_generations.len(), 1);
-        assert_eq!(details.merged_generations[0].region_id, region1);
+        assert_eq!(details.merged_generations[0].shard_id, shard1);
         assert_eq!(details.merged_generations[0].generation, 5);
 
-        // Second update - updates existing region
+        // Second update - updates existing shard
         update_mem_wal_index_merged_generations(
             &mut indices,
             2,
-            vec![MergedGeneration::new(region1, 10)],
+            vec![MergedGeneration::new(shard1, 10)],
         )
         .unwrap();
 
@@ -450,11 +452,11 @@ mod tests {
         assert_eq!(details.merged_generations.len(), 1);
         assert_eq!(details.merged_generations[0].generation, 10);
 
-        // Third update - adds new region
+        // Third update - adds new shard
         update_mem_wal_index_merged_generations(
             &mut indices,
             3,
-            vec![MergedGeneration::new(region2, 3)],
+            vec![MergedGeneration::new(shard2, 3)],
         )
         .unwrap();
 
@@ -466,7 +468,7 @@ mod tests {
         update_mem_wal_index_merged_generations(
             &mut indices,
             4,
-            vec![MergedGeneration::new(region1, 8)], // lower than 10
+            vec![MergedGeneration::new(shard1, 8)], // lower than 10
         )
         .unwrap();
 
@@ -474,7 +476,7 @@ mod tests {
         let r1_mg = details
             .merged_generations
             .iter()
-            .find(|mg| mg.region_id == region1)
+            .find(|mg| mg.shard_id == shard1)
             .unwrap();
         assert_eq!(r1_mg.generation, 10); // Should still be 10
     }
diff --git a/rust/lance/src/index/prefilter.rs b/rust/lance/src/index/prefilter.rs
index a86660acfa0..de1d97e1e31 100644
--- a/rust/lance/src/index/prefilter.rs
+++ b/rust/lance/src/index/prefilter.rs
@@ -49,6 +49,9 @@ pub struct DatasetPreFilter {
     // these tasks only when we've done as much work as we can without them.
     pub(super) deleted_ids: Option<Arc<SharedPrerequisite<Arc<RowAddrMask>>>>,
     pub(super) filtered_ids: Option<Arc<SharedPrerequisite<RowAddrMask>>>,
+    // Fragment IDs whose data is still in the index but has been removed from the dataset.
+    // Used by FTS merge-on-read to prune stale fragments at search time.
+    pub(super) deleted_fragments: Option<RoaringBitmap>,
     // When the tasks are finished this is the combined filter
     pub(super) final_mask: Mutex<OnceCell<Arc<RowAddrMask>>>,
 }
@@ -74,6 +77,7 @@ impl DatasetPreFilter {
         Self {
             deleted_ids,
             filtered_ids,
+            deleted_fragments: None,
             final_mask: Mutex::new(OnceCell::new()),
         }
     }
@@ -174,6 +178,14 @@ impl DatasetPreFilter {
             .await
     }
 
+    /// Sets the deleted fragment IDs to block during search.
+    ///
+    /// Used by FTS indices which track fragments that have been removed from the
+    /// dataset but whose data is still present in the index (merge-on-read).
+    pub fn set_deleted_fragments(&mut self, fragments: RoaringBitmap) {
+        self.deleted_fragments = Some(fragments);
+    }
+
     /// Creates a task to load mask to filter out deleted rows.
     ///
     /// Sometimes this will be a block list of row ids that are deleted, based
@@ -244,6 +256,13 @@ impl PreFilter for DatasetPreFilter {
             if let Some(deleted_ids) = &self.deleted_ids {
                 combined = combined & (*deleted_ids.get_ready()).clone();
             }
+            if let Some(deleted) = &self.deleted_fragments {
+                let mut block_list = RowAddrTreeMap::new();
+                for frag_id in deleted.iter() {
+                    block_list.insert_fragment(frag_id);
+                }
+                combined = combined & RowAddrMask::from_block(block_list);
+            }
             Arc::new(combined)
         });
 
@@ -251,7 +270,9 @@ impl PreFilter for DatasetPreFilter {
     }
 
     fn is_empty(&self) -> bool {
-        self.deleted_ids.is_none() && self.filtered_ids.is_none()
+        self.deleted_ids.is_none()
+            && self.filtered_ids.is_none()
+            && self.deleted_fragments.is_none()
     }
 
     /// Get the row id mask for this prefilter
diff --git a/rust/lance/src/index/scalar.rs b/rust/lance/src/index/scalar.rs
index 0365f306c9b..44739454bec 100644
--- a/rust/lance/src/index/scalar.rs
+++ b/rust/lance/src/index/scalar.rs
@@ -6,6 +6,7 @@
 
 use std::sync::{Arc, LazyLock};
 
+use crate::index::DatasetIndexExt;
 use crate::index::DatasetIndexInternalExt;
 use crate::session::index_caches::ProstAny;
 use crate::{
@@ -28,6 +29,9 @@ use lance_index::progress::IndexBuildProgress;
 use lance_index::registry::IndexPluginRegistry;
 use lance_index::scalar::IndexStore;
 use lance_index::scalar::inverted::METADATA_FILE;
+use lance_index::scalar::label_list::{
+    LABEL_LIST_NULLS_METADATA_KEY, LABEL_LIST_NULLS_MIN_VERSION,
+};
 use lance_index::scalar::registry::{
     ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, VALUE_COLUMN_NAME,
 };
@@ -36,7 +40,7 @@ use lance_index::scalar::{
     ScalarIndex, ScalarIndexParams, bitmap::BITMAP_LOOKUP_NAME, inverted::INVERT_LIST_FILE,
     lance_format::LanceIndexStore,
 };
-use lance_index::{DatasetIndexExt, IndexCriteria, IndexType};
+use lance_index::{IndexCriteria, IndexType};
 use lance_table::format::{Fragment, IndexMetadata};
 use log::info;
 use tracing::instrument;
@@ -299,7 +303,7 @@ pub(super) async fn build_scalar_index(
     };
     progress.stage_complete("load_data").await?;
 
-    plugin
+    let created_index = plugin
         .train_index(
             training_data,
             &index_store,
@@ -307,7 +311,9 @@ pub(super) async fn build_scalar_index(
             fragment_ids,
             progress,
         )
-        .await
+        .await?;
+
+    Ok(created_index)
 }
 
 /// Fetches the scalar index plugin for a given index metadata
@@ -329,6 +335,43 @@ pub async fn fetch_index_details(
     Ok(index_details)
 }
 
+async fn validate_label_list_index_compatibility(
+    dataset: &Dataset,
+    column: &str,
+    index: &IndexMetadata,
+    index_store: &Arc<LanceIndexStore>,
+) -> Result<()> {
+    let Some(field) = dataset.schema().field(column) else {
+        return Ok(());
+    };
+
+    if !field.nullable {
+        return Ok(());
+    }
+
+    if index.index_version < LABEL_LIST_NULLS_MIN_VERSION {
+        log::warn!(
+            "LabelList index {} is old; NOT filters may be incorrect on nullable lists. Consider rebuilding.",
+            index.name
+        );
+        return Ok(());
+    }
+
+    let reader = index_store.open_index_file(BITMAP_LOOKUP_NAME).await?;
+    if !reader
+        .schema()
+        .metadata
+        .contains_key(LABEL_LIST_NULLS_METADATA_KEY)
+    {
+        return Err(Error::internal(format!(
+            "LabelList index {} is missing required metadata key {}",
+            index.name, LABEL_LIST_NULLS_METADATA_KEY
+        )));
+    }
+
+    Ok(())
+}
+
 pub async fn open_scalar_index(
     dataset: &Dataset,
     column: &str,
@@ -341,6 +384,10 @@ pub async fn open_scalar_index(
     let index_details = fetch_index_details(dataset, column, index).await?;
     let plugin = SCALAR_INDEX_PLUGIN_REGISTRY.get_plugin_by_details(index_details.as_ref())?;
 
+    if index_details.type_url.ends_with("LabelListIndexDetails") {
+        validate_label_list_index_compatibility(dataset, column, index, &index_store).await?;
+    }
+
     let frag_reuse_index = dataset.open_frag_reuse_index(metrics).await?;
 
     let index_cache = dataset
@@ -461,22 +508,20 @@ pub fn index_matches_criteria(
         return Ok(true);
     };
 
-    if index_details.is_vector() {
-        // This method is only for finding matching scalar indexes today so reject any vector indexes
-        return Ok(false);
-    }
-
-    if criteria.must_support_fts && !index_details.supports_fts() {
-        return Ok(false);
-    }
-
-    // We should not use FTS / NGram indices for exact equality queries
-    // (i.e. merge insert with a join on the indexed column)
-    if criteria.must_support_exact_equality {
-        let plugin = index_details.get_plugin()?;
-        if !plugin.provides_exact_answer() {
+    // Only apply scalar-specific checks to scalar indices
+    if !index_details.is_vector() {
+        if criteria.must_support_fts && !index_details.supports_fts() {
             return Ok(false);
         }
+
+        // We should not use FTS / NGram indices for exact equality queries
+        // (i.e. merge insert with a join on the indexed column)
+        if criteria.must_support_exact_equality {
+            let plugin = index_details.get_plugin()?;
+            if !plugin.provides_exact_answer() {
+                return Ok(false);
+            }
+        }
     }
     Ok(true)
 }
@@ -596,6 +641,7 @@ mod tests {
             index_version: 0,
             created_at: None,
             base_id: None,
+            files: None,
         }
     }
 
@@ -615,11 +661,12 @@ mod tests {
             fields: vec![field.clone()],
             metadata: Default::default(),
         };
+        // Vector indices should now match basic criteria
         let result = index_matches_criteria(&index1, &criteria, &[&field], true, &schema).unwrap();
-        assert!(!result);
+        assert!(result);
 
         let result = index_matches_criteria(&index1, &criteria, &[&field], false, &schema).unwrap();
-        assert!(!result);
+        assert!(result);
     }
 
     #[test]
@@ -773,9 +820,9 @@ mod tests {
     #[tokio::test]
     async fn test_initialize_scalar_index_btree() {
         use crate::dataset::Dataset;
+        use crate::index::DatasetIndexExt;
         use arrow_array::types::Float32Type;
         use lance_datagen::{BatchCount, RowCount, array};
-        use lance_index::DatasetIndexExt;
         use lance_index::metrics::NoOpMetricsCollector;
         use lance_index::scalar::ScalarIndexParams;
 
@@ -879,9 +926,9 @@ mod tests {
     #[tokio::test]
     async fn test_optimize_scalar_index_btree() {
         use crate::dataset::Dataset;
+        use crate::index::DatasetIndexExt;
         use arrow_array::types::Float32Type;
         use lance_datagen::{BatchCount, RowCount, array};
-        use lance_index::DatasetIndexExt;
         use lance_index::metrics::NoOpMetricsCollector;
         use lance_index::scalar::ScalarIndexParams;
 
@@ -997,9 +1044,9 @@ mod tests {
     #[tokio::test]
     async fn test_initialize_scalar_index_bitmap() {
         use crate::dataset::Dataset;
+        use crate::index::DatasetIndexExt;
         use arrow_array::types::Float32Type;
         use lance_datagen::{BatchCount, RowCount, array};
-        use lance_index::DatasetIndexExt;
         use lance_index::scalar::ScalarIndexParams;
 
         let test_dir = TempStrDir::default();
@@ -1077,8 +1124,8 @@ mod tests {
     #[tokio::test]
     async fn test_initialize_scalar_index_inverted() {
         use crate::dataset::Dataset;
+        use crate::index::DatasetIndexExt;
         use lance_datagen::{BatchCount, ByteCount, RowCount, array};
-        use lance_index::DatasetIndexExt;
         use lance_index::metrics::NoOpMetricsCollector;
         use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
 
@@ -1216,9 +1263,9 @@ mod tests {
     #[tokio::test]
     async fn test_initialize_scalar_index_zonemap() {
         use crate::dataset::Dataset;
+        use crate::index::DatasetIndexExt;
         use arrow_array::types::Float32Type;
         use lance_datagen::{BatchCount, RowCount, array};
-        use lance_index::DatasetIndexExt;
         use lance_index::metrics::NoOpMetricsCollector;
         use lance_index::scalar::ScalarIndexParams;
         use lance_index::scalar::zonemap::ZoneMapIndexBuilderParams;
diff --git a/rust/lance/src/index/vector.rs b/rust/lance/src/index/vector.rs
index 4fc2f910527..e137237b9a0 100644
--- a/rust/lance/src/index/vector.rs
+++ b/rust/lance/src/index/vector.rs
@@ -16,8 +16,11 @@ pub mod utils;
 mod fixture_test;
 
 use self::{ivf::*, pq::PQIndex};
-use arrow_schema::DataType;
+use arrow_schema::{DataType, Schema};
 use builder::IvfIndexBuilder;
+use datafusion::physical_plan::SendableRecordBatchStream;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+use futures::stream;
 use lance_core::utils::tempfile::TempStdDir;
 use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_index::frag_reuse::FragReuseIndex;
@@ -35,7 +38,7 @@ use object_store::path::Path;
 use lance_arrow::FixedSizeListArrayExt;
 use lance_index::vector::pq::ProductQuantizer;
 use lance_index::vector::quantizer::QuantizationType;
-use lance_index::vector::v3::shuffler::IvfShuffler;
+use lance_index::vector::v3::shuffler::{Shuffler, create_ivf_shuffler};
 use lance_index::vector::v3::subindex::SubIndexType;
 use lance_index::vector::{
     VectorIndex,
@@ -47,23 +50,176 @@ use lance_index::vector::{
     pq::PQBuildParams,
     sq::{ScalarQuantizer, builder::SQBuildParams},
 };
-use lance_index::{
-    DatasetIndexExt, INDEX_AUXILIARY_FILE_NAME, INDEX_METADATA_SCHEMA_KEY, IndexType,
-};
+use lance_index::{INDEX_AUXILIARY_FILE_NAME, INDEX_METADATA_SCHEMA_KEY, IndexType};
 use lance_io::traits::Reader;
 use lance_linalg::distance::*;
-use lance_table::format::IndexMetadata;
+use lance_table::format::{IndexMetadata, list_index_files_with_sizes};
 use serde::Serialize;
 use tracing::instrument;
 use utils::get_vector_type;
 use uuid::Uuid;
 
-use super::{DatasetIndexInternalExt, IndexParams, pb, vector_index_details};
+use super::{DatasetIndexExt, DatasetIndexInternalExt, IndexParams, pb, vector_index_details};
+use crate::dataset::index::dataset_format_version;
 use crate::dataset::transaction::{Operation, Transaction};
 use crate::{Error, Result, dataset::Dataset, index::pb::vector_index_stage::Stage};
 
 pub const LANCE_VECTOR_INDEX: &str = "__lance_vector_index";
 
+/// A materialized snapshot of one logical vector index and all of its segments.
+#[derive(Debug)]
+pub struct LogicalVectorIndex {
+    name: String,
+    column: String,
+    segments: Vec<(IndexMetadata, Arc<dyn VectorIndex>)>,
+}
+
+/// An IVF-specific inspection and maintenance view over a [`LogicalVectorIndex`].
+///
+/// Callers must explicitly opt into this view before accessing partition-level
+/// APIs so that IVF semantics stay separated from the generic logical vector
+/// index abstraction.
+#[derive(Clone, Copy, Debug)]
+pub struct LogicalIvfView<'a> {
+    logical_index: &'a LogicalVectorIndex,
+}
+
+impl LogicalVectorIndex {
+    pub(crate) fn try_new(
+        name: String,
+        column: String,
+        segments: Vec<(IndexMetadata, Arc<dyn VectorIndex>)>,
+    ) -> Result<Self> {
+        if segments.is_empty() {
+            return Err(Error::invalid_input(format!(
+                "LogicalVectorIndex '{}' on column '{}' must contain at least one segment",
+                name, column
+            )));
+        }
+
+        Ok(Self {
+            name,
+            column,
+            segments,
+        })
+    }
+
+    /// Returns the logical index name.
+    pub fn name(&self) -> &str {
+        &self.name
+    }
+
+    /// Returns the vector column that this logical index is built on.
+    pub fn column(&self) -> &str {
+        &self.column
+    }
+
+    /// Returns the number of physical segments in this logical index.
+    pub fn num_segments(&self) -> usize {
+        self.segments.len()
+    }
+
+    /// Returns the committed metadata for all physical segments.
+    pub fn metadatas(&self) -> impl ExactSizeIterator<Item = &IndexMetadata> + '_ {
+        self.segments.iter().map(|(metadata, _)| metadata)
+    }
+
+    /// Returns the indexed row count for each segment.
+    pub fn num_rows_per_segment(&self) -> Vec<(Uuid, u64)> {
+        self.segments
+            .iter()
+            .map(|(metadata, index)| (metadata.uuid, index.num_rows()))
+            .collect()
+    }
+
+    /// Returns an IVF view over this logical index.
+    ///
+    /// This is currently fallible only to leave room for future non-IVF vector
+    /// index families. All vector indices currently opened through this path are
+    /// IVF-backed.
+    pub fn as_ivf(&self) -> Result<LogicalIvfView<'_>> {
+        Ok(LogicalIvfView {
+            logical_index: self,
+        })
+    }
+
+    pub(crate) fn iter(
+        &self,
+    ) -> impl ExactSizeIterator<Item = (&IndexMetadata, &Arc<dyn VectorIndex>)> + '_ {
+        self.segments
+            .iter()
+            .map(|(metadata, index)| (metadata, index))
+    }
+}
+
+impl<'a> LogicalIvfView<'a> {
+    pub(crate) fn indices(&self) -> impl ExactSizeIterator<Item = &Arc<dyn VectorIndex>> + '_ {
+        self.logical_index.iter().map(|(_, index)| index)
+    }
+
+    pub(crate) fn segments(
+        &self,
+    ) -> impl ExactSizeIterator<Item = (&IndexMetadata, &Arc<dyn VectorIndex>)> + '_ {
+        self.logical_index.iter()
+    }
+
+    /// Returns the partition count for each segment in this IVF index.
+    pub fn num_partitions_per_segment(&self) -> Vec<(Uuid, usize)> {
+        self.logical_index
+            .iter()
+            .map(|(metadata, index)| (metadata.uuid, index.ivf_model().num_partitions()))
+            .collect()
+    }
+
+    /// Returns the partition sizes for each segment in this IVF index.
+    pub fn partition_sizes(&self) -> Vec<(Uuid, Vec<usize>)> {
+        let mut partition_sizes = Vec::with_capacity(self.logical_index.num_segments());
+        for (metadata, index) in self.logical_index.iter() {
+            let num_partitions = index.ivf_model().num_partitions();
+            let mut sizes = Vec::with_capacity(num_partitions);
+            for partition_id in 0..num_partitions {
+                sizes.push(index.partition_size(partition_id));
+            }
+            partition_sizes.push((metadata.uuid, sizes));
+        }
+        partition_sizes
+    }
+
+    /// Reads one IVF partition across all segments in this logical index.
+    ///
+    /// The returned stream preserves segment boundaries only implicitly through
+    /// concatenation order; callers should treat it as a merged partition view.
+    pub async fn read_partition(
+        &self,
+        partition_id: usize,
+        with_vector: bool,
+    ) -> Result<SendableRecordBatchStream> {
+        let mut schema: Option<Arc<Schema>> = None;
+        let mut partition_streams = Vec::with_capacity(self.logical_index.num_segments());
+        for index in self.indices() {
+            let stream = index
+                .partition_reader(partition_id, with_vector, &NoOpMetricsCollector)
+                .await?;
+            if schema.is_none() {
+                schema = Some(stream.schema());
+            }
+            partition_streams.push(stream);
+        }
+
+        match schema {
+            Some(schema) => {
+                let merged = stream::select_all(partition_streams);
+                let stream = RecordBatchStreamAdapter::new(schema, merged);
+                Ok(Box::pin(stream))
+            }
+            None => Ok(Box::pin(RecordBatchStreamAdapter::new(
+                Arc::new(Schema::empty()),
+                stream::empty(),
+            ))),
+        }
+    }
+}
+
 /// Parameters of each index stage.
 #[derive(Debug, Clone)]
 pub enum StageParams {
@@ -328,56 +484,51 @@ impl IndexParams for VectorIndexParams {
     }
 }
 
-/// Build a Distributed Vector Index for specific fragments
-#[allow(clippy::too_many_arguments)]
-#[instrument(level = "debug", skip(dataset))]
-pub(crate) async fn build_distributed_vector_index(
+/// Prepare the shared build inputs used by both direct local builds and
+/// staged shard builds.
+///
+/// These paths emit different file layouts, but they follow the same rules for
+/// validating the vector column, deriving the effective index type, sizing IVF
+/// partitions, and constructing the shuffler.
+async fn prepare_vector_segment_build(
     dataset: &Dataset,
     column: &str,
-    _name: &str,
-    uuid: &str,
     params: &VectorIndexParams,
-    frag_reuse_index: Option<Arc<FragReuseIndex>>,
-    fragment_ids: &[u32],
     progress: Arc<dyn IndexBuildProgress>,
-) -> Result<()> {
+    mode: &str,
+    require_precomputed_ivf: bool,
+) -> Result<(DataType, IndexType, IvfBuildParams, Box<dyn Shuffler>)> {
     let stages = &params.stages;
 
     if stages.is_empty() {
-        return Err(Error::index(
-            "Build Distributed Vector Index: must have at least 1 stage".to_string(),
-        ));
-    };
+        return Err(Error::index(format!("{mode}: must have at least 1 stage")));
+    }
 
     let StageParams::Ivf(ivf_params0) = &stages[0] else {
         return Err(Error::index(format!(
-            "Build Distributed Vector Index: invalid stages: {:?}",
+            "{mode}: invalid stages: {:?}",
             stages
         )));
     };
 
-    if ivf_params0.centroids.is_none() {
-        return Err(Error::index(
-            "Build Distributed Vector Index: missing precomputed IVF centroids; \
-        please provide IvfBuildParams.centroids \
-        for concurrent distributed create_index"
-                .to_string(),
-        ));
+    if require_precomputed_ivf && ivf_params0.centroids.is_none() {
+        return Err(Error::index(format!(
+            "{mode}: missing precomputed IVF centroids; please provide \
+                 IvfBuildParams.centroids for distributed segment build"
+        )));
     }
 
     let (vector_type, element_type) = get_vector_type(dataset.schema(), column)?;
     if let DataType::List(_) = vector_type
         && params.metric_type != DistanceType::Cosine
     {
-        return Err(Error::index(
-            "Build Distributed Vector Index: multivector type supports only cosine distance"
-                .to_string(),
-        ));
+        return Err(Error::index(format!(
+            "{mode}: multivector type supports only cosine distance"
+        )));
     }
 
     let num_rows = dataset.count_rows(None).await?;
     let index_type = params.index_type();
-
     let num_partitions = ivf_params0.num_partitions.unwrap_or_else(|| {
         recommended_num_partitions(
             num_rows,
@@ -386,10 +537,46 @@ pub(crate) async fn build_distributed_vector_index(
                 .unwrap_or(index_type.target_partition_size()),
         )
     });
-
     let mut ivf_params = ivf_params0.clone();
     ivf_params.num_partitions = Some(num_partitions);
 
+    let format_version = dataset_format_version(dataset);
+    let temp_dir = TempStdDir::default();
+    let temp_dir_path = Path::from_filesystem_path(&temp_dir)?;
+    let shuffler = create_ivf_shuffler(
+        temp_dir_path,
+        num_partitions,
+        format_version,
+        Some(progress),
+    );
+
+    Ok((element_type, index_type, ivf_params, shuffler))
+}
+
+/// Build a Distributed Vector Index for specific fragments
+#[allow(clippy::too_many_arguments)]
+#[instrument(level = "debug", skip(dataset))]
+pub(crate) async fn build_distributed_vector_index(
+    dataset: &Dataset,
+    column: &str,
+    _name: &str,
+    uuid: &str,
+    params: &VectorIndexParams,
+    frag_reuse_index: Option<Arc<FragReuseIndex>>,
+    fragment_ids: &[u32],
+    progress: Arc<dyn IndexBuildProgress>,
+) -> Result<Uuid> {
+    let (element_type, index_type, ivf_params, shuffler) = prepare_vector_segment_build(
+        dataset,
+        column,
+        params,
+        progress.clone(),
+        "Build Distributed Vector Index",
+        true,
+    )
+    .await?;
+    let stages = &params.stages;
+
     let ivf_centroids = ivf_params
         .centroids
         .as_ref()
@@ -397,19 +584,11 @@ pub(crate) async fn build_distributed_vector_index(
         .as_ref()
         .clone();
 
-    let temp_dir = TempStdDir::default();
-    let temp_dir_path = Path::from_filesystem_path(&temp_dir)?;
-    let shuffler = IvfShuffler::new(temp_dir_path, num_partitions).with_progress(progress.clone());
-
     let filtered_dataset = dataset.clone();
 
-    let out_base = dataset.indices_dir().child(uuid);
-
-    let make_partial_index_dir = |out_base: &Path| -> Path {
-        let shard_uuid = Uuid::new_v4();
-        out_base.child(format!("partial_{}", shard_uuid))
-    };
-    let new_index_dir = || make_partial_index_dir(&out_base);
+    let segment_uuid = Uuid::parse_str(uuid)
+        .map_err(|err| Error::invalid_input(format!("Invalid index UUID '{uuid}': {err}")))?;
+    let index_dir = dataset.indices_dir().child(segment_uuid.to_string());
 
     let fragment_filter = fragment_ids.to_vec();
 
@@ -450,15 +629,14 @@ pub(crate) async fn build_distributed_vector_index(
     match index_type {
         IndexType::IvfFlat => match element_type {
             DataType::Float16 | DataType::Float32 | DataType::Float64 => {
-                let index_dir = new_index_dir();
                 let ivf_model = make_ivf_model();
 
                 IvfIndexBuilder::<FlatIndex, FlatQuantizer>::new(
                     filtered_dataset,
                     column.to_owned(),
-                    index_dir,
+                    index_dir.clone(),
                     params.metric_type,
-                    Box::new(shuffler),
+                    shuffler,
                     Some(ivf_params),
                     Some(()),
                     (),
@@ -471,15 +649,14 @@ pub(crate) async fn build_distributed_vector_index(
                 .await?;
             }
             DataType::UInt8 => {
-                let index_dir = new_index_dir();
                 let ivf_model = make_ivf_model();
 
                 IvfIndexBuilder::<FlatIndex, FlatBinQuantizer>::new(
                     filtered_dataset,
                     column.to_owned(),
-                    index_dir,
+                    index_dir.clone(),
                     params.metric_type,
-                    Box::new(shuffler),
+                    shuffler,
                     Some(ivf_params),
                     Some(()),
                     (),
@@ -515,16 +692,15 @@ pub(crate) async fn build_distributed_vector_index(
                     ));
                 }
                 IndexFileVersion::V3 => {
-                    let index_dir = new_index_dir();
                     let ivf_model = make_ivf_model();
                     let global_pq = make_global_pq(pq_params)?;
 
                     IvfIndexBuilder::<FlatIndex, ProductQuantizer>::new(
                         filtered_dataset,
                         column.to_owned(),
-                        index_dir,
+                        index_dir.clone(),
                         params.metric_type,
-                        Box::new(shuffler),
+                        shuffler,
                         Some(ivf_params),
                         Some(pq_params.clone()),
                         (),
@@ -550,15 +726,12 @@ pub(crate) async fn build_distributed_vector_index(
                     stages
                 )));
             };
-
-            let index_dir = new_index_dir();
-
             IvfIndexBuilder::<FlatIndex, ScalarQuantizer>::new(
                 filtered_dataset,
                 column.to_owned(),
-                index_dir,
+                index_dir.clone(),
                 params.metric_type,
-                Box::new(shuffler),
+                shuffler,
                 Some(ivf_params),
                 Some(sq_params.clone()),
                 (),
@@ -578,23 +751,42 @@ pub(crate) async fn build_distributed_vector_index(
                 )));
             };
 
-            let index_dir = new_index_dir();
-
-            IvfIndexBuilder::<HNSW, FlatQuantizer>::new(
-                filtered_dataset,
-                column.to_owned(),
-                index_dir,
-                params.metric_type,
-                Box::new(shuffler),
-                Some(ivf_params),
-                Some(()),
-                hnsw_params.clone(),
-                frag_reuse_index,
-            )?
-            .with_fragment_filter(fragment_filter)
-            .with_progress(progress.clone())
-            .build()
-            .await?;
+            match element_type {
+                DataType::UInt8 => {
+                    IvfIndexBuilder::<HNSW, FlatBinQuantizer>::new(
+                        filtered_dataset,
+                        column.to_owned(),
+                        index_dir.clone(),
+                        params.metric_type,
+                        shuffler,
+                        Some(ivf_params),
+                        Some(()),
+                        hnsw_params.clone(),
+                        frag_reuse_index,
+                    )?
+                    .with_fragment_filter(fragment_filter)
+                    .with_progress(progress.clone())
+                    .build()
+                    .await?;
+                }
+                _ => {
+                    IvfIndexBuilder::<HNSW, FlatQuantizer>::new(
+                        filtered_dataset,
+                        column.to_owned(),
+                        index_dir.clone(),
+                        params.metric_type,
+                        shuffler,
+                        Some(ivf_params),
+                        Some(()),
+                        hnsw_params.clone(),
+                        frag_reuse_index,
+                    )?
+                    .with_fragment_filter(fragment_filter)
+                    .with_progress(progress.clone())
+                    .build()
+                    .await?;
+                }
+            }
         }
 
         IndexType::IvfHnswPq => {
@@ -611,16 +803,15 @@ pub(crate) async fn build_distributed_vector_index(
                 )));
             };
 
-            let index_dir = new_index_dir();
             let ivf_model = make_ivf_model();
             let global_pq = make_global_pq(pq_params)?;
 
             IvfIndexBuilder::<HNSW, ProductQuantizer>::new(
                 filtered_dataset,
                 column.to_owned(),
-                index_dir,
+                index_dir.clone(),
                 params.metric_type,
-                Box::new(shuffler),
+                shuffler,
                 Some(ivf_params),
                 Some(pq_params.clone()),
                 hnsw_params.clone(),
@@ -650,15 +841,12 @@ pub(crate) async fn build_distributed_vector_index(
                     stages
                 )));
             };
-
-            let index_dir = new_index_dir();
-
             IvfIndexBuilder::<HNSW, ScalarQuantizer>::new(
                 filtered_dataset,
                 column.to_owned(),
-                index_dir,
+                index_dir.clone(),
                 params.metric_type,
-                Box::new(shuffler),
+                shuffler,
                 Some(ivf_params),
                 Some(sq_params.clone()),
                 hnsw_params.clone(),
@@ -671,11 +859,34 @@ pub(crate) async fn build_distributed_vector_index(
         }
 
         IndexType::IvfRq => {
-            return Err(Error::index(format!(
-                "Build Distributed Vector Index: invalid index type: {:?} \
-            is not supported in distributed mode; skipping this shard",
-                index_type
-            )));
+            let StageParams::RQ(rq_params) = &stages[1] else {
+                return Err(Error::index(format!(
+                    "Build Distributed Vector Index: invalid stages: {:?}",
+                    stages
+                )));
+            };
+
+            let ivf_model = make_ivf_model();
+
+            IvfIndexBuilder::<FlatIndex, RabitQuantizer>::new(
+                filtered_dataset,
+                column.to_owned(),
+                index_dir.clone(),
+                params.metric_type,
+                shuffler,
+                Some(ivf_params),
+                Some(rq_params.clone()),
+                (),
+                frag_reuse_index,
+            )?
+            .with_ivf(ivf_model)
+            // For distributed shards, keep RQ codes in row-major layout.
+            // A single packing pass is performed in the distributed merge stage.
+            .with_transpose(false)
+            .with_fragment_filter(fragment_filter)
+            .with_progress(progress.clone())
+            .build()
+            .await?;
         }
 
         _ => {
@@ -686,7 +897,7 @@ pub(crate) async fn build_distributed_vector_index(
         }
     };
 
-    Ok(())
+    Ok(segment_uuid)
 }
 
 /// Build a Vector Index
@@ -700,46 +911,17 @@ pub(crate) async fn build_vector_index(
     frag_reuse_index: Option<Arc<FragReuseIndex>>,
     progress: Arc<dyn IndexBuildProgress>,
 ) -> Result<()> {
+    let (element_type, index_type, ivf_params, shuffler) = prepare_vector_segment_build(
+        dataset,
+        column,
+        params,
+        progress.clone(),
+        "Build Vector Index",
+        false,
+    )
+    .await?;
     let stages = &params.stages;
 
-    if stages.is_empty() {
-        return Err(Error::index(
-            "Build Vector Index: must have at least 1 stage".to_string(),
-        ));
-    };
-
-    let StageParams::Ivf(ivf_params) = &stages[0] else {
-        return Err(Error::index(format!(
-            "Build Vector Index: invalid stages: {:?}",
-            stages
-        )));
-    };
-
-    let (vector_type, element_type) = get_vector_type(dataset.schema(), column)?;
-    if let DataType::List(_) = vector_type
-        && params.metric_type != DistanceType::Cosine
-    {
-        return Err(Error::index(
-            "Build Vector Index: multivector type supports only cosine distance".to_string(),
-        ));
-    }
-
-    let num_rows = dataset.count_rows(None).await?;
-    let index_type = params.index_type();
-    let num_partitions = ivf_params.num_partitions.unwrap_or_else(|| {
-        recommended_num_partitions(
-            num_rows,
-            ivf_params
-                .target_partition_size
-                .unwrap_or(index_type.target_partition_size()),
-        )
-    });
-    let mut ivf_params = ivf_params.clone();
-    ivf_params.num_partitions = Some(num_partitions);
-
-    let temp_dir = TempStdDir::default();
-    let temp_dir_path = Path::from_filesystem_path(&temp_dir)?;
-    let shuffler = IvfShuffler::new(temp_dir_path, num_partitions).with_progress(progress.clone());
     match index_type {
         IndexType::IvfFlat => match element_type {
             DataType::Float16 | DataType::Float32 | DataType::Float64 => {
@@ -748,7 +930,7 @@ pub(crate) async fn build_vector_index(
                     column.to_owned(),
                     dataset.indices_dir().child(uuid),
                     params.metric_type,
-                    Box::new(shuffler),
+                    shuffler,
                     Some(ivf_params),
                     Some(()),
                     (),
@@ -764,7 +946,7 @@ pub(crate) async fn build_vector_index(
                     column.to_owned(),
                     dataset.indices_dir().child(uuid),
                     params.metric_type,
-                    Box::new(shuffler),
+                    shuffler,
                     Some(ivf_params),
                     Some(()),
                     (),
@@ -810,7 +992,7 @@ pub(crate) async fn build_vector_index(
                         column.to_owned(),
                         dataset.indices_dir().child(uuid),
                         params.metric_type,
-                        Box::new(shuffler),
+                        shuffler,
                         Some(ivf_params),
                         Some(pq_params.clone()),
                         (),
@@ -838,7 +1020,7 @@ pub(crate) async fn build_vector_index(
                 column.to_owned(),
                 dataset.indices_dir().child(uuid),
                 params.metric_type,
-                Box::new(shuffler),
+                shuffler,
                 Some(ivf_params),
                 Some(sq_params.clone()),
                 (),
@@ -861,7 +1043,7 @@ pub(crate) async fn build_vector_index(
                 column.to_owned(),
                 dataset.indices_dir().child(uuid),
                 params.metric_type,
-                Box::new(shuffler),
+                shuffler,
                 Some(ivf_params),
                 Some(rq_params.clone()),
                 (),
@@ -881,20 +1063,40 @@ pub(crate) async fn build_vector_index(
                     stages
                 )));
             };
-            IvfIndexBuilder::<HNSW, FlatQuantizer>::new(
-                dataset.clone(),
-                column.to_owned(),
-                dataset.indices_dir().child(uuid),
-                params.metric_type,
-                Box::new(shuffler),
-                Some(ivf_params),
-                Some(()),
-                hnsw_params.clone(),
-                frag_reuse_index,
-            )?
-            .with_progress(progress.clone())
-            .build()
-            .await?;
+            match element_type {
+                DataType::UInt8 => {
+                    IvfIndexBuilder::<HNSW, FlatBinQuantizer>::new(
+                        dataset.clone(),
+                        column.to_owned(),
+                        dataset.indices_dir().child(uuid),
+                        params.metric_type,
+                        shuffler,
+                        Some(ivf_params),
+                        Some(()),
+                        hnsw_params.clone(),
+                        frag_reuse_index,
+                    )?
+                    .with_progress(progress.clone())
+                    .build()
+                    .await?;
+                }
+                _ => {
+                    IvfIndexBuilder::<HNSW, FlatQuantizer>::new(
+                        dataset.clone(),
+                        column.to_owned(),
+                        dataset.indices_dir().child(uuid),
+                        params.metric_type,
+                        shuffler,
+                        Some(ivf_params),
+                        Some(()),
+                        hnsw_params.clone(),
+                        frag_reuse_index,
+                    )?
+                    .with_progress(progress.clone())
+                    .build()
+                    .await?;
+                }
+            }
         }
         IndexType::IvfHnswPq => {
             let StageParams::Hnsw(hnsw_params) = &stages[1] else {
@@ -914,7 +1116,7 @@ pub(crate) async fn build_vector_index(
                 column.to_owned(),
                 dataset.indices_dir().child(uuid),
                 params.metric_type,
-                Box::new(shuffler),
+                shuffler,
                 Some(ivf_params),
                 Some(pq_params.clone()),
                 hnsw_params.clone(),
@@ -942,7 +1144,7 @@ pub(crate) async fn build_vector_index(
                 column.to_owned(),
                 dataset.indices_dir().child(uuid),
                 params.metric_type,
-                Box::new(shuffler),
+                shuffler,
                 Some(ivf_params),
                 Some(sq_params.clone()),
                 hnsw_params.clone(),
@@ -989,7 +1191,7 @@ pub(crate) async fn build_vector_index_incremental(
         )));
     };
 
-    let (vector_type, element_type) = get_vector_type(dataset.schema(), column)?;
+    let (vector_type, _) = get_vector_type(dataset.schema(), column)?;
     if let DataType::List(_) = vector_type
         && params.metric_type != DistanceType::Cosine
     {
@@ -1014,10 +1216,15 @@ pub(crate) async fn build_vector_index_incremental(
         )));
     }
 
+    let format_version = dataset_format_version(dataset);
+
     let temp_dir = TempStdDir::default();
     let temp_dir_path = Path::from_filesystem_path(&temp_dir)?;
-    let shuffler = Box::new(
-        IvfShuffler::new(temp_dir_path, ivf_model.num_partitions()).with_progress(progress.clone()),
+    let shuffler = create_ivf_shuffler(
+        temp_dir_path,
+        ivf_model.num_partitions(),
+        format_version,
+        Some(progress.clone()),
     );
 
     let index_dir = dataset.indices_dir().child(uuid);
@@ -1027,48 +1234,40 @@ pub(crate) async fn build_vector_index_incremental(
 
     match (sub_index_type, quantization_type) {
         // IVF_FLAT
-        (SubIndexType::Flat, QuantizationType::Flat) => match element_type {
-            DataType::Float16 | DataType::Float32 | DataType::Float64 => {
-                IvfIndexBuilder::<FlatIndex, FlatQuantizer>::new_incremental(
-                    dataset.clone(),
-                    column.to_owned(),
-                    index_dir,
-                    params.metric_type,
-                    shuffler,
-                    (),
-                    frag_reuse_index,
-                    OptimizeOptions::append(),
-                )?
-                .with_ivf(ivf_model)
-                .with_quantizer(quantizer.try_into()?)
-                .with_progress(progress.clone())
-                .build()
-                .await?;
-            }
-            DataType::UInt8 => {
-                IvfIndexBuilder::<FlatIndex, FlatBinQuantizer>::new_incremental(
-                    dataset.clone(),
-                    column.to_owned(),
-                    index_dir,
-                    params.metric_type,
-                    shuffler,
-                    (),
-                    frag_reuse_index,
-                    OptimizeOptions::append(),
-                )?
-                .with_ivf(ivf_model)
-                .with_quantizer(quantizer.try_into()?)
-                .with_progress(progress.clone())
-                .build()
-                .await?;
-            }
-            _ => {
-                return Err(Error::index(format!(
-                    "Build Vector Index: invalid data type: {:?}",
-                    element_type
-                )));
-            }
-        },
+        (SubIndexType::Flat, QuantizationType::Flat) => {
+            IvfIndexBuilder::<FlatIndex, FlatQuantizer>::new_incremental(
+                dataset.clone(),
+                column.to_owned(),
+                index_dir,
+                params.metric_type,
+                shuffler,
+                (),
+                frag_reuse_index,
+                OptimizeOptions::append(),
+            )?
+            .with_ivf(ivf_model)
+            .with_quantizer(quantizer.try_into()?)
+            .with_progress(progress.clone())
+            .build()
+            .await?;
+        }
+        (SubIndexType::Flat, QuantizationType::FlatBin) => {
+            IvfIndexBuilder::<FlatIndex, FlatBinQuantizer>::new_incremental(
+                dataset.clone(),
+                column.to_owned(),
+                index_dir,
+                params.metric_type,
+                shuffler,
+                (),
+                frag_reuse_index,
+                OptimizeOptions::append(),
+            )?
+            .with_ivf(ivf_model)
+            .with_quantizer(quantizer.try_into()?)
+            .with_progress(progress.clone())
+            .build()
+            .await?;
+        }
         // IVF_PQ
         (SubIndexType::Flat, QuantizationType::Product) => {
             let mut builder = IvfIndexBuilder::<FlatIndex, ProductQuantizer>::new_incremental(
@@ -1154,6 +1353,23 @@ pub(crate) async fn build_vector_index_incremental(
                     .build()
                     .await?;
                 }
+                QuantizationType::FlatBin => {
+                    IvfIndexBuilder::<HNSW, FlatBinQuantizer>::new_incremental(
+                        dataset.clone(),
+                        column.to_owned(),
+                        index_dir,
+                        params.metric_type,
+                        shuffler,
+                        hnsw_params.clone(),
+                        frag_reuse_index,
+                        OptimizeOptions::append(),
+                    )?
+                    .with_ivf(ivf_model)
+                    .with_quantizer(quantizer.try_into()?)
+                    .with_progress(progress.clone())
+                    .build()
+                    .await?;
+                }
                 QuantizationType::Product => {
                     IvfIndexBuilder::<HNSW, ProductQuantizer>::new_incremental(
                         dataset.clone(),
@@ -1485,7 +1701,8 @@ pub async fn initialize_vector_index(
     let ivf_params = derive_ivf_params(ivf_model);
 
     let params = match (sub_index_type, quantization_type) {
-        (SubIndexType::Flat, QuantizationType::Flat) => {
+        (SubIndexType::Flat, QuantizationType::Flat)
+        | (SubIndexType::Flat, QuantizationType::FlatBin) => {
             VectorIndexParams::with_ivf_flat_params(metric_type, ivf_params)
         }
         (SubIndexType::Flat, QuantizationType::Product) => {
@@ -1506,7 +1723,7 @@ pub async fn initialize_vector_index(
         (SubIndexType::Hnsw, quantization_type) => {
             let hnsw_params = derive_hnsw_params(source_vector_index.as_ref());
             match quantization_type {
-                QuantizationType::Flat => {
+                QuantizationType::Flat | QuantizationType::FlatBin => {
                     VectorIndexParams::ivf_hnsw(metric_type, ivf_params, hnsw_params)
                 }
                 QuantizationType::Product => {
@@ -1554,6 +1771,10 @@ pub async fn initialize_vector_index(
     )
     .await?;
 
+    // Capture file sizes for the new vector index
+    let index_dir = target_dataset.indices_dir().child(new_uuid.to_string());
+    let files = list_index_files_with_sizes(&target_dataset.object_store, &index_dir).await?;
+
     let field = target_dataset.schema().field(column_name).ok_or_else(|| {
         Error::index(format!(
             "Column '{}' not found in target dataset",
@@ -1561,17 +1782,7 @@ pub async fn initialize_vector_index(
         ))
     })?;
 
-    let fragment_bitmap = if target_dataset.get_fragments().is_empty() {
-        Some(roaring::RoaringBitmap::new())
-    } else {
-        Some(
-            target_dataset
-                .get_fragments()
-                .iter()
-                .map(|f| f.id() as u32)
-                .collect(),
-        )
-    };
+    let fragment_bitmap = Some(target_dataset.fragment_bitmap.as_ref().clone());
 
     let new_idx = IndexMetadata {
         uuid: new_uuid,
@@ -1583,6 +1794,7 @@ pub async fn initialize_vector_index(
         index_version: source_index.index_version,
         created_at: Some(chrono::Utc::now()),
         base_id: None,
+        files: Some(files),
     };
 
     let transaction = Transaction::new(
@@ -1703,6 +1915,7 @@ fn derive_hnsw_params(source_index: &dyn VectorIndex) -> HnswBuildParams {
 mod tests {
     use super::*;
     use crate::dataset::Dataset;
+    use crate::index::DatasetIndexExt;
     use arrow_array::Array;
     use arrow_array::RecordBatch;
     use arrow_array::types::{Float32Type, Int32Type};
@@ -1710,7 +1923,6 @@ mod tests {
     use lance_core::utils::tempfile::TempStrDir;
     use lance_datagen::{BatchCount, RowCount, array};
     use lance_file::writer::FileWriterOptions;
-    use lance_index::DatasetIndexExt;
     use lance_index::metrics::NoOpMetricsCollector;
     use lance_linalg::distance::MetricType;
 
@@ -2158,6 +2370,7 @@ mod tests {
             dim,
             MetricType::L2,
             &ivf_params,
+            None,
             noop_progress(),
         )
         .await
@@ -2210,6 +2423,7 @@ mod tests {
             dim,
             MetricType::L2,
             &ivf_params,
+            None,
             noop_progress(),
         )
         .await
diff --git a/rust/lance/src/index/vector/builder.rs b/rust/lance/src/index/vector/builder.rs
index 14b134092cc..96bca9daef3 100644
--- a/rust/lance/src/index/vector/builder.rs
+++ b/rust/lance/src/index/vector/builder.rs
@@ -1,8 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use std::cmp::Ordering;
 use std::collections::HashSet;
-use std::future;
 use std::sync::Arc;
 use std::{collections::HashMap, pin::Pin};
 
@@ -28,12 +28,13 @@ use lance_core::datatypes::Schema;
 use lance_core::utils::tempfile::TempStdDir;
 use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu};
 use lance_core::{Error, ROW_ID_FIELD, Result};
-use lance_file::writer::FileWriter;
+use lance_encoding::version::LanceFileVersion;
+use lance_file::writer::{FileWriter, FileWriterOptions};
 use lance_index::frag_reuse::FragReuseIndex;
 use lance_index::metrics::NoOpMetricsCollector;
 use lance_index::optimize::OptimizeOptions;
 use lance_index::progress::{IndexBuildProgress, NoopIndexBuildProgress};
-use lance_index::vector::bq::storage::{RABIT_CODE_COLUMN, pack_codes, unpack_codes};
+use lance_index::vector::bq::storage::{RABIT_CODE_COLUMN, unpack_codes};
 use lance_index::vector::kmeans::KMeansParams;
 use lance_index::vector::pq::storage::transpose;
 use lance_index::vector::quantizer::{
@@ -77,8 +78,9 @@ use tracing::{Level, instrument, span};
 
 use crate::Dataset;
 use crate::dataset::ProjectionRequest;
+use crate::dataset::index::dataset_format_version;
 use crate::index::vector::ivf::v2::PartitionEntry;
-use crate::index::vector::utils::{infer_vector_dim, infer_vector_element_type};
+use crate::index::vector::utils::infer_vector_dim;
 
 use super::v2::IVFIndex;
 use super::{
@@ -86,26 +88,6 @@ use super::{
     utils::{self, get_vector_type},
 };
 
-/// Stably sort a RecordBatch by the ROW_ID column in ascending order.
-///
-/// If the batch has no ROW_ID column or has fewer than 2 rows, it is
-/// returned unchanged. When sorting, the relative order of rows with the
-/// same ROW_ID is preserved.
-fn stable_sort_batch_by_row_id(batch: &RecordBatch) -> Result<RecordBatch> {
-    if let Some(row_id_col) = batch.column_by_name(ROW_ID) {
-        let row_ids = row_id_col.as_primitive::<UInt64Type>();
-        if row_ids.len() > 1 {
-            let mut order: Vec<usize> = (0..row_ids.len()).collect();
-            // Vec::sort_by is stable, so equal ROW_IDs keep their
-            // original relative order.
-            order.sort_by(|&i, &j| row_ids.value(i).cmp(&row_ids.value(j)));
-            let indices = UInt32Array::from_iter_values(order.into_iter().map(|i| i as u32));
-            return Ok(batch.take(&indices)?);
-        }
-    }
-    Ok(batch.clone())
-}
-
 // the number of partitions to evaluate for reassigning
 const REASSIGN_RANGE: usize = 64;
 
@@ -149,6 +131,9 @@ pub struct IvfIndexBuilder<S: IvfSubIndex, Q: Quantization> {
     // whether to transpose codes when building storage
     transpose_codes: bool,
 
+    // lance file version for writing index files
+    format_version: LanceFileVersion,
+
     progress: Arc<dyn IndexBuildProgress>,
 }
 
@@ -170,6 +155,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
     ) -> Result<Self> {
         let temp_dir = TempStdDir::default();
         let temp_dir_path = Path::from_filesystem_path(&temp_dir)?;
+        let format_version = dataset_format_version(&dataset);
         Ok(Self {
             store: dataset.object_store().clone(),
             column,
@@ -192,6 +178,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             optimize_options: None,
             merged_num: 0,
             transpose_codes: true,
+            format_version,
             progress: Arc::new(NoopIndexBuildProgress),
         })
     }
@@ -235,6 +222,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
 
         let temp_dir = TempStdDir::default();
         let temp_dir_path = Path::from_filesystem_path(&temp_dir)?;
+        let format_version = dataset_format_version(&dataset);
         Ok(Self {
             store: dataset.object_store().clone(),
             column,
@@ -256,6 +244,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             optimize_options: None,
             merged_num: 0,
             transpose_codes: true,
+            format_version,
             progress: Arc::new(NoopIndexBuildProgress),
         })
     }
@@ -270,7 +259,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
         progress
             .stage_start("train_ivf", max_iters, "iterations")
             .await?;
-        self.with_ivf(self.load_or_build_ivf().await?);
+        self.with_ivf(self.load_or_build_ivf().boxed().await?);
         progress.stage_complete("train_ivf").await?;
 
         progress.stage_start("train_quantizer", None, "").await?;
@@ -279,21 +268,20 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
 
         // step 2. shuffle the dataset
         if self.shuffle_reader.is_none() {
-            progress.stage_start("shuffle", None, "batches").await?;
-            self.shuffle_dataset().await?;
+            let num_rows = self.num_rows_to_shuffle().await?;
+            progress.stage_start("shuffle", num_rows, "rows").await?;
+            self.shuffle_dataset().boxed().await?;
             progress.stage_complete("shuffle").await?;
         }
 
-        // step 3. build partitions
+        // step 3. build and merge partitions
         let num_partitions = self.ivf.as_ref().map(|ivf| ivf.num_partitions() as u64);
         progress
-            .stage_start("build_partitions", num_partitions, "partitions")
+            .stage_start("merge_partitions", num_partitions, "partitions")
             .await?;
         let build_idx_stream = self.build_partitions().boxed().await?;
-
-        // step 4. merge all partitions
         self.merge_partitions(build_idx_stream).await?;
-        progress.stage_complete("build_partitions").await?;
+        progress.stage_complete("merge_partitions").await?;
 
         Ok(self.merged_num)
     }
@@ -397,6 +385,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
                     dim,
                     self.distance_type,
                     ivf_params,
+                    self.fragment_filter.as_deref(),
                     self.progress.clone(),
                 )
                 .await
@@ -425,8 +414,13 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             "loading training data for quantizer. sample size: {}",
             sample_size_hint
         );
-        let training_data =
-            utils::maybe_sample_training_data(dataset, &self.column, sample_size_hint).await?;
+        let training_data = utils::maybe_sample_training_data(
+            dataset,
+            &self.column,
+            sample_size_hint,
+            self.fragment_filter.as_deref(),
+        )
+        .await?;
         info!(
             "Finished loading training data in {:02} seconds",
             start.elapsed().as_secs_f32()
@@ -506,6 +500,28 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
         )
     }
 
+    async fn num_rows_to_shuffle(&self) -> Result<Option<u64>> {
+        let Some(dataset) = self.dataset.as_ref() else {
+            return Ok(None);
+        };
+        match &self.fragment_filter {
+            Some(fragment_ids) => {
+                let fragments: Vec<_> = dataset
+                    .get_fragments()
+                    .into_iter()
+                    .filter(|f| fragment_ids.contains(&(f.id() as u32)))
+                    .collect();
+                let counts = futures::stream::iter(fragments)
+                    .map(|f| async move { f.count_rows(None).await })
+                    .buffer_unordered(16) // ref: Dataset::count_all_rows()
+                    .try_collect::<Vec<_>>()
+                    .await?;
+                Ok(Some(counts.iter().sum::<usize>() as u64))
+            }
+            None => Ok(Some(dataset.count_rows(None).await? as u64)),
+        }
+    }
+
     async fn shuffle_dataset(&mut self) -> Result<()> {
         let Some(dataset) = self.dataset.as_ref() else {
             return Err(Error::invalid_input("dataset not set before shuffling"));
@@ -748,53 +764,52 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             )
         };
 
-        let (assign_batches, merge_indices, partition_adjustment) = if num_indices_to_merge
-            .is_some()
-            || self.optimize_options.is_none()
-        {
-            no_partition_adjustment()
-        } else {
-            match Self::check_partition_adjustment(ivf, reader.as_ref(), &self.existing_indices)? {
-                Some(partition_adjustment) => match partition_adjustment {
-                    PartitionAdjustment::Split(partition) => {
-                        // Perform split and record the fact for downstream build/merge
-                        log::info!(
-                            "split partition {}, will merge all {} delta indices",
-                            partition,
-                            self.existing_indices.len()
-                        );
-                        let split_results = self.split_partition(partition, ivf).await?;
-                        let Some(ivf) = self.ivf.as_mut() else {
-                            return Err(Error::invalid_input(
-                                "IVF not set before building partitions",
-                            ));
-                        };
-                        ivf.centroids = Some(split_results.new_centroids);
-                        (
-                            split_results.assign_batches,
-                            Arc::new(self.existing_indices.clone()),
-                            Some(partition_adjustment),
-                        )
-                    }
-                    PartitionAdjustment::Join(partition) => {
-                        log::info!("join partition {}", partition);
-                        let results = self.join_partition(partition, ivf).await?;
-                        let Some(ivf) = self.ivf.as_mut() else {
-                            return Err(Error::invalid_input(
-                                "IVF model not set before joining partition",
-                            ));
-                        };
-                        ivf.centroids = Some(results.new_centroids);
-                        (
-                            results.assign_batches,
-                            Arc::new(self.existing_indices.clone()),
-                            Some(partition_adjustment),
-                        )
+        let (assign_batches, merge_indices, partition_adjustment) =
+            if num_indices_to_merge.is_some() || self.optimize_options.is_none() {
+                no_partition_adjustment()
+            } else {
+                let (split_partitions, join_partition) =
+                    Self::check_partition_adjustment(ivf, reader.as_ref(), &self.existing_indices)?;
+                if !split_partitions.is_empty() {
+                    log::info!(
+                        "split partitions {:?}, will merge all {} delta indices",
+                        split_partitions,
+                        self.existing_indices.len()
+                    );
+                    let split_results = self.split_partitions(&split_partitions, ivf).await?;
+                    let actual_split_partitions = split_results.split_partitions.clone();
+                    let Some(ivf) = self.ivf.as_mut() else {
+                        return Err(Error::invalid_input(
+                            "IVF not set before building partitions",
+                        ));
+                    };
+                    ivf.centroids = Some(split_results.new_centroids);
+                    (
+                        split_results.assign_batches,
+                        Arc::new(self.existing_indices.clone()),
+                        Some(PartitionAdjustment::Split(actual_split_partitions)),
+                    )
+                } else {
+                    match join_partition {
+                        Some(partition) => {
+                            log::info!("join partition {}", partition);
+                            let results = self.join_partition(partition, ivf).await?;
+                            let Some(ivf) = self.ivf.as_mut() else {
+                                return Err(Error::invalid_input(
+                                    "IVF model not set before joining partition",
+                                ));
+                            };
+                            ivf.centroids = Some(results.new_centroids);
+                            (
+                                results.assign_batches,
+                                Arc::new(self.existing_indices.clone()),
+                                Some(PartitionAdjustment::Join(partition)),
+                            )
+                        }
+                        None => no_partition_adjustment(),
                     }
-                },
-                None => no_partition_adjustment(),
-            }
-        };
+                }
+            };
         self.merged_num = merge_indices.len();
         log::info!(
             "merge {}/{} delta indices",
@@ -817,11 +832,15 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
                     let sub_index_params = sub_index_params.clone();
                     let column = column.clone();
                     let frag_reuse_index = frag_reuse_index.clone();
-                    let skip_existing_batches =
-                        partition_adjustment == Some(PartitionAdjustment::Split(partition));
-                    let partition = match partition_adjustment {
+                    let skip_existing_batches = match &partition_adjustment {
+                        Some(PartitionAdjustment::Split(split_partitions)) => {
+                            split_partitions.binary_search(&partition).is_ok()
+                        }
+                        _ => false,
+                    };
+                    let partition = match &partition_adjustment {
                         Some(PartitionAdjustment::Join(joined_partition))
-                            if partition >= joined_partition =>
+                            if partition >= *joined_partition =>
                         {
                             partition + 1
                         }
@@ -964,14 +983,6 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
                 _ => {}
             }
 
-            // Normalize each batch for this partition to be stably sorted by ROW_ID.
-            for batch in part_batches.iter_mut() {
-                if batch.num_rows() == 0 {
-                    continue;
-                }
-                *batch = stable_sort_batch_by_row_id(batch)?;
-            }
-
             batches.extend(part_batches);
         }
 
@@ -995,9 +1006,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
                     .get(LOSS_METADATA_KEY)
                     .map(|s| s.parse::<f64>().unwrap_or(0.0))
                     .unwrap_or(0.0);
-                let batch = batch.drop_column(PART_ID_COLUMN)?;
-                let batch = stable_sort_batch_by_row_id(&batch)?;
-                batches.push(batch);
+                batches.push(batch.drop_column(PART_ID_COLUMN)?);
             }
         }
 
@@ -1015,25 +1024,35 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             ));
         };
 
-        let is_pq = Q::quantization_type() == QuantizationType::Product;
-        let is_rq = Q::quantization_type() == QuantizationType::Rabit;
+        let quantization_type = Q::quantization_type();
+        let is_pq = quantization_type == QuantizationType::Product;
+        let is_rq = quantization_type == QuantizationType::Rabit;
+        let is_flat = quantization_type == QuantizationType::Flat;
 
         // prepare the final writers
         let storage_path = self.index_dir.child(INDEX_AUXILIARY_FILE_NAME);
         let index_path = self.index_dir.child(INDEX_FILE_NAME);
 
-        let mut fields = vec![ROW_ID_FIELD.clone(), quantizer.field()];
-        fields.extend(quantizer.extra_fields());
-        let storage_schema: Schema = (&arrow_schema::Schema::new(fields)).try_into()?;
-        let mut storage_writer = FileWriter::try_new(
-            self.store.create(&storage_path).await?,
-            storage_schema.clone(),
-            Default::default(),
-        )?;
+        let writer_options = FileWriterOptions {
+            format_version: Some(self.format_version),
+            ..Default::default()
+        };
+        let mut storage_writer = if is_flat {
+            None
+        } else {
+            let mut fields = vec![ROW_ID_FIELD.clone(), quantizer.field()];
+            fields.extend(quantizer.extra_fields());
+            let storage_schema: Schema = (&arrow_schema::Schema::new(fields)).try_into()?;
+            Some(FileWriter::try_new(
+                self.store.create(&storage_path).await?,
+                storage_schema,
+                writer_options.clone(),
+            )?)
+        };
         let mut index_writer = FileWriter::try_new(
             self.store.create(&index_path).await?,
             S::schema().as_ref().try_into()?,
-            Default::default(),
+            writer_options.clone(),
         )?;
 
         // maintain the IVF partitions
@@ -1047,7 +1066,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
         log::info!("merging {} partitions", ivf.num_partitions());
         while let Some(part) = build_stream.try_next().await? {
             part_id += 1;
-            progress.stage_progress("build_partitions", part_id).await?;
+            progress.stage_progress("merge_partitions", part_id).await?;
             let Some((storage, index, loss)) = part else {
                 log::warn!("partition {} is empty, skipping", part_id);
 
@@ -1062,79 +1081,55 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             if storage.len() == 0 {
                 storage_ivf.add_partition(0);
             } else {
-                let batches = storage.to_batches()?.collect::<Vec<_>>();
-                let mut batch =
-                    arrow::compute::concat_batches(&batches[0].schema(), batches.iter())?;
-
-                if is_pq && batch.column_by_name(PQ_CODE_COLUMN).is_some() {
-                    // The PQ storage keeps codes in a transposed layout (bytes grouped
-                    // across all rows). Convert them back to per-row layout so that a
-                    // stable ROW_ID sort moves PQ_CODE_COLUMN together with ROW_ID.
-                    let codes_fsl = batch
-                        .column_by_name(PQ_CODE_COLUMN)
-                        .unwrap()
-                        .as_fixed_size_list();
-                    let num_rows = batch.num_rows();
-                    let bytes_per_code = codes_fsl.value_length() as usize;
-                    let codes = codes_fsl.values().as_primitive::<datatypes::UInt8Type>();
-                    let original_codes = transpose(codes, bytes_per_code, num_rows);
-                    let original_fsl = Arc::new(FixedSizeListArray::try_new_from_values(
-                        original_codes,
-                        bytes_per_code as i32,
-                    )?);
-                    batch = batch.replace_column_by_name(PQ_CODE_COLUMN, original_fsl)?;
-                }
-
-                if is_rq && batch.column_by_name(RABIT_CODE_COLUMN).is_some() {
-                    // RQ storage batches reaching merge_partitions always come
-                    // from RabitQuantizationStorage, which canonicalizes codes
-                    // into packed layout in try_from_batch/remap. Materialize
-                    // row-major bytes so row-wise sort operates on per-row codes.
-                    let codes_fsl = batch
-                        .column_by_name(RABIT_CODE_COLUMN)
-                        .unwrap()
-                        .as_fixed_size_list();
-                    let unpacked = Arc::new(unpack_codes(codes_fsl));
-                    batch = batch.replace_column_by_name(RABIT_CODE_COLUMN, unpacked)?;
-                }
+                for mut batch in storage.to_batches()? {
+                    if is_pq
+                        && !self.transpose_codes
+                        && batch.num_rows() > 0
+                        && batch.column_by_name(PQ_CODE_COLUMN).is_some()
+                    {
+                        let codes_fsl = batch
+                            .column_by_name(PQ_CODE_COLUMN)
+                            .unwrap()
+                            .as_fixed_size_list();
+                        let num_rows = batch.num_rows();
+                        let bytes_per_code = codes_fsl.value_length() as usize;
+                        let codes = codes_fsl.values().as_primitive::<datatypes::UInt8Type>();
+                        let original_codes = transpose(codes, bytes_per_code, num_rows);
+                        let original_fsl = Arc::new(FixedSizeListArray::try_new_from_values(
+                            original_codes,
+                            bytes_per_code as i32,
+                        )?);
+                        batch = batch.replace_column_by_name(PQ_CODE_COLUMN, original_fsl)?;
+                    }
 
-                // Enforce a stable ROW_ID ordering for all auxiliary batches so that the
-                // PQ code column moves together with ROW_ID.
-                batch = stable_sort_batch_by_row_id(&batch)?;
-
-                // For PQ storages, optionally convert codes back to transposed layout
-                // in the unified auxiliary file. This keeps final PQ storage column-major
-                // when `transpose_pq_codes` is enabled.
-                if is_pq && self.transpose_codes && batch.column_by_name(PQ_CODE_COLUMN).is_some() {
-                    let codes_fsl = batch
-                        .column_by_name(PQ_CODE_COLUMN)
-                        .unwrap()
-                        .as_fixed_size_list();
-                    let num_rows = batch.num_rows();
-                    let bytes_per_code = codes_fsl.value_length() as usize;
-                    let codes = codes_fsl.values().as_primitive::<datatypes::UInt8Type>();
-                    let transposed_codes = transpose(codes, num_rows, bytes_per_code);
-                    let transposed_fsl = Arc::new(FixedSizeListArray::try_new_from_values(
-                        transposed_codes,
-                        bytes_per_code as i32,
-                    )?);
-                    batch = batch.replace_column_by_name(PQ_CODE_COLUMN, transposed_fsl)?;
-                }
+                    if is_rq
+                        && !self.transpose_codes
+                        && batch.num_rows() > 0
+                        && batch.column_by_name(RABIT_CODE_COLUMN).is_some()
+                    {
+                        let codes_fsl = batch
+                            .column_by_name(RABIT_CODE_COLUMN)
+                            .unwrap()
+                            .as_fixed_size_list();
+                        let unpacked = Arc::new(unpack_codes(codes_fsl));
+                        batch = batch.replace_column_by_name(RABIT_CODE_COLUMN, unpacked)?;
+                    }
 
-                if is_rq
-                    && self.transpose_codes
-                    && batch.column_by_name(RABIT_CODE_COLUMN).is_some()
-                {
-                    let codes_fsl = batch
-                        .column_by_name(RABIT_CODE_COLUMN)
-                        .unwrap()
-                        .as_fixed_size_list();
-                    let packed = Arc::new(pack_codes(codes_fsl));
-                    batch = batch.replace_column_by_name(RABIT_CODE_COLUMN, packed)?;
+                    if storage_writer.is_none() {
+                        let storage_schema: Schema = batch.schema_ref().as_ref().try_into()?;
+                        storage_writer = Some(FileWriter::try_new(
+                            self.store.create(&storage_path).await?,
+                            storage_schema,
+                            writer_options.clone(),
+                        )?);
+                    }
+                    storage_writer
+                        .as_mut()
+                        .expect("storage writer must be initialized before write")
+                        .write_batch(&batch)
+                        .await?;
+                    storage_ivf.add_partition(batch.num_rows() as u32);
                 }
-
-                storage_writer.write_batch(&batch).await?;
-                storage_ivf.add_partition(batch.num_rows() as u32);
             }
 
             let index_batch = index.to_batch()?;
@@ -1168,14 +1163,45 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             }
         }
 
+        if storage_writer.is_none() {
+            let Some(centroids) = ivf.centroids.as_ref() else {
+                return Err(Error::invalid_input(
+                    "flat storage writer could not infer schema from empty partitions without IVF centroids",
+                ));
+            };
+            let flat_schema = arrow_schema::Schema::new(vec![
+                ROW_ID_FIELD.as_ref().clone(),
+                arrow_schema::Field::new(
+                    lance_index::vector::flat::storage::FLAT_COLUMN,
+                    DataType::FixedSizeList(
+                        Arc::new(arrow_schema::Field::new(
+                            "item",
+                            centroids.value_type(),
+                            true,
+                        )),
+                        centroids.value_length(),
+                    ),
+                    true,
+                ),
+            ]);
+            let storage_schema: Schema = (&flat_schema).try_into()?;
+            storage_writer = Some(FileWriter::try_new(
+                self.store.create(&storage_path).await?,
+                storage_schema,
+                writer_options.clone(),
+            )?);
+        }
+
+        let storage_writer = storage_writer
+            .as_mut()
+            .expect("storage writer must be initialized before final metadata write");
         let storage_ivf_pb = pb::Ivf::try_from(&storage_ivf)?;
         storage_writer.add_schema_metadata(DISTANCE_TYPE_KEY, self.distance_type.to_string());
         let ivf_buffer_pos = storage_writer
             .add_global_buffer(storage_ivf_pb.encode_to_vec().into())
             .await?;
         storage_writer.add_schema_metadata(IVF_METADATA_KEY, ivf_buffer_pos.to_string());
-        let quant_type = Q::quantization_type();
-        let transposed = match quant_type {
+        let transposed = match quantization_type {
             QuantizationType::Product | QuantizationType::Rabit => self.transpose_codes,
             _ => false,
         };
@@ -1312,25 +1338,21 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
         ivf: &IvfModel,
         reader: &dyn ShuffleReader,
         existing_indices: &[Arc<dyn VectorIndex>],
-    ) -> Result<Option<PartitionAdjustment>> {
+    ) -> Result<(Vec<usize>, Option<usize>)> {
         let index_type = IndexType::try_from(
             index_type_string(S::name().try_into()?, Q::quantization_type()).as_str(),
         )?;
 
-        let mut split_partition = None;
+        let mut split_partitions = Vec::new();
         let mut join_partition = None;
-        let mut max_partition_size = 0;
         let mut min_partition_size = usize::MAX;
         for partition in 0..ivf.num_partitions() {
             let mut num_rows = reader.partition_size(partition)?;
             for index in existing_indices.iter() {
                 num_rows += index.partition_size(partition);
             }
-            if num_rows > max_partition_size
-                && num_rows > MAX_PARTITION_SIZE_FACTOR * index_type.target_partition_size()
-            {
-                max_partition_size = num_rows;
-                split_partition = Some(partition);
+            if num_rows > MAX_PARTITION_SIZE_FACTOR * index_type.target_partition_size() {
+                split_partitions.push(partition);
             }
             if ivf.num_partitions() > 1
                 && num_rows < min_partition_size
@@ -1341,44 +1363,36 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             }
         }
 
-        if let Some(partition) = split_partition {
-            Ok(Some(PartitionAdjustment::Split(partition)))
-        } else if let Some(partition) = join_partition {
-            Ok(Some(PartitionAdjustment::Join(partition)))
-        } else {
-            Ok(None)
-        }
+        Ok((split_partitions, join_partition))
     }
 
-    // split this partition,
-    // 1. take raw vectors by row ids in this partition
-    // 2. run KMeans with k=2 to get 2 new centroids
-    // 3. reassign the vectors to the 2 new partitions
-    async fn split_partition(&self, part_idx: usize, ivf: &IvfModel) -> Result<AssignResult> {
-        // take the raw vectors from dataset
-        let Some((row_ids, vectors)) = self.load_partition_raw_vectors(part_idx).await? else {
-            return Ok(AssignResult {
-                assign_batches: vec![None; ivf.num_partitions()],
-                new_centroids: ivf.centroids_array().unwrap().clone(),
-            });
+    async fn split_partitions(
+        &self,
+        split_partitions: &[usize],
+        ivf: &IvfModel,
+    ) -> Result<AssignResult> {
+        let Some(dataset) = self.dataset.as_ref() else {
+            return Err(Error::invalid_input(
+                "dataset not set before split partition",
+            ));
         };
 
-        let element_type = infer_vector_element_type(vectors.data_type())?;
+        let (_, element_type) = get_vector_type(dataset.schema(), &self.column)?;
         match element_type {
             DataType::Float16 => {
-                self.split_partition_impl::<Float16Type>(part_idx, ivf, &row_ids, &vectors)
+                self.split_partitions_impl::<Float16Type>(split_partitions, ivf)
                     .await
             }
             DataType::Float32 => {
-                self.split_partition_impl::<Float32Type>(part_idx, ivf, &row_ids, &vectors)
+                self.split_partitions_impl::<Float32Type>(split_partitions, ivf)
                     .await
             }
             DataType::Float64 => {
-                self.split_partition_impl::<Float64Type>(part_idx, ivf, &row_ids, &vectors)
+                self.split_partitions_impl::<Float64Type>(split_partitions, ivf)
                     .await
             }
             DataType::UInt8 => {
-                self.split_partition_impl::<UInt8Type>(part_idx, ivf, &row_ids, &vectors)
+                self.split_partitions_impl::<UInt8Type>(split_partitions, ivf)
                     .await
             }
             dt => Err(Error::invalid_input(format!(
@@ -1388,26 +1402,101 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
         }
     }
 
-    async fn split_partition_impl<T: ArrowPrimitiveType>(
+    async fn split_partitions_impl<T: ArrowPrimitiveType>(
         &self,
-        part_idx: usize,
+        split_partitions: &[usize],
         ivf: &IvfModel,
-        row_ids: &UInt64Array,
-        vectors: &FixedSizeListArray,
     ) -> Result<AssignResult>
     where
         T::Native: Dot + L2 + Normalize,
         PrimitiveArray<T>: From<Vec<T::Native>>,
     {
         let centroids = ivf.centroids_array().unwrap();
-        let mut new_centroids: Vec<ArrayRef> = Vec::with_capacity(ivf.num_partitions() + 1);
+        let mut new_centroids: Vec<ArrayRef> =
+            Vec::with_capacity(ivf.num_partitions() + split_partitions.len());
         new_centroids.extend(centroids.iter().map(|vec| vec.unwrap()));
+        let split_plans = stream::iter(split_partitions.iter().copied().enumerate())
+            .map(|(split_order, part_idx)| async move {
+                let centroid2_part_idx = ivf.num_partitions() + split_order;
+                self.build_split_plan::<T>(part_idx, centroid2_part_idx, ivf)
+                    .await
+            })
+            .buffered(get_num_compute_intensive_cpus())
+            .try_collect::<Vec<_>>()
+            .await?;
+        let mut split_plans = split_plans.into_iter().flatten().collect::<Vec<_>>();
+        split_plans.sort_by_key(|plan| plan.part_idx);
+        Self::finalize_split_plans(&mut split_plans, ivf.num_partitions());
+
+        if split_plans.is_empty() {
+            return Ok(AssignResult {
+                assign_batches: vec![None; ivf.num_partitions()],
+                new_centroids: centroids.clone(),
+                split_partitions: Vec::new(),
+            });
+        }
+
+        for split_plan in &split_plans {
+            new_centroids[split_plan.part_idx] = split_plan.centroid1.clone();
+        }
+        new_centroids.extend(split_plans.iter().map(|plan| plan.centroid2.clone()));
+
+        let split_partition_set =
+            HashSet::<usize>::from_iter(split_plans.iter().map(|plan| plan.part_idx));
+        let new_centroids = new_centroids
+            .iter()
+            .map(|vec| vec.as_ref())
+            .collect::<Vec<_>>();
+        let new_centroids = arrow::compute::concat(&new_centroids)?;
+        let new_centroids =
+            FixedSizeListArray::try_new_from_values(new_centroids, centroids.value_length())?;
+        let mut assign_ops = vec![Vec::new(); new_centroids.len()];
+
+        for split_plan in &split_plans {
+            for (target_idx, op) in &split_plan.original_assign_ops {
+                assign_ops[*target_idx].push(op.clone());
+            }
+        }
+
+        for candidate_move in self
+            .collect_candidate_moves::<T>(&split_plans, &split_partition_set, ivf)
+            .await?
+        {
+            assign_ops[candidate_move.source_part_idx]
+                .push(AssignOp::Remove(candidate_move.row_id));
+            assign_ops[candidate_move.dest_part_idx].push(AssignOp::Add((
+                candidate_move.row_id,
+                candidate_move.vector,
+            )));
+        }
+
+        let assign_batches = self.build_assign_batch::<T>(&new_centroids, &assign_ops)?;
+
+        Ok(AssignResult {
+            assign_batches,
+            new_centroids,
+            split_partitions: split_plans.iter().map(|plan| plan.part_idx).collect(),
+        })
+    }
+
+    async fn build_split_plan<T: ArrowPrimitiveType>(
+        &self,
+        part_idx: usize,
+        centroid2_part_idx: usize,
+        ivf: &IvfModel,
+    ) -> Result<Option<SplitPlan>>
+    where
+        T::Native: Dot + L2 + Normalize,
+        PrimitiveArray<T>: From<Vec<T::Native>>,
+    {
+        let Some((row_ids, vectors)) = self.load_partition_raw_vectors(part_idx).await? else {
+            return Ok(None);
+        };
 
         let dimension = infer_vector_dim(vectors.data_type())?;
-        // train kmeans to get 2 new centroids
         let (normalized_dist_type, normalized_vectors) = match self.distance_type {
             DistanceType::Cosine => {
-                let vectors = normalize_fsl(vectors)?;
+                let vectors = normalize_fsl(&vectors)?;
                 (DistanceType::L2, vectors)
             }
             _ => (self.distance_type, vectors.clone()),
@@ -1420,121 +1509,132 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             2,
             256,
         )?;
-        // the original centroid
+
         let c0 = ivf
             .centroid(part_idx)
             .ok_or(Error::invalid_input("original centroid not found"))?;
-        // the 2 new centroids
-        let c1 = kmeans.centroids.slice(0, dimension);
-        let c2 = kmeans.centroids.slice(dimension, dimension);
-        // replace the original centroid with the first new one
-        new_centroids[part_idx] = c1.clone();
-        // append the second new one
-        new_centroids.push(c2.clone());
-        let centroid1_part_idx = part_idx;
-        let centroid2_part_idx = new_centroids.len() - 1;
-
-        let new_centroids = new_centroids
-            .iter()
-            .map(|vec| vec.as_ref())
-            .collect::<Vec<_>>();
-        let new_centroids = arrow::compute::concat(&new_centroids)?;
-
-        // get top REASSIGN_RANGE centroids from c0
+        let centroid1 = kmeans.centroids.slice(0, dimension);
+        let centroid2 = kmeans.centroids.slice(dimension, dimension);
         let (reassign_part_ids, reassign_part_centroids) =
             self.select_reassign_candidates(ivf, part_idx, &c0)?;
 
-        // compute the distance between the vectors and the 3 centroids (original one and the 2 new ones)
-        let d0 = self.distance_type.arrow_batch_func()(&c0, vectors)?;
-        let d1 = self.distance_type.arrow_batch_func()(&c1, vectors)?;
-        let d2 = self.distance_type.arrow_batch_func()(&c2, vectors)?;
-        let d0 = d0.values();
-        let d1 = d1.values();
-        let d2 = d2.values();
-
-        let mut assign_ops = vec![Vec::new(); ivf.num_partitions() + 1];
-        // assign the vectors in the original partition
-        self.assign_vectors::<T>(
+        let d0 = self.distance_type.arrow_batch_func()(&c0, &vectors)?;
+        let d1 = self.distance_type.arrow_batch_func()(centroid1.as_ref(), &vectors)?;
+        let d2 = self.distance_type.arrow_batch_func()(centroid2.as_ref(), &vectors)?;
+        let mut original_assign_ops = Vec::with_capacity(row_ids.len());
+        Self::assign_vectors_impl::<T, _>(
+            self.distance_type,
+            part_idx,
             part_idx,
-            centroid1_part_idx,
             centroid2_part_idx,
-            row_ids,
-            vectors,
-            d0,
-            d1,
-            d2,
+            &row_ids,
+            &vectors,
+            d0.values(),
+            d1.values(),
+            d2.values(),
             &reassign_part_ids,
             &reassign_part_centroids,
             true,
-            &mut assign_ops,
+            |idx, op| original_assign_ops.push((idx, op)),
         )?;
-        // assign the vectors in the reassigned partitions
-        let reassign_targets = reassign_part_ids
-            .values()
-            .iter()
-            .copied()
-            .enumerate()
-            .collect::<Vec<_>>();
-        if !reassign_targets.is_empty() {
-            let builder = self;
-            let distance_type = self.distance_type;
-            let reassign_part_ids_clone = reassign_part_ids.clone();
-            let reassign_part_centroids_clone = reassign_part_centroids.clone();
-            stream::iter(
-                reassign_targets
-                    .into_iter()
-                    .map(move |(candidate_idx, part_id)| {
-                        let builder = builder;
-                        let reassign_part_ids = reassign_part_ids_clone.clone();
-                        let reassign_part_centroids = reassign_part_centroids_clone.clone();
-                        let centroid1 = c1.clone();
-                        let centroid2 = c2.clone();
-                        async move {
-                            let part_idx = part_id as usize;
-                            let Some((row_ids, vectors)) =
-                                builder.load_partition_raw_vectors(part_idx).await?
-                            else {
-                                // all vectors in this partition have been deleted
-                                return Ok::<Vec<(usize, AssignOp)>, Error>(Vec::new());
-                            };
-                            let ops = spawn_cpu(move || {
-                                Self::compute_reassign_assign_ops::<T>(
-                                    distance_type,
-                                    part_idx,
-                                    candidate_idx,
-                                    centroid1_part_idx,
-                                    centroid2_part_idx,
-                                    &row_ids,
-                                    &vectors,
-                                    centroid1,
-                                    centroid2,
-                                    &reassign_part_ids,
-                                    &reassign_part_centroids,
-                                )
-                            })
-                            .await?;
-                            Ok(ops)
-                        }
-                    }),
-            )
-            .buffered(get_num_compute_intensive_cpus())
-            .try_for_each(|ops| {
-                for (target_idx, op) in ops {
-                    assign_ops[target_idx].push(op);
+
+        Ok(Some(SplitPlan {
+            part_idx,
+            centroid2_part_idx,
+            centroid1,
+            centroid2,
+            reassign_part_ids,
+            original_assign_ops,
+        }))
+    }
+
+    async fn collect_candidate_moves<T: ArrowPrimitiveType>(
+        &self,
+        split_plans: &[SplitPlan],
+        split_partition_set: &HashSet<usize>,
+        ivf: &IvfModel,
+    ) -> Result<Vec<CandidateMove>>
+    where
+        T::Native: Dot + L2 + Normalize,
+        PrimitiveArray<T>: From<Vec<T::Native>>,
+    {
+        let mut candidate_partitions = HashMap::<usize, Vec<CandidateRequest>>::new();
+        for split_plan in split_plans {
+            for part_id in split_plan.reassign_part_ids.values().iter().copied() {
+                let part_idx = part_id as usize;
+                if split_partition_set.contains(&part_idx) {
+                    continue;
                 }
-                future::ready(Ok(()))
+                candidate_partitions
+                    .entry(part_idx)
+                    .or_default()
+                    .push(CandidateRequest {
+                        centroid1_part_idx: split_plan.part_idx,
+                        centroid2_part_idx: split_plan.centroid2_part_idx,
+                        centroid1: split_plan.centroid1.clone(),
+                        centroid2: split_plan.centroid2.clone(),
+                    });
+            }
+        }
+
+        let candidate_moves = stream::iter(candidate_partitions.into_iter())
+            .map(|(part_idx, requests)| async move {
+                let Some((row_ids, vectors)) = self.load_partition_raw_vectors(part_idx).await?
+                else {
+                    return Ok::<Vec<CandidateMove>, Error>(Vec::new());
+                };
+
+                let candidate_centroid = ivf.centroid(part_idx).ok_or(Error::invalid_input(
+                    format!("candidate centroid not found for partition {part_idx}"),
+                ))?;
+                let baseline_dists =
+                    self.distance_type.arrow_batch_func()(candidate_centroid.as_ref(), &vectors)?;
+                let mut best_moves = vec![None; row_ids.len()];
+                for request in requests {
+                    let d1 = self.distance_type.arrow_batch_func()(
+                        request.centroid1.as_ref(),
+                        &vectors,
+                    )?;
+                    let d2 = self.distance_type.arrow_batch_func()(
+                        request.centroid2.as_ref(),
+                        &vectors,
+                    )?;
+                    Self::update_best_candidate_moves::<T>(
+                        request.centroid1_part_idx,
+                        request.centroid2_part_idx,
+                        &row_ids,
+                        &vectors,
+                        baseline_dists.values(),
+                        d1.values(),
+                        d2.values(),
+                        &mut best_moves,
+                        part_idx,
+                    );
+                }
+
+                Ok(best_moves.into_iter().flatten().collect::<Vec<_>>())
             })
+            .buffered(get_num_compute_intensive_cpus())
+            .try_collect::<Vec<_>>()
             .await?;
-        }
 
-        let new_centroids =
-            FixedSizeListArray::try_new_from_values(new_centroids, dimension as i32)?;
-        let assign_batches = self.build_assign_batch::<T>(&new_centroids, &assign_ops)?;
+        Ok(candidate_moves.into_iter().flatten().collect())
+    }
 
-        Ok(AssignResult {
-            assign_batches,
-            new_centroids,
-        })
+    fn finalize_split_plans(split_plans: &mut [SplitPlan], base_num_partitions: usize) {
+        for (split_order, split_plan) in split_plans.iter_mut().enumerate() {
+            let actual_centroid2_part_idx = base_num_partitions + split_order;
+            if split_plan.centroid2_part_idx == actual_centroid2_part_idx {
+                continue;
+            }
+            let placeholder_centroid2_part_idx = split_plan.centroid2_part_idx;
+            for (target_idx, _) in &mut split_plan.original_assign_ops {
+                if *target_idx == placeholder_centroid2_part_idx {
+                    *target_idx = actual_centroid2_part_idx;
+                }
+            }
+            split_plan.centroid2_part_idx = actual_centroid2_part_idx;
+        }
     }
 
     // join the given partition:
@@ -1563,6 +1663,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             return Ok(AssignResult {
                 assign_batches: vec![None; ivf.num_partitions() - 1],
                 new_centroids,
+                split_partitions: Vec::new(),
             });
         };
 
@@ -1666,6 +1767,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
         Ok(AssignResult {
             assign_batches,
             new_centroids,
+            split_partitions: Vec::new(),
         })
     }
 
@@ -1834,42 +1936,6 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
     ) -> Result<(UInt32Array, FixedSizeListArray)> {
         select_reassign_candidates_impl(self.distance_type, ivf, part_idx, c0)
     }
-    // assign the vectors of original partition
-    #[allow(clippy::too_many_arguments)]
-    fn assign_vectors<T: ArrowPrimitiveType>(
-        &self,
-        part_idx: usize,
-        centroid1_part_idx: usize,
-        centroid2_part_idx: usize,
-        row_ids: &UInt64Array,
-        vectors: &FixedSizeListArray,
-        d0: &[f32],
-        d1: &[f32],
-        d2: &[f32],
-        reassign_part_ids: &UInt32Array,
-        reassign_part_centroids: &FixedSizeListArray,
-        // the assign ops for each partition
-        // the length must be `old_num_partitions + 1`
-        deleted_original_partition: bool,
-        assign_ops: &mut [Vec<AssignOp>],
-    ) -> Result<()> {
-        Self::assign_vectors_impl::<T, _>(
-            self.distance_type,
-            part_idx,
-            centroid1_part_idx,
-            centroid2_part_idx,
-            row_ids,
-            vectors,
-            d0,
-            d1,
-            d2,
-            reassign_part_ids,
-            reassign_part_centroids,
-            deleted_original_partition,
-            |idx, op| assign_ops[idx].push(op),
-        )
-    }
-
     #[allow(clippy::too_many_arguments)]
     fn assign_vectors_impl<T: ArrowPrimitiveType, F: FnMut(usize, AssignOp)>(
         distance_type: DistanceType,
@@ -1942,50 +2008,54 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
     }
 
     #[allow(clippy::too_many_arguments)]
-    fn compute_reassign_assign_ops<T: ArrowPrimitiveType>(
-        distance_type: DistanceType,
-        part_idx: usize,
-        candidate_idx: usize,
+    fn update_best_candidate_moves<T: ArrowPrimitiveType>(
         centroid1_part_idx: usize,
         centroid2_part_idx: usize,
         row_ids: &UInt64Array,
         vectors: &FixedSizeListArray,
-        centroid1: ArrayRef,
-        centroid2: ArrayRef,
-        reassign_part_ids: &UInt32Array,
-        reassign_part_centroids: &FixedSizeListArray,
-    ) -> Result<Vec<(usize, AssignOp)>>
-    where
+        baseline_dists: &[f32],
+        centroid1_dists: &[f32],
+        centroid2_dists: &[f32],
+        best_moves: &mut [Option<CandidateMove>],
+        part_idx: usize,
+    ) where
         T::Native: Dot + L2 + Normalize,
         PrimitiveArray<T>: From<Vec<T::Native>>,
     {
-        let d0 = distance_type.arrow_batch_func()(
-            reassign_part_centroids.value(candidate_idx).as_ref(),
-            vectors,
-        )?;
-        let d1 = distance_type.arrow_batch_func()(centroid1.as_ref(), vectors)?;
-        let d2 = distance_type.arrow_batch_func()(centroid2.as_ref(), vectors)?;
-        let d0 = d0.values();
-        let d1 = d1.values();
-        let d2 = d2.values();
+        for (i, &row_id) in row_ids.values().iter().enumerate() {
+            if baseline_dists[i] <= centroid1_dists[i] && baseline_dists[i] <= centroid2_dists[i] {
+                continue;
+            }
+            let (dest_part_idx, dest_distance) = if centroid1_dists[i] <= centroid2_dists[i] {
+                (centroid1_part_idx, centroid1_dists[i])
+            } else {
+                (centroid2_part_idx, centroid2_dists[i])
+            };
+            let candidate_move = CandidateMove {
+                row_id,
+                source_part_idx: part_idx,
+                dest_part_idx,
+                dest_distance,
+                vector: vectors.value(i),
+            };
+            match best_moves[i].as_mut() {
+                Some(best_move) if Self::is_better_candidate_move(&candidate_move, best_move) => {
+                    *best_move = candidate_move;
+                }
+                None => {
+                    best_moves[i] = Some(candidate_move);
+                }
+                _ => {}
+            }
+        }
+    }
 
-        let mut ops = Vec::new();
-        Self::assign_vectors_impl::<T, _>(
-            distance_type,
-            part_idx,
-            centroid1_part_idx,
-            centroid2_part_idx,
-            row_ids,
-            vectors,
-            d0,
-            d1,
-            d2,
-            reassign_part_ids,
-            reassign_part_centroids,
-            false,
-            |idx, op| ops.push((idx, op)),
-        )?;
-        Ok(ops)
+    fn is_better_candidate_move(candidate: &CandidateMove, current: &CandidateMove) -> bool {
+        match candidate.dest_distance.total_cmp(&current.dest_distance) {
+            Ordering::Less => true,
+            Ordering::Equal => candidate.dest_part_idx < current.dest_part_idx,
+            Ordering::Greater => false,
+        }
     }
 
     // assign a vector to the closest partition among:
@@ -2072,6 +2142,7 @@ struct AssignResult {
     // and the deleted row ids
     assign_batches: Vec<Option<(RecordBatch, UInt64Array)>>,
     new_centroids: FixedSizeListArray,
+    split_partitions: Vec<usize>,
 }
 
 #[derive(Debug, Clone)]
@@ -2090,15 +2161,46 @@ enum ReassignPartition {
     ReassignCandidate(u32),
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 enum PartitionAdjustment {
-    /// Split partition at given id
-    Split(usize),
+    /// Split partitions at the given ids.
+    Split(Vec<usize>),
     /// Join partition at given id
     Join(usize),
 }
 
+struct SplitPlan {
+    part_idx: usize,
+    centroid2_part_idx: usize,
+    centroid1: ArrayRef,
+    centroid2: ArrayRef,
+    reassign_part_ids: UInt32Array,
+    original_assign_ops: Vec<(usize, AssignOp)>,
+}
+
+struct CandidateRequest {
+    centroid1_part_idx: usize,
+    centroid2_part_idx: usize,
+    centroid1: ArrayRef,
+    centroid2: ArrayRef,
+}
+
+#[derive(Clone)]
+struct CandidateMove {
+    row_id: u64,
+    source_part_idx: usize,
+    dest_part_idx: usize,
+    dest_distance: f32,
+    vector: ArrayRef,
+}
+
 pub(crate) fn index_type_string(sub_index: SubIndexType, quantizer: QuantizationType) -> String {
+    // FlatBin is a QuantizationType variant used internally for reconstruction,
+    // but the persisted index type string uses "FLAT" (differentiated by DataType).
+    let quantizer = match quantizer {
+        QuantizationType::FlatBin => QuantizationType::Flat,
+        other => other,
+    };
     match (sub_index, quantizer) {
         // ignore FLAT sub index,
         // IVF_FLAT_FLAT => IVF_FLAT
@@ -2119,9 +2221,44 @@ pub(crate) fn index_type_string(sub_index: SubIndexType, quantizer: Quantization
 #[cfg(test)]
 mod tests {
     use super::*;
-    use arrow_array::Float32Array;
+    use arrow_array::{Array, Float32Array, NullArray};
     use lance_index::vector::flat::index::{FlatIndex, FlatQuantizer};
 
+    struct SingleBatchReader {
+        batch: RecordBatch,
+        partition_id: usize,
+    }
+
+    #[async_trait::async_trait]
+    impl ShuffleReader for SingleBatchReader {
+        async fn read_partition(
+            &self,
+            partition_id: usize,
+        ) -> Result<Option<Box<dyn RecordBatchStream + Unpin + 'static>>> {
+            if partition_id != self.partition_id || self.batch.num_rows() == 0 {
+                return Ok(None);
+            }
+
+            let schema = self.batch.schema();
+            let stream = stream::iter(vec![Ok(self.batch.clone())]);
+            Ok(Some(Box::new(RecordBatchStreamAdapter::new(
+                schema, stream,
+            ))))
+        }
+
+        fn partition_size(&self, partition_id: usize) -> Result<usize> {
+            Ok(if partition_id == self.partition_id {
+                self.batch.num_rows()
+            } else {
+                0
+            })
+        }
+
+        fn total_loss(&self) -> Option<f64> {
+            None
+        }
+    }
+
     #[test]
     fn select_reassign_candidates_skips_deleted_partition() {
         let dim = 4;
@@ -2151,48 +2288,152 @@ mod tests {
     }
 
     #[test]
-    fn compute_reassign_assign_ops_moves_vectors_to_new_centroids() {
+    fn compute_reassign_candidate_moves_vectors_to_new_centroids() {
         let row_ids = UInt64Array::from(vec![1_u64, 2_u64]);
         let vectors = FixedSizeListArray::try_new_from_values(
             Float32Array::from(vec![0.0_f32, 0.0, 10.0, 10.0]),
             2,
         )
         .unwrap();
-        let reassign_part_ids = UInt32Array::from(vec![0_u32]);
         let reassign_part_centroids =
             FixedSizeListArray::try_new_from_values(Float32Array::from(vec![9.0_f32, 9.0]), 2)
                 .unwrap();
-        let centroid1: ArrayRef = Arc::new(Float32Array::from(vec![0.0_f32, 0.0]));
-        let centroid2: ArrayRef = Arc::new(Float32Array::from(vec![20.0_f32, 20.0]));
+        let baseline_dists = DistanceType::L2.arrow_batch_func()(
+            reassign_part_centroids.value(0).as_ref(),
+            &vectors,
+        )
+        .unwrap();
+        let centroid1_dists =
+            DistanceType::L2.arrow_batch_func()(&Float32Array::from(vec![0.0_f32, 0.0]), &vectors)
+                .unwrap();
+        let centroid2_dists = DistanceType::L2.arrow_batch_func()(
+            &Float32Array::from(vec![20.0_f32, 20.0]),
+            &vectors,
+        )
+        .unwrap();
 
-        let ops = IvfIndexBuilder::<FlatIndex, FlatQuantizer>::compute_reassign_assign_ops::<
-            Float32Type,
-        >(
-            DistanceType::L2,
-            0,
+        let mut best_moves = vec![None; row_ids.len()];
+        IvfIndexBuilder::<FlatIndex, FlatQuantizer>::update_best_candidate_moves::<Float32Type>(
+            1,
+            2,
+            &row_ids,
+            &vectors,
+            baseline_dists.values(),
+            centroid1_dists.values(),
+            centroid2_dists.values(),
+            &mut best_moves,
             0,
+        );
+
+        assert_eq!(best_moves.iter().flatten().count(), 1);
+        let candidate_move = best_moves[0].as_ref().unwrap();
+        assert_eq!(candidate_move.row_id, 1);
+        assert_eq!(candidate_move.source_part_idx, 0);
+        assert_eq!(candidate_move.dest_part_idx, 1);
+        assert_eq!(
+            candidate_move.vector.as_primitive::<Float32Type>().values(),
+            &[0.0_f32, 0.0]
+        );
+    }
+
+    #[test]
+    fn update_best_candidate_moves_preserves_multivector_entries() {
+        let row_ids = UInt64Array::from(vec![7_u64, 7_u64]);
+        let vectors = FixedSizeListArray::try_new_from_values(
+            Float32Array::from(vec![0.0_f32, 0.0, 1.0, 1.0]),
+            2,
+        )
+        .unwrap();
+        let baseline_dists = Float32Array::from(vec![10.0_f32, 10.0]);
+        let centroid1_dists = Float32Array::from(vec![1.0_f32, 2.0]);
+        let centroid2_dists = Float32Array::from(vec![3.0_f32, 4.0]);
+
+        let mut best_moves = vec![None; row_ids.len()];
+        IvfIndexBuilder::<FlatIndex, FlatQuantizer>::update_best_candidate_moves::<Float32Type>(
             1,
             2,
             &row_ids,
             &vectors,
+            baseline_dists.values(),
+            centroid1_dists.values(),
+            centroid2_dists.values(),
+            &mut best_moves,
+            0,
+        );
+
+        let best_moves = best_moves.into_iter().flatten().collect::<Vec<_>>();
+        assert_eq!(best_moves.len(), 2);
+        assert_eq!(best_moves[0].row_id, 7);
+        assert_eq!(best_moves[1].row_id, 7);
+        assert_eq!(
+            best_moves[0].vector.as_primitive::<Float32Type>().values(),
+            &[0.0_f32, 0.0]
+        );
+        assert_eq!(
+            best_moves[1].vector.as_primitive::<Float32Type>().values(),
+            &[1.0_f32, 1.0]
+        );
+    }
+
+    #[test]
+    fn finalize_split_plans_reassigns_filtered_centroid_ids() {
+        let centroid1: ArrayRef = Arc::new(Float32Array::from(vec![0.0_f32, 0.0]));
+        let centroid2: ArrayRef = Arc::new(Float32Array::from(vec![1.0_f32, 1.0]));
+        let vector: ArrayRef = Arc::new(Float32Array::from(vec![2.0_f32, 2.0]));
+        let mut split_plans = vec![SplitPlan {
+            part_idx: 3,
+            centroid2_part_idx: 5,
             centroid1,
             centroid2,
-            &reassign_part_ids,
-            &reassign_part_centroids,
+            reassign_part_ids: UInt32Array::from(vec![0_u32]),
+            original_assign_ops: vec![
+                (3, AssignOp::Add((10, vector.clone()))),
+                (5, AssignOp::Add((11, vector))),
+            ],
+        }];
+
+        IvfIndexBuilder::<FlatIndex, FlatQuantizer>::finalize_split_plans(&mut split_plans, 4);
+
+        assert_eq!(split_plans[0].centroid2_part_idx, 4);
+        assert_eq!(split_plans[0].original_assign_ops[0].0, 3);
+        assert_eq!(split_plans[0].original_assign_ops[1].0, 4);
+    }
+
+    #[tokio::test]
+    async fn take_partition_batches_preserves_partition_order_for_large_fixed_size_list() {
+        let value_length = 1_073_741_824i32;
+        let num_rows = 5usize;
+        let row_ids = UInt64Array::from(vec![4_u64, 3, 2, 1, 0]);
+        let part_ids = UInt32Array::from(vec![0_u32; num_rows]);
+        let values = Arc::new(NullArray::new(num_rows * value_length as usize));
+        let item_field = Arc::new(Field::new("item", DataType::Null, true));
+        let codes = FixedSizeListArray::try_new(item_field, value_length, values, None).unwrap();
+        let batch = RecordBatch::try_new(
+            Arc::new(arrow_schema::Schema::new(vec![
+                ROW_ID_FIELD.clone(),
+                PART_ID_FIELD.clone(),
+                Field::new(PQ_CODE_COLUMN, codes.data_type().clone(), true),
+            ])),
+            vec![Arc::new(row_ids), Arc::new(part_ids), Arc::new(codes)],
         )
         .unwrap();
+        let reader = SingleBatchReader {
+            batch,
+            partition_id: 0,
+        };
 
-        assert_eq!(ops.len(), 2);
-        assert!(matches!(ops[0], (0, AssignOp::Remove(1))));
-        match &ops[1] {
-            (1, AssignOp::Add((row_id, vector))) => {
-                assert_eq!(*row_id, 1);
-                assert_eq!(
-                    vector.as_primitive::<Float32Type>().values(),
-                    &[0.0_f32, 0.0]
-                );
-            }
-            other => panic!("unexpected op: {:?}", other),
-        }
+        let (batches, loss) = IvfIndexBuilder::<FlatIndex, FlatQuantizer>::take_partition_batches(
+            0,
+            &[],
+            Some(&reader),
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(loss, 0.0);
+        assert_eq!(batches.len(), 1);
+        assert!(batches[0].column_by_name(PART_ID_COLUMN).is_none());
+        let row_ids = batches[0][ROW_ID].as_primitive::<UInt64Type>();
+        assert_eq!(row_ids.values(), &[4, 3, 2, 1, 0]);
     }
 }
diff --git a/rust/lance/src/index/vector/fixture_test.rs b/rust/lance/src/index/vector/fixture_test.rs
index bf4f122be4b..facfd1f0cb3 100644
--- a/rust/lance/src/index/vector/fixture_test.rs
+++ b/rust/lance/src/index/vector/fixture_test.rs
@@ -278,6 +278,7 @@ mod test {
                 Arc::new(DatasetPreFilter {
                     deleted_ids: None,
                     filtered_ids: None,
+                    deleted_fragments: None,
                     final_mask: Mutex::new(OnceCell::new()),
                 }),
                 &NoOpMetricsCollector,
diff --git a/rust/lance/src/index/vector/ivf.rs b/rust/lance/src/index/vector/ivf.rs
index 67906bd742b..a38cf3a57de 100644
--- a/rust/lance/src/index/vector/ivf.rs
+++ b/rust/lance/src/index/vector/ivf.rs
@@ -3,11 +3,16 @@
 
 //! IVF - Inverted File index.
 
-use super::{builder::IvfIndexBuilder, utils::PartitionLoadLock};
 use super::{
+    LogicalIvfView,
     pq::{PQIndex, build_pq_model},
     utils::{filter_finite_training_data, maybe_sample_training_data},
 };
+use super::{
+    builder::{IvfIndexBuilder, index_type_string},
+    utils::PartitionLoadLock,
+};
+use crate::dataset::index::dataset_format_version;
 use crate::index::DatasetIndexInternalExt;
 use crate::index::vector::utils::{get_vector_dim, get_vector_type};
 use crate::{
@@ -55,14 +60,16 @@ use lance_index::vector::bq::builder::RabitQuantizer;
 use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex, FlatQuantizer};
 use lance_index::vector::hnsw::HnswMetadata;
 use lance_index::vector::hnsw::builder::HNSW_METADATA_KEY;
-use lance_index::vector::ivf::storage::{IVF_METADATA_KEY, IvfModel};
+use lance_index::vector::ivf::storage::IVF_METADATA_KEY;
+use lance_index::vector::ivf::storage::IvfModel;
 use lance_index::vector::kmeans::KMeansParams;
 use lance_index::vector::pq::storage::transpose;
 use lance_index::vector::quantizer::QuantizationType;
-use lance_index::vector::v3::shuffler::IvfShuffler;
+use lance_index::vector::v3::shuffler::create_ivf_shuffler;
 use lance_index::vector::v3::subindex::{IvfSubIndex, SubIndexType};
 use lance_index::{
     INDEX_AUXILIARY_FILE_NAME, INDEX_METADATA_SCHEMA_KEY, Index, IndexMetadata, IndexType,
+    MAX_PARTITION_SIZE_FACTOR, MIN_PARTITION_SIZE_PERCENT,
     optimize::OptimizeOptions,
     vector::{
         Query, VectorIndex,
@@ -87,13 +94,13 @@ use lance_io::{
 };
 use lance_linalg::distance::{DistanceType, Dot, L2, MetricType};
 use lance_linalg::{distance::Normalize, kernels::normalize_fsl_owned};
+use lance_table::format::{IndexMetadata as TableIndexMetadata, list_index_files_with_sizes};
 use log::{info, warn};
 use object_store::path::Path;
 use prost::Message;
 use roaring::RoaringBitmap;
 use serde::Serialize;
 use serde_json::json;
-use std::collections::HashSet;
 use std::{any::Any, collections::HashMap, sync::Arc};
 use tokio::sync::mpsc;
 use tracing::instrument;
@@ -101,6 +108,7 @@ use uuid::Uuid;
 
 pub mod builder;
 pub mod io;
+mod partition_serde;
 pub mod v2;
 
 // Cache wrapper for vector index trait objects
@@ -122,6 +130,10 @@ impl UnsizedCacheKey for LegacyIVFPartitionKey {
     fn key(&self) -> std::borrow::Cow<'_, str> {
         format!("ivf-{}", self.partition_id).into()
     }
+
+    fn type_name() -> &'static str {
+        "LegacyIVFPartition"
+    }
 }
 
 /// IVF Index.
@@ -263,6 +275,95 @@ impl std::fmt::Debug for IVFIndex {
     }
 }
 
+#[derive(Clone, Copy, Debug)]
+struct SegmentRebalanceCandidate {
+    segment_id: Uuid,
+    score: usize,
+    created_at_ms: i64,
+}
+
+fn candidate_is_better(
+    candidate: SegmentRebalanceCandidate,
+    current_best: Option<SegmentRebalanceCandidate>,
+) -> bool {
+    match current_best {
+        None => true,
+        Some(current_best) => {
+            candidate.score > current_best.score
+                || (candidate.score == current_best.score
+                    && (candidate.created_at_ms, candidate.segment_id.as_bytes())
+                        < (
+                            current_best.created_at_ms,
+                            current_best.segment_id.as_bytes(),
+                        ))
+        }
+    }
+}
+
+fn index_type_for_segmented_optimize(index: &dyn VectorIndex) -> Result<IndexType> {
+    let (sub_index_type, quantization_type) = index.sub_index_type();
+    IndexType::try_from(index_type_string(sub_index_type, quantization_type).as_str())
+}
+
+pub(crate) fn select_segment_for_single_rebalance(
+    logical_index: &LogicalIvfView<'_>,
+) -> Result<Option<Uuid>> {
+    let mut best_split = None;
+    let mut best_join = None;
+
+    for (metadata, index) in logical_index.segments() {
+        let index_type = index_type_for_segmented_optimize(index.as_ref())?;
+        let split_threshold = MAX_PARTITION_SIZE_FACTOR * index_type.target_partition_size();
+        let join_threshold = MIN_PARTITION_SIZE_PERCENT * index_type.target_partition_size() / 100;
+        let num_partitions = index.ivf_model().num_partitions();
+        if num_partitions == 0 {
+            continue;
+        }
+
+        let mut split_partition_count = 0usize;
+        let mut join_partition_count = 0usize;
+        for partition_id in 0..num_partitions {
+            let partition_size = index.partition_size(partition_id);
+            if partition_size > split_threshold {
+                split_partition_count += 1;
+            }
+            if num_partitions > 1 && partition_size < join_threshold {
+                join_partition_count += 1;
+            }
+        }
+
+        let created_at_ms = metadata
+            .created_at
+            .map(|dt| dt.timestamp_millis())
+            .unwrap_or(i64::MIN);
+
+        let split_candidate = (split_partition_count > 0).then_some(SegmentRebalanceCandidate {
+            segment_id: metadata.uuid,
+            score: split_partition_count,
+            created_at_ms,
+        });
+        if let Some(candidate) = split_candidate
+            && candidate_is_better(candidate, best_split)
+        {
+            best_split = Some(candidate);
+        }
+
+        let join_candidate = (join_partition_count > 0).then_some(SegmentRebalanceCandidate {
+            segment_id: metadata.uuid,
+            score: join_partition_count,
+            created_at_ms,
+        });
+        if let Some(candidate) = join_candidate
+            && candidate_is_better(candidate, best_join)
+        {
+            best_join = Some(candidate);
+        }
+    }
+
+    let selected = best_split.or(best_join);
+    Ok(selected.map(|candidate| candidate.segment_id))
+}
+
 // TODO: move to `lance-index` crate.
 ///
 /// Returns (new_uuid, num_indices_merged)
@@ -270,9 +371,10 @@ pub(crate) async fn optimize_vector_indices(
     dataset: Dataset,
     unindexed: Option<impl RecordBatchStream + Unpin + 'static>,
     vector_column: &str,
-    existing_indices: &[Arc<dyn Index>],
+    logical_index: &LogicalIvfView<'_>,
     options: &OptimizeOptions,
 ) -> Result<(Uuid, usize)> {
+    let existing_indices = logical_index.indices().cloned().collect::<Vec<_>>();
     // Sanity check the indices
     if existing_indices.is_empty() {
         return Err(Error::index(
@@ -287,7 +389,7 @@ pub(crate) async fn optimize_vector_indices(
             &dataset,
             unindexed,
             vector_column,
-            existing_indices,
+            &existing_indices,
             options,
         )
         .await;
@@ -314,7 +416,7 @@ pub(crate) async fn optimize_vector_indices(
             pq_index,
             vector_column,
             unindexed,
-            existing_indices,
+            &existing_indices,
             options,
             writer,
             dataset.version().version,
@@ -336,7 +438,7 @@ pub(crate) async fn optimize_vector_indices(
             hnsw_sq,
             vector_column,
             unindexed,
-            existing_indices,
+            &existing_indices,
             options,
             writer,
             aux_writer,
@@ -357,7 +459,7 @@ pub(crate) async fn optimize_vector_indices_v2(
     dataset: &Dataset,
     unindexed: Option<impl RecordBatchStream + Unpin + 'static>,
     vector_column: &str,
-    existing_indices: &[Arc<dyn Index>],
+    existing_indices: &[Arc<dyn VectorIndex>],
     options: &OptimizeOptions,
 ) -> Result<(Uuid, usize)> {
     // Sanity check the indices
@@ -366,11 +468,7 @@ pub(crate) async fn optimize_vector_indices_v2(
             "optimizing vector index: no existing index found".to_string(),
         ));
     }
-    let existing_indices = existing_indices
-        .iter()
-        .cloned()
-        .map(|idx| idx.as_vector_index())
-        .collect::<Result<Vec<_>>>()?;
+    let existing_indices = existing_indices.to_vec();
 
     let new_uuid = Uuid::new_v4();
     let index_dir = dataset.indices_dir().child(new_uuid.to_string());
@@ -381,9 +479,11 @@ pub(crate) async fn optimize_vector_indices_v2(
     let index_type = existing_indices[0].sub_index_type();
     let frag_reuse_index = dataset.open_frag_reuse_index(&NoOpMetricsCollector).await?;
 
+    let format_version = dataset_format_version(dataset);
+
     let temp_dir = lance_core::utils::tempfile::TempStdDir::default();
     let temp_dir_path = Path::from_filesystem_path(&temp_dir)?;
-    let shuffler = Box::new(IvfShuffler::new(temp_dir_path, num_partitions));
+    let shuffler = create_ivf_shuffler(temp_dir_path, num_partitions, format_version, None);
 
     let (_, element_type) = get_vector_type(dataset.schema(), vector_column)?;
     let merged_num = match index_type {
@@ -427,9 +527,9 @@ pub(crate) async fn optimize_vector_indices_v2(
                 .await?
             }
         }
-        // IVF_PQ
-        (SubIndexType::Flat, QuantizationType::Product) => {
-            IvfIndexBuilder::<FlatIndex, ProductQuantizer>::new_incremental(
+        // IVF_FLAT (binary vectors)
+        (SubIndexType::Flat, QuantizationType::FlatBin) => {
+            IvfIndexBuilder::<FlatIndex, FlatBinQuantizer>::new_incremental(
                 dataset.clone(),
                 vector_column.to_owned(),
                 index_dir,
@@ -447,9 +547,9 @@ pub(crate) async fn optimize_vector_indices_v2(
             .build()
             .await?
         }
-        // IVF_SQ
-        (SubIndexType::Flat, QuantizationType::Scalar) => {
-            IvfIndexBuilder::<FlatIndex, ScalarQuantizer>::new_incremental(
+        // IVF_PQ
+        (SubIndexType::Flat, QuantizationType::Product) => {
+            IvfIndexBuilder::<FlatIndex, ProductQuantizer>::new_incremental(
                 dataset.clone(),
                 vector_column.to_owned(),
                 index_dir,
@@ -467,8 +567,9 @@ pub(crate) async fn optimize_vector_indices_v2(
             .build()
             .await?
         }
-        (SubIndexType::Flat, QuantizationType::Rabit) => {
-            IvfIndexBuilder::<FlatIndex, RabitQuantizer>::new_incremental(
+        // IVF_SQ
+        (SubIndexType::Flat, QuantizationType::Scalar) => {
+            IvfIndexBuilder::<FlatIndex, ScalarQuantizer>::new_incremental(
                 dataset.clone(),
                 vector_column.to_owned(),
                 index_dir,
@@ -486,15 +587,14 @@ pub(crate) async fn optimize_vector_indices_v2(
             .build()
             .await?
         }
-        // IVF_HNSW_FLAT
-        (SubIndexType::Hnsw, QuantizationType::Flat) => {
-            IvfIndexBuilder::<HNSW, FlatQuantizer>::new_incremental(
+        (SubIndexType::Flat, QuantizationType::Rabit) => {
+            IvfIndexBuilder::<FlatIndex, RabitQuantizer>::new_incremental(
                 dataset.clone(),
                 vector_column.to_owned(),
                 index_dir,
                 distance_type,
                 shuffler,
-                HnswBuildParams::default(),
+                (),
                 frag_reuse_index,
                 options.clone(),
             )?
@@ -506,6 +606,46 @@ pub(crate) async fn optimize_vector_indices_v2(
             .build()
             .await?
         }
+        // IVF_HNSW_FLAT
+        (SubIndexType::Hnsw, QuantizationType::Flat) => {
+            if element_type == DataType::UInt8 {
+                IvfIndexBuilder::<HNSW, FlatBinQuantizer>::new_incremental(
+                    dataset.clone(),
+                    vector_column.to_owned(),
+                    index_dir,
+                    distance_type,
+                    shuffler,
+                    HnswBuildParams::default(),
+                    frag_reuse_index,
+                    options.clone(),
+                )?
+                .with_ivf(ivf_model.clone())
+                .with_quantizer(quantizer.try_into()?)
+                .with_existing_indices(existing_indices.clone())
+                .shuffle_data(unindexed)
+                .await?
+                .build()
+                .await?
+            } else {
+                IvfIndexBuilder::<HNSW, FlatQuantizer>::new_incremental(
+                    dataset.clone(),
+                    vector_column.to_owned(),
+                    index_dir,
+                    distance_type,
+                    shuffler,
+                    HnswBuildParams::default(),
+                    frag_reuse_index,
+                    options.clone(),
+                )?
+                .with_ivf(ivf_model.clone())
+                .with_quantizer(quantizer.try_into()?)
+                .with_existing_indices(existing_indices.clone())
+                .shuffle_data(unindexed)
+                .await?
+                .build()
+                .await?
+            }
+        }
         // IVF_HNSW_SQ
         (SubIndexType::Hnsw, QuantizationType::Scalar) => {
             IvfIndexBuilder::<HNSW, ScalarQuantizer>::new_incremental(
@@ -564,7 +704,7 @@ async fn optimize_ivf_pq_indices(
     pq_index: &PQIndex,
     vector_column: &str,
     unindexed: Option<impl RecordBatchStream + Unpin + 'static>,
-    existing_indices: &[Arc<dyn Index>],
+    existing_indices: &[Arc<dyn VectorIndex>],
     options: &OptimizeOptions,
     mut writer: Box<dyn Writer>,
     dataset_version: u64,
@@ -647,7 +787,7 @@ async fn optimize_ivf_hnsw_indices<Q: Quantization>(
     hnsw_index: &HNSWIndex<Q>,
     vector_column: &str,
     unindexed: Option<impl RecordBatchStream + Unpin + 'static>,
-    existing_indices: &[Arc<dyn Index>],
+    existing_indices: &[Arc<dyn VectorIndex>],
     options: &OptimizeOptions,
     writer: Box<dyn Writer>,
     aux_writer: Box<dyn Writer>,
@@ -1216,6 +1356,7 @@ pub async fn build_ivf_model(
     dim: usize,
     metric_type: MetricType,
     params: &IvfBuildParams,
+    fragment_ids: Option<&[u32]>,
     progress: std::sync::Arc<dyn lance_index::progress::IndexBuildProgress>,
 ) -> Result<IvfModel> {
     let num_partitions = params.num_partitions.unwrap();
@@ -1238,7 +1379,8 @@ pub async fn build_ivf_model(
         "Loading training data for IVF. Sample size: {}",
         sample_size_hint
     );
-    let training_data = maybe_sample_training_data(dataset, column, sample_size_hint).await?;
+    let training_data =
+        maybe_sample_training_data(dataset, column, sample_size_hint, fragment_ids).await?;
     info!(
         "Finished loading training data in {:02} seconds",
         start.elapsed().as_secs_f32()
@@ -1295,8 +1437,16 @@ async fn build_ivf_model_and_pq(
     get_vector_type(dataset.schema(), column)?;
     let dim = get_vector_dim(dataset.schema(), column)?;
 
-    let ivf_model =
-        build_ivf_model(dataset, column, dim, metric_type, ivf_params, progress).await?;
+    let ivf_model = build_ivf_model(
+        dataset,
+        column,
+        dim,
+        metric_type,
+        ivf_params,
+        None,
+        progress,
+    )
+    .await?;
 
     let ivf_residual = if matches!(metric_type, MetricType::Cosine | MetricType::L2) {
         Some(&ivf_model)
@@ -1530,6 +1680,13 @@ pub(crate) async fn remap_index_file_v3(
             .remap(mapping)
             .await
         }
+        (SubIndexType::Flat, QuantizationType::FlatBin) => {
+            IvfIndexBuilder::<FlatIndex, FlatBinQuantizer>::new_remapper(
+                dataset, column, index_dir, index,
+            )?
+            .remap(mapping)
+            .await
+        }
         (SubIndexType::Flat, QuantizationType::Rabit) => {
             IvfIndexBuilder::<FlatIndex, RabitQuantizer>::new_remapper(
                 dataset, column, index_dir, index,
@@ -1542,6 +1699,13 @@ pub(crate) async fn remap_index_file_v3(
                 .remap(mapping)
                 .await
         }
+        (SubIndexType::Hnsw, QuantizationType::FlatBin) => {
+            IvfIndexBuilder::<HNSW, FlatBinQuantizer>::new_remapper(
+                dataset, column, index_dir, index,
+            )?
+            .remap(mapping)
+            .await
+        }
         (SubIndexType::Hnsw, QuantizationType::Product) => {
             IvfIndexBuilder::<HNSW, ProductQuantizer>::new_remapper(
                 dataset, column, index_dir, index,
@@ -1854,29 +2018,150 @@ async fn write_ivf_hnsw_file(
     Ok(())
 }
 
-/// Finalize distributed merge for IVF-based vector indices.
-///
-/// This helper merges partial auxiliary index files produced by distributed
-/// jobs into a unified `auxiliary.idx` and then creates a root `index.idx`
-/// using the v2 index format so that `open_vector_index_v2` can load it.
+/// Merge one caller-defined group of source segments into a single segment.
+pub(crate) async fn merge_segments(
+    object_store: &ObjectStore,
+    indices_dir: &Path,
+    segments: Vec<TableIndexMetadata>,
+) -> Result<TableIndexMetadata> {
+    merge_segments_with_progress(
+        object_store,
+        indices_dir,
+        segments,
+        lance_index::progress::noop_progress(),
+    )
+    .await
+}
+
+/// Merge one caller-defined group of source segments into a single segment and
+/// report progress through the provided callback.
+pub(crate) async fn merge_segments_with_progress(
+    object_store: &ObjectStore,
+    indices_dir: &Path,
+    segments: Vec<TableIndexMetadata>,
+    progress: Arc<dyn lance_index::progress::IndexBuildProgress>,
+) -> Result<TableIndexMetadata> {
+    if segments.is_empty() {
+        return Err(Error::index("No segment metadata was provided".to_string()));
+    }
+    if segments.len() == 1 {
+        return Ok(segments.into_iter().next().unwrap());
+    }
+
+    let mut merged_segment = segments[0].clone();
+    let mut fragment_bitmap = RoaringBitmap::new();
+    for segment in &segments {
+        let source_fragment_bitmap = segment.fragment_bitmap.as_ref().ok_or_else(|| {
+            Error::index(format!(
+                "Segment '{}' is missing fragment coverage",
+                segment.uuid
+            ))
+        })?;
+        fragment_bitmap |= source_fragment_bitmap.clone();
+    }
+
+    let index_version = infer_source_index_version(&segments)?;
+    let segment_uuid = Uuid::new_v4();
+    let final_dir = indices_dir.child(segment_uuid.to_string());
+    merge_segments_to_dir(object_store, indices_dir, &final_dir, &segments, progress).await?;
+    let files = list_index_files_with_sizes(object_store, &final_dir).await?;
+
+    merged_segment = TableIndexMetadata {
+        uuid: segment_uuid,
+        fragment_bitmap: Some(fragment_bitmap),
+        index_details: Some(Arc::new(crate::index::vector_index_details())),
+        index_version,
+        created_at: Some(chrono::Utc::now()),
+        base_id: None,
+        files: Some(files),
+        ..merged_segment
+    };
+    Ok(merged_segment)
+}
+
+/// Merge the selected input segments into `final_dir`.
 ///
-/// The caller must pass `index_dir` pointing at the index UUID directory
-/// (e.g. `<table>/indices/<uuid>`). `requested_index_type` is only used as
-/// a fallback when the unified auxiliary file does not contain index
-/// metadata.
-pub async fn finalize_distributed_merge(
+/// The caller defines the source segment group explicitly. This helper reads
+/// those input segments directly from `indices/<segment_uuid>/` and writes the
+/// merged auxiliary/index files into `final_dir`.
+async fn merge_segments_to_dir(
     object_store: &ObjectStore,
-    index_dir: &object_store::path::Path,
-    requested_index_type: Option<IndexType>,
+    indices_dir: &Path,
+    final_dir: &Path,
+    segments: &[TableIndexMetadata],
+    progress: Arc<dyn lance_index::progress::IndexBuildProgress>,
 ) -> Result<()> {
-    // Merge per-shard auxiliary files into a unified auxiliary.idx.
+    reset_final_segment_dir(object_store, final_dir).await?;
+
+    debug_assert!(
+        segments.len() > 1,
+        "merge helper should only be used for multi-source groups"
+    );
+
+    let aux_paths = segments
+        .iter()
+        .map(|segment| {
+            indices_dir
+                .child(segment.uuid.to_string())
+                .child(INDEX_AUXILIARY_FILE_NAME)
+        })
+        .collect::<Vec<_>>();
+    let source_index_paths = segments
+        .iter()
+        .map(|segment| {
+            indices_dir
+                .child(segment.uuid.to_string())
+                .child(INDEX_FILE_NAME)
+        })
+        .collect::<Vec<_>>();
+
     lance_index::vector::distributed::index_merger::merge_partial_vector_auxiliary_files(
         object_store,
-        index_dir,
+        &aux_paths,
+        final_dir,
+        progress.clone(),
+    )
+    .await?;
+    write_root_vector_index_from_auxiliary(
+        object_store,
+        final_dir,
+        None,
+        &source_index_paths,
+        progress.clone(),
     )
     .await?;
 
-    // Open the unified auxiliary file.
+    Ok(())
+}
+
+fn infer_source_index_version(group: &[TableIndexMetadata]) -> Result<i32> {
+    debug_assert!(!group.is_empty());
+    let first = group[0].index_version;
+    if group.iter().any(|segment| segment.index_version != first) {
+        return Err(Error::index(
+            "Distributed vector segments must all have the same index version".to_string(),
+        ));
+    }
+    Ok(first)
+}
+
+/// Best-effort reset of one target directory before rewriting it.
+async fn reset_final_segment_dir(object_store: &ObjectStore, final_dir: &Path) -> Result<()> {
+    match object_store.remove_dir_all(final_dir.clone()).await {
+        Ok(()) => {}
+        Err(Error::NotFound { .. }) => {}
+        Err(err) => return Err(err),
+    }
+    Ok(())
+}
+
+async fn write_root_vector_index_from_auxiliary(
+    object_store: &ObjectStore,
+    index_dir: &Path,
+    requested_index_type: Option<IndexType>,
+    centroid_source_index_paths: &[Path],
+    progress: Arc<dyn lance_index::progress::IndexBuildProgress>,
+) -> Result<()> {
     let aux_path = index_dir.child(INDEX_AUXILIARY_FILE_NAME);
     let scheduler = ScanScheduler::new(
         Arc::new(object_store.clone()),
@@ -1895,6 +2180,8 @@ pub async fn finalize_distributed_merge(
     .await?;
 
     let meta = aux_reader.metadata();
+    // Inherit file format version from the unified auxiliary (which inherited it from shards)
+    let format_version = meta.version();
     let ivf_buf_idx: u32 = meta
         .file_schema
         .metadata
@@ -1907,36 +2194,20 @@ pub async fn finalize_distributed_merge(
     let mut pb_ivf: lance_index::pb::Ivf = Message::decode(raw_ivf_bytes.clone())?;
 
     // If the unified IVF metadata does not contain centroids, try to source them
-    // from any partial_* index.idx under this index directory.
+    // from one of the shard index files that fed this merge.
     if pb_ivf.centroids_tensor.is_none() {
-        let mut stream = object_store.list(Some(index_dir.clone()));
-        let mut partial_index_path = None;
-
-        while let Some(item) = stream.next().await {
-            let meta = item?;
-            if let Some(fname) = meta.location.filename()
-                && fname == INDEX_FILE_NAME
-            {
-                let parts: Vec<_> = meta.location.parts().collect();
-                if parts.len() >= 2 {
-                    let parent = parts[parts.len() - 2].as_ref();
-                    if parent.starts_with("partial_") {
-                        partial_index_path = Some(meta.location.clone());
-                        break;
-                    }
-                }
+        for partial_index_path in centroid_source_index_paths {
+            if !object_store.exists(partial_index_path).await? {
+                continue;
             }
-        }
-
-        if let Some(partial_index_path) = partial_index_path {
             let fh = scheduler
-                .open_file(&partial_index_path, &CachedFileSize::unknown())
+                .open_file(partial_index_path, &CachedFileSize::unknown())
                 .await?;
             let partial_reader = V2Reader::try_open(
                 fh,
                 None,
                 Arc::default(),
-                &lance_core::cache::LanceCache::no_cache(),
+                &LanceCache::no_cache(),
                 V2ReaderOptions::default(),
             )
             .await?;
@@ -1948,6 +2219,7 @@ pub async fn finalize_distributed_merge(
                 let partial_pb_ivf: lance_index::pb::Ivf = Message::decode(partial_ivf_bytes)?;
                 if partial_pb_ivf.centroids_tensor.is_some() {
                     pb_ivf.centroids_tensor = partial_pb_ivf.centroids_tensor;
+                    break;
                 }
             }
         }
@@ -1983,11 +2255,21 @@ pub async fn finalize_distributed_merge(
     // Write root index.idx via V2 writer so downstream opens through v2 path.
     let index_path = index_dir.child(INDEX_FILE_NAME);
     let obj_writer = object_store.create(&index_path).await?;
+    progress
+        .stage_start("write_root_index", Some(1), "files")
+        .await?;
 
     // Schema for HNSW sub-index: include neighbors/dist fields; empty batch is fine.
     let arrow_schema = HNSW::schema();
     let schema = lance_core::datatypes::Schema::try_from(arrow_schema.as_ref())?;
-    let mut v2_writer = V2Writer::try_new(obj_writer, schema, V2WriterOptions::default())?;
+    let mut v2_writer = V2Writer::try_new(
+        obj_writer,
+        schema,
+        V2WriterOptions {
+            format_version: Some(format_version),
+            ..Default::default()
+        },
+    )?;
 
     // Attach precise index metadata (type + distance).
     v2_writer.add_schema_metadata(INDEX_METADATA_SCHEMA_KEY, &index_meta_json);
@@ -2002,7 +2284,7 @@ pub async fn finalize_distributed_merge(
     let is_hnsw = idx_meta.index_type.starts_with("IVF_HNSW");
     let is_flat_based = matches!(
         idx_meta.index_type.as_str(),
-        "IVF_FLAT" | "IVF_PQ" | "IVF_SQ"
+        "IVF_FLAT" | "IVF_PQ" | "IVF_SQ" | "IVF_RQ"
     );
 
     if is_hnsw {
@@ -2021,68 +2303,8 @@ pub async fn finalize_distributed_merge(
     let empty_batch = RecordBatch::new_empty(arrow_schema);
     v2_writer.write_batch(&empty_batch).await?;
     v2_writer.finish().await?;
-
-    if let Err(err) = cleanup_partial_vector_dirs(object_store, index_dir).await {
-        warn!(
-            "Failed to cleanup partial_* vector index directories under '{}': {}",
-            index_dir.as_ref(),
-            err
-        );
-    }
-
-    Ok(())
-}
-
-/// Cleanup for distributed partial vector index directories after
-/// a distributed merge.
-///
-/// This helper scans `index_dir` for direct child directories whose names
-/// start with `partial_` (e.g. `<index_dir>/partial_0`, `<index_dir>/partial_1`)
-/// and attempts to recursively delete them via [`ObjectStore::remove_dir_all`].
-///
-/// Listing and deletion failures are logged with [`warn!`] and ignored so that
-/// index finalization is never blocked by cleanup. The function always returns
-/// `Ok(())`.
-async fn cleanup_partial_vector_dirs(
-    object_store: &ObjectStore,
-    index_dir: &object_store::path::Path,
-) -> Result<()> {
-    let mut partial_dirs: HashSet<Path> = HashSet::new();
-    let mut list_stream = object_store.list(Some(index_dir.clone()));
-
-    while let Some(item) = list_stream.next().await {
-        match item {
-            Ok(meta) => {
-                if let Some(relative_parts) = meta.location.prefix_match(index_dir) {
-                    let rel_parts: Vec<_> = relative_parts.collect();
-                    // Expect paths like: <index_dir>/partial_*/<file>
-                    if rel_parts.len() >= 2 {
-                        let parent_name = rel_parts[0].as_ref();
-                        if parent_name.starts_with("partial_") {
-                            partial_dirs.insert(index_dir.child(parent_name));
-                        }
-                    }
-                }
-            }
-            Err(e) => {
-                warn!(
-                    "Failed to list index directory '{}' while collecting partial_* dirs: {}",
-                    index_dir.as_ref(),
-                    e
-                );
-            }
-        }
-    }
-
-    for dir in partial_dirs {
-        if let Err(e) = object_store.remove_dir_all(dir.clone()).await {
-            warn!(
-                "Failed to remove partial_* directory '{}' after distributed merge: {}",
-                dir.as_ref(),
-                e
-            );
-        }
-    }
+    progress.stage_progress("write_root_index", 1).await?;
+    progress.stage_complete("write_root_index").await?;
 
     Ok(())
 }
@@ -2236,11 +2458,12 @@ mod tests {
 
     use arrow_array::types::UInt64Type;
     use arrow_array::{
-        FixedSizeListArray, Float32Array, RecordBatch, RecordBatchIterator, RecordBatchReader,
-        UInt64Array, make_array,
+        FixedSizeListArray, Float16Array, Float32Array, RecordBatch, RecordBatchIterator,
+        RecordBatchReader, UInt64Array, make_array,
     };
     use arrow_buffer::{BooleanBuffer, NullBuffer};
     use arrow_schema::{DataType, Field, Schema};
+    use half::f16;
     use itertools::Itertools;
     use lance_core::ROW_ID;
     use lance_core::utils::address::RowAddress;
@@ -2262,6 +2485,7 @@ mod tests {
     use crate::index::vector::IndexFileVersion;
     use crate::index::vector_index_details;
     use crate::index::{DatasetIndexExt, DatasetIndexInternalExt, vector::VectorIndexParams};
+    use crate::utils::test::copy_test_data_to_tmp;
 
     const DIM: usize = 32;
 
@@ -2300,28 +2524,19 @@ mod tests {
             }
         }
 
-        fn distance_between_points(&self) -> f32 {
-            (self.dim as f32).sqrt()
-        }
-
-        fn generate_centroids(&self) -> Float32Array {
+        fn generate_centroids(dim: u32, num_centroids: u32) -> Float32Array {
             const MAX_ATTEMPTS: u32 = 10;
-            let distance_needed =
-                self.distance_between_points() * Self::VALS_PER_CODE as f32 * 2_f32;
+            let distance_needed = (dim as f32).sqrt() * Self::VALS_PER_CODE as f32 * 2_f32;
             let mut attempts_remaining = MAX_ATTEMPTS;
-            let num_values = self.dim * self.num_centroids;
+            let num_values = dim * num_centroids;
             while attempts_remaining > 0 {
                 // Use some biggish numbers to ensure we get the distance we want but make them positive
                 // and not too big for easier debugging.
                 let centroids: Float32Array =
                     generate_scaled_random_array(num_values as usize, 0_f32, 1000_f32);
                 let mut broken = false;
-                for (index, centroid) in centroids
-                    .values()
-                    .chunks_exact(self.dim as usize)
-                    .enumerate()
-                {
-                    let offset = (index + 1) * self.dim as usize;
+                for (index, centroid) in centroids.values().chunks_exact(dim as usize).enumerate() {
+                    let offset = (index + 1) * dim as usize;
                     let length = centroids.len() - offset;
                     if length == 0 {
                         // This will be true for the last item since we ignore comparison with self
@@ -2330,7 +2545,7 @@ mod tests {
                     let distances = l2_distance_batch(
                         centroid,
                         &centroids.values()[offset..offset + length],
-                        self.dim as usize,
+                        dim as usize,
                     );
                     let min_distance = distances.min_by(|a, b| a.total_cmp(b)).unwrap();
                     // In theory we could just replace this one vector but, out of laziness, we just retry all of them
@@ -2351,11 +2566,10 @@ mod tests {
         }
 
         fn get_centroids(&mut self) -> &Float32Array {
-            if self.centroids.is_some() {
-                return self.centroids.as_ref().unwrap();
-            }
-            self.centroids = Some(self.generate_centroids());
-            self.centroids.as_ref().unwrap()
+            let dim = self.dim;
+            let num_centroids = self.num_centroids;
+            self.centroids
+                .get_or_insert_with(|| Self::generate_centroids(dim, num_centroids))
         }
 
         fn get_centroids_as_list_arr(&mut self) -> Arc<FixedSizeListArray> {
@@ -2368,10 +2582,12 @@ mod tests {
             )
         }
 
-        fn generate_vectors(&mut self) -> Float32Array {
-            let dim = self.dim as usize;
-            let num_centroids = self.num_centroids;
-            let centroids = self.get_centroids();
+        fn generate_vectors(
+            dim: u32,
+            num_centroids: u32,
+            centroids: &Float32Array,
+        ) -> Float32Array {
+            let dim = dim as usize;
             let mut vectors: Vec<f32> =
                 vec![0_f32; Self::VALS_PER_CODE as usize * dim * num_centroids as usize];
             for (centroid, dst_batch) in centroids
@@ -2389,11 +2605,11 @@ mod tests {
         }
 
         fn get_vectors(&mut self) -> &Float32Array {
-            if self.vectors.is_some() {
-                return self.vectors.as_ref().unwrap();
-            }
-            self.vectors = Some(self.generate_vectors());
-            self.vectors.as_ref().unwrap()
+            let dim = self.dim;
+            let num_centroids = self.num_centroids;
+            let centroids = self.get_centroids().clone();
+            self.vectors
+                .get_or_insert_with(|| Self::generate_vectors(dim, num_centroids, &centroids))
         }
 
         fn get_vector(&mut self, idx: u32) -> Float32Array {
@@ -2651,17 +2867,12 @@ mod tests {
             dataset_version: dataset.version().version,
             fields: vec![field.id],
             name: INDEX_NAME.to_string(),
-            fragment_bitmap: Some(
-                dataset
-                    .get_fragments()
-                    .iter()
-                    .map(|f| f.id() as u32)
-                    .collect(),
-            ),
+            fragment_bitmap: Some(dataset.fragment_bitmap.as_ref().clone()),
             index_details: Some(Arc::new(vector_index_details())),
             index_version: VECTOR_INDEX_VERSION as i32,
             created_at: Some(chrono::Utc::now()),
             base_id: None,
+            files: None,
         };
 
         // We need to commit this index to the dataset so that it can be found
@@ -2700,6 +2911,7 @@ mod tests {
             index_version: VECTOR_INDEX_VERSION as i32,
             created_at: None, // Test index, not setting timestamp
             base_id: None,
+            files: None,
         };
 
         let prefilter = Arc::new(DatasetPreFilter::new(dataset.clone(), &[index_meta], None));
@@ -2754,17 +2966,12 @@ mod tests {
             dataset_version: dataset_mut.version().version,
             fields: vec![field.id],
             name: format!("{}_remapped", INDEX_NAME),
-            fragment_bitmap: Some(
-                dataset_mut
-                    .get_fragments()
-                    .iter()
-                    .map(|f| f.id() as u32)
-                    .collect(),
-            ),
+            fragment_bitmap: Some(dataset_mut.fragment_bitmap.as_ref().clone()),
             index_details: Some(Arc::new(vector_index_details())),
             index_version: VECTOR_INDEX_VERSION as i32,
             created_at: Some(chrono::Utc::now()),
             base_id: None,
+            files: None,
         };
 
         // We need to commit this new index to the dataset so it can be found
@@ -3095,6 +3302,7 @@ mod tests {
             DIM,
             MetricType::L2,
             &ivf_params,
+            None,
             lance_index::progress::noop_progress(),
         )
         .await
@@ -3130,6 +3338,7 @@ mod tests {
             DIM,
             MetricType::Cosine,
             &ivf_params,
+            None,
             lance_index::progress::noop_progress(),
         )
         .await
@@ -3338,6 +3547,106 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_create_ivf_flat_f16() {
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        const DIM: usize = 32;
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "vector",
+            DataType::FixedSizeList(
+                Arc::new(Field::new("item", DataType::Float16, true)),
+                DIM as i32,
+            ),
+            true,
+        )]));
+
+        let arr = generate_random_array_with_seed::<Float16Type>(1000 * DIM, [22; 32]);
+        let fsl = FixedSizeListArray::try_new_from_values(arr, DIM as i32).unwrap();
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(fsl)]).unwrap();
+        let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(batches, test_uri, None).await.unwrap();
+
+        let params = VectorIndexParams::ivf_flat(2, MetricType::L2);
+        dataset
+            .create_index(&["vector"], IndexType::Vector, None, &params, false)
+            .await
+            .unwrap();
+
+        let query = Float16Array::from_iter_values(repeat_n(f16::from_f32(0.5), DIM));
+        let results = dataset
+            .scan()
+            .nearest("vector", &query, 5)
+            .unwrap()
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0].num_rows(), 5);
+        let schema = results[0].schema();
+        let field = schema.field(0);
+        let DataType::FixedSizeList(item, _) = field.data_type() else {
+            panic!("vector column should remain fixed size list");
+        };
+        assert_eq!(item.data_type(), &DataType::Float16);
+    }
+
+    #[tokio::test]
+    async fn test_create_ivf_hnsw_flat_f16() {
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        const DIM: usize = 32;
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "vector",
+            DataType::FixedSizeList(
+                Arc::new(Field::new("item", DataType::Float16, true)),
+                DIM as i32,
+            ),
+            true,
+        )]));
+
+        let arr = generate_random_array_with_seed::<Float16Type>(1000 * DIM, [22; 32]);
+        let fsl = FixedSizeListArray::try_new_from_values(arr, DIM as i32).unwrap();
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(fsl)]).unwrap();
+        let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(batches, test_uri, None).await.unwrap();
+
+        let params = VectorIndexParams::ivf_hnsw(
+            MetricType::L2,
+            IvfBuildParams::new(2),
+            HnswBuildParams::default(),
+        );
+        dataset
+            .create_index(&["vector"], IndexType::Vector, None, &params, false)
+            .await
+            .unwrap();
+
+        let query = Float16Array::from_iter_values(repeat_n(f16::from_f32(0.5), DIM));
+        let results = dataset
+            .scan()
+            .nearest("vector", &query, 5)
+            .unwrap()
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0].num_rows(), 5);
+        let schema = results[0].schema();
+        let field = schema.field(0);
+        let DataType::FixedSizeList(item, _) = field.data_type() else {
+            panic!("vector column should remain fixed size list");
+        };
+        assert_eq!(item.data_type(), &DataType::Float16);
+    }
+
     #[tokio::test]
     async fn test_create_ivf_pq_with_invalid_num_sub_vectors() {
         let test_dir = TempStrDir::default();
@@ -3648,50 +3957,6 @@ mod tests {
         assert!(correct_times >= 9, "correct: {}", correct_times);
     }
 
-    #[tokio::test]
-    async fn test_cleanup_removes_only_partial_dirs() {
-        let object_store = ObjectStore::memory();
-        let index_dir = Path::from("index/uuid_test_cleanup");
-
-        // partial_* directories that should be removed
-        let partial0_file = index_dir.child("partial_0").child("file.bin");
-        let partial_abc_file = index_dir.child("partial_abc").child("file.bin");
-
-        // Non-partial paths that must be preserved
-        let partialx_file = index_dir.child("partialX").child("file.bin");
-        let shard_file = index_dir.child("shard_0").child("file.bin");
-        let keep_root_file = index_dir.child("keep_root.txt");
-
-        object_store.put(&partial0_file, b"partial0").await.unwrap();
-        object_store
-            .put(&partial_abc_file, b"partial_abc")
-            .await
-            .unwrap();
-        object_store.put(&partialx_file, b"partialx").await.unwrap();
-        object_store.put(&shard_file, b"shard").await.unwrap();
-        object_store.put(&keep_root_file, b"root").await.unwrap();
-
-        // Sanity: all files exist before cleanup
-        assert!(object_store.exists(&partial0_file).await.unwrap());
-        assert!(object_store.exists(&partial_abc_file).await.unwrap());
-        assert!(object_store.exists(&partialx_file).await.unwrap());
-        assert!(object_store.exists(&shard_file).await.unwrap());
-        assert!(object_store.exists(&keep_root_file).await.unwrap());
-
-        cleanup_partial_vector_dirs(&object_store, &index_dir)
-            .await
-            .unwrap();
-
-        // partial_* directories should be removed
-        assert!(!object_store.exists(&partial0_file).await.unwrap());
-        assert!(!object_store.exists(&partial_abc_file).await.unwrap());
-
-        // Non-partial directories and root files must be preserved
-        assert!(object_store.exists(&partialx_file).await.unwrap());
-        assert!(object_store.exists(&shard_file).await.unwrap());
-        assert!(object_store.exists(&keep_root_file).await.unwrap());
-    }
-
     #[tokio::test(flavor = "multi_thread")]
     async fn test_build_ivf_model_progress_callback() {
         use lance_index::progress::IndexBuildProgress;
@@ -3733,6 +3998,7 @@ mod tests {
             DIM,
             MetricType::L2,
             &ivf_params,
+            None,
             progress,
         )
         .await
@@ -3762,27 +4028,6 @@ mod tests {
         }
     }
 
-    #[tokio::test]
-    async fn test_cleanup_idempotent() {
-        let object_store = ObjectStore::memory();
-        let index_dir = Path::from("index/uuid_test_cleanup_idempotent");
-
-        let partial_file = index_dir.child("partial_0").child("file.bin");
-        object_store.put(&partial_file, b"partial").await.unwrap();
-
-        assert!(object_store.exists(&partial_file).await.unwrap());
-
-        cleanup_partial_vector_dirs(&object_store, &index_dir)
-            .await
-            .unwrap();
-        assert!(!object_store.exists(&partial_file).await.unwrap());
-
-        // Second call should succeed even when there are no partial_* directories left.
-        cleanup_partial_vector_dirs(&object_store, &index_dir)
-            .await
-            .unwrap();
-    }
-
     #[tokio::test]
     async fn test_prewarm_ivf_legacy() {
         use lance_io::assert_io_eq;
@@ -3856,4 +4101,126 @@ mod tests {
         let stats = dataset.object_store().io_stats_incremental();
         assert_io_eq!(stats, read_iops, 0, "second prewarm should not perform IO");
     }
+
+    #[tokio::test]
+    async fn test_prewarm_ivf_legacy_multiple_deltas() {
+        use lance_io::assert_io_eq;
+
+        let test_dir = copy_test_data_to_tmp("v0.21.0/bad_index_fragment_bitmap").unwrap();
+        let test_uri = test_dir.path_str();
+        let test_uri = &test_uri;
+
+        // Trigger migration to repair legacy corrupt fragment bitmaps.
+        let mut dataset = Dataset::open(test_uri).await.unwrap();
+        dataset.index_statistics("vector_idx").await.unwrap();
+        dataset.checkout_latest().await.unwrap();
+
+        // Reopen dataset to avoid carrying index state in-memory from migration.
+        let dataset = Dataset::open(test_uri).await.unwrap();
+        let indices = dataset.load_indices_by_name("vector_idx").await.unwrap();
+        assert_eq!(indices.len(), 2, "expected two index deltas for vector_idx");
+        let unique_uuids: HashSet<_> = indices.iter().map(|meta| meta.uuid).collect();
+        assert_eq!(unique_uuids.len(), 2, "expected two unique index UUIDs");
+
+        let sample_batch = dataset
+            .scan()
+            .limit(Some(1), None)
+            .unwrap()
+            .project(&["vector"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let q = sample_batch["vector"]
+            .as_any()
+            .downcast_ref::<FixedSizeListArray>()
+            .unwrap()
+            .value(0)
+            .as_any()
+            .downcast_ref::<Float32Array>()
+            .unwrap()
+            .clone();
+
+        // Reset IO stats after migration and sampling.
+        dataset.object_store().io_stats_incremental();
+
+        // Prewarm should perform IO to load all index deltas into cache.
+        dataset.prewarm_index("vector_idx").await.unwrap();
+        let stats = dataset.object_store().io_stats_incremental();
+        assert!(
+            stats.read_iops > 0,
+            "prewarm should have read from disk, but read_iops was 0"
+        );
+
+        // Query should not perform index IO after prewarm of all deltas.
+        dataset
+            .scan()
+            .nearest("vector", &q, 10)
+            .unwrap()
+            .project(&["_rowid"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let stats = dataset.object_store().io_stats_incremental();
+        assert_io_eq!(
+            stats,
+            read_iops,
+            0,
+            "query should not perform IO after prewarm"
+        );
+
+        // Second prewarm should not need IO (already cached).
+        dataset.prewarm_index("vector_idx").await.unwrap();
+        let stats = dataset.object_store().io_stats_incremental();
+        assert_io_eq!(stats, read_iops, 0, "second prewarm should not perform IO");
+    }
+
+    #[tokio::test]
+    async fn test_optimize_ivf_flat_binary_vectors() {
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        const BIN_DIM: usize = 16;
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "bin_vector",
+            DataType::FixedSizeList(
+                Arc::new(Field::new("item", DataType::UInt8, true)),
+                BIN_DIM as i32,
+            ),
+            true,
+        )]));
+
+        let arr = arrow_array::UInt8Array::from_iter_values((0..1000 * BIN_DIM).map(|i| i as u8));
+        let fsl = FixedSizeListArray::try_new_from_values(arr, BIN_DIM as i32).unwrap();
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(fsl)]).unwrap();
+        let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(batches, test_uri, None).await.unwrap();
+
+        let params = VectorIndexParams::ivf_flat(2, MetricType::Hamming);
+        dataset
+            .create_index(&["bin_vector"], IndexType::Vector, None, &params, false)
+            .await
+            .unwrap();
+
+        // Append more data so optimize_indices has unindexed fragments to merge
+        let arr2 =
+            arrow_array::UInt8Array::from_iter_values((0..500 * BIN_DIM).map(|i| (i + 7) as u8));
+        let fsl2 = FixedSizeListArray::try_new_from_values(arr2, BIN_DIM as i32).unwrap();
+        let batch2 = RecordBatch::try_new(schema.clone(), vec![Arc::new(fsl2)]).unwrap();
+        let mut dataset = InsertBuilder::new(Arc::new(dataset))
+            .with_params(&WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            })
+            .execute(vec![batch2])
+            .await
+            .unwrap();
+
+        // This used to panic with "unsupported index type: FLAT, FLATBIN"
+        dataset.optimize_indices(&Default::default()).await.unwrap();
+
+        let indices = dataset.load_indices().await.unwrap();
+        assert!(!indices.is_empty(), "should have at least one index");
+    }
 }
diff --git a/rust/lance/src/index/vector/ivf/partition_serde.rs b/rust/lance/src/index/vector/ivf/partition_serde.rs
new file mode 100644
index 00000000000..f8d13a2f0b5
--- /dev/null
+++ b/rust/lance/src/index/vector/ivf/partition_serde.rs
@@ -0,0 +1,1134 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Serialization and zero-copy deserialization for IVF partition cache entries.
+//!
+//! The format is:
+//!
+//! ```text
+//! [header_len: u64 LE]
+//! [header: JSON bytes]
+//! [sub_index Arrow IPC stream]
+//! [... quantizer-specific IPC streams ...]
+//! [storage Arrow IPC stream]
+//! ```
+//!
+//! Each IPC section is a self-delimiting Arrow IPC stream (schema + batches + EOS
+//! marker), written directly to the underlying writer without buffering. On
+//! deserialization, each message is read into a per-message buffer and zero-copy
+//! decoded via [`lance_arrow::ipc`].
+
+use std::io::Write;
+use std::sync::Arc;
+
+use arrow_array::{FixedSizeListArray, RecordBatch};
+use arrow_schema::{DataType, Field, Schema};
+use bytes::Bytes;
+use lance_arrow::ipc::{
+    read_ipc_stream_at, read_ipc_stream_single_at, read_len_prefixed_bytes_at, write_ipc_stream,
+    write_ipc_stream_batches, write_len_prefixed_bytes,
+};
+use lance_core::cache::CacheCodecImpl;
+use lance_core::{Error, Result};
+use lance_index::vector::bq::RQRotationType;
+use lance_index::vector::bq::builder::RabitQuantizer;
+use lance_index::vector::bq::storage::RabitQuantizationMetadata;
+use lance_index::vector::flat::index::{FlatBinQuantizer, FlatMetadata, FlatQuantizer};
+use lance_index::vector::pq::ProductQuantizer;
+use lance_index::vector::pq::storage::ProductQuantizationMetadata;
+use lance_index::vector::quantizer::{Quantization, QuantizerStorage};
+use lance_index::vector::sq::ScalarQuantizer;
+use lance_index::vector::sq::storage::ScalarQuantizationMetadata;
+use lance_index::vector::storage::VectorStore;
+use lance_index::vector::v3::subindex::IvfSubIndex;
+use lance_linalg::distance::DistanceType;
+use serde::{Deserialize, Serialize};
+
+use super::v2::PartitionEntry;
+
+/// Returns an erased codec for `PartitionEntry<S, Q>` by matching on the
+/// quantizer's [`QuantizationType`]. Returns `None` for quantizer types
+/// that don't have a `CacheCodecImpl` implementation.
+///
+/// Uses enum dispatch rather than trait bounds to avoid propagating
+/// `CacheCodecImpl` constraints through the `IVFIndex<S, Q>` type hierarchy.
+pub fn partition_entry_codec<S: IvfSubIndex + 'static, Q: Quantization + 'static>()
+-> Option<lance_core::cache::CacheCodec> {
+    use lance_index::vector::quantizer::QuantizationType;
+    match Q::quantization_type() {
+        QuantizationType::Product => Some(codec_for::<S, Q, ProductQuantizer>()),
+        QuantizationType::Flat => Some(codec_for::<S, Q, FlatQuantizer>()),
+        QuantizationType::FlatBin => Some(codec_for::<S, Q, FlatBinQuantizer>()),
+        QuantizationType::Scalar => Some(codec_for::<S, Q, ScalarQuantizer>()),
+        QuantizationType::Rabit => Some(codec_for::<S, Q, RabitQuantizer>()),
+    }
+}
+
+type ArcAny = Arc<dyn std::any::Any + Send + Sync>;
+
+fn serialize_partition_entry<S, Concrete>(
+    any: &ArcAny,
+    writer: &mut dyn Write,
+) -> lance_core::Result<()>
+where
+    S: IvfSubIndex + 'static,
+    Concrete: Quantization + 'static,
+    PartitionEntry<S, Concrete>: CacheCodecImpl,
+{
+    let concrete = any
+        .downcast_ref::<PartitionEntry<S, Concrete>>()
+        .expect("quantization_type matched but downcast failed (this is a bug)");
+    concrete.serialize(writer)
+}
+
+fn deserialize_partition_entry<S, Q, Concrete>(data: &Bytes) -> lance_core::Result<ArcAny>
+where
+    S: IvfSubIndex + 'static,
+    Q: Quantization + 'static,
+    Concrete: Quantization + 'static,
+    PartitionEntry<S, Concrete>: CacheCodecImpl,
+{
+    let concrete = PartitionEntry::<S, Concrete>::deserialize(data)?;
+    let any: ArcAny = Arc::new(concrete);
+    Ok(any
+        .downcast::<PartitionEntry<S, Q>>()
+        .expect("quantization_type matched but downcast failed (this is a bug)"))
+}
+
+/// Build a CacheCodec for `PartitionEntry<S, Q>` by delegating to the
+/// CacheCodecImpl for `PartitionEntry<S, Concrete>`.
+///
+/// Q and Concrete must be the same type (enforced by the QuantizationType
+/// match in the caller). Uses Any-based downcasting to bridge the types.
+fn codec_for<
+    S: IvfSubIndex + 'static,
+    Q: Quantization + 'static,
+    Concrete: Quantization + 'static,
+>() -> lance_core::cache::CacheCodec
+where
+    PartitionEntry<S, Concrete>: CacheCodecImpl,
+{
+    lance_core::cache::CacheCodec::new(
+        serialize_partition_entry::<S, Concrete>,
+        deserialize_partition_entry::<S, Q, Concrete>,
+    )
+}
+
+// ---------------------------------------------------------------------------
+// Common helpers
+// ---------------------------------------------------------------------------
+
+fn distance_type_to_u8(dt: DistanceType) -> u8 {
+    match dt {
+        DistanceType::L2 => 0,
+        DistanceType::Cosine => 1,
+        DistanceType::Dot => 2,
+        DistanceType::Hamming => 3,
+    }
+}
+
+fn u8_to_distance_type(v: u8) -> Result<DistanceType> {
+    match v {
+        0 => Ok(DistanceType::L2),
+        1 => Ok(DistanceType::Cosine),
+        2 => Ok(DistanceType::Dot),
+        3 => Ok(DistanceType::Hamming),
+        _ => Err(Error::io(format!("unknown distance type: {v}"))),
+    }
+}
+
+fn rotation_type_to_u8(rt: RQRotationType) -> u8 {
+    match rt {
+        RQRotationType::Matrix => 0,
+        RQRotationType::Fast => 1,
+    }
+}
+
+fn u8_to_rotation_type(v: u8) -> Result<RQRotationType> {
+    match v {
+        0 => Ok(RQRotationType::Matrix),
+        1 => Ok(RQRotationType::Fast),
+        _ => Err(Error::io(format!("unknown rotation type: {v}"))),
+    }
+}
+
+/// Write a JSON-serializable header using [`write_len_prefixed_bytes`].
+fn write_json_header(writer: &mut dyn Write, header: &impl Serialize) -> Result<()> {
+    let header_json = serde_json::to_vec(header)?;
+    write_len_prefixed_bytes(writer, &header_json)?;
+    Ok(())
+}
+
+/// Read a JSON header written by [`write_json_header`].
+fn read_json_header<T: serde::de::DeserializeOwned>(data: &Bytes, offset: &mut usize) -> Result<T> {
+    let bytes = read_len_prefixed_bytes_at(data, offset).map_err(|e| Error::io(e.to_string()))?;
+    serde_json::from_slice(&bytes).map_err(|e| Error::io(e.to_string()))
+}
+
+/// Wrap a `FixedSizeListArray` in a single-column `RecordBatch` with the given
+/// column name.
+fn fsl_to_batch(arr: &FixedSizeListArray, name: &str) -> Result<RecordBatch> {
+    let field = Field::new(
+        name,
+        DataType::FixedSizeList(
+            Arc::new(Field::new("item", arr.value_type(), true)),
+            arr.value_length(),
+        ),
+        false,
+    );
+    let schema = Arc::new(Schema::new(vec![field]));
+    Ok(RecordBatch::try_new(schema, vec![Arc::new(arr.clone())])?)
+}
+
+/// Extract a `FixedSizeListArray` from the first column of a `RecordBatch`.
+fn batch_to_fsl(batch: &RecordBatch) -> Result<FixedSizeListArray> {
+    batch
+        .column(0)
+        .as_any()
+        .downcast_ref::<FixedSizeListArray>()
+        .cloned()
+        .ok_or_else(|| Error::io("column is not FixedSizeListArray".to_string()))
+}
+
+fn codebook_to_batch(codebook: &FixedSizeListArray) -> Result<RecordBatch> {
+    fsl_to_batch(codebook, "codebook")
+}
+
+fn batch_to_codebook(batch: &RecordBatch) -> Result<FixedSizeListArray> {
+    batch_to_fsl(batch)
+}
+
+// ---------------------------------------------------------------------------
+// PQ
+// ---------------------------------------------------------------------------
+
+#[derive(Serialize, Deserialize)]
+struct PqPartitionHeader {
+    distance_type: u8,
+    nbits: u32,
+    num_sub_vectors: usize,
+    dimension: usize,
+    transposed: bool,
+}
+
+impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, ProductQuantizer> {
+    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
+        let metadata = self.storage.metadata();
+        let distance_type = self.storage.distance_type();
+
+        let codebook = metadata.codebook.as_ref().ok_or_else(|| {
+            Error::io("PQ metadata missing codebook during serialization".to_string())
+        })?;
+
+        let header = PqPartitionHeader {
+            distance_type: distance_type_to_u8(distance_type),
+            nbits: metadata.nbits,
+            num_sub_vectors: metadata.num_sub_vectors,
+            dimension: metadata.dimension,
+            transposed: metadata.transposed,
+        };
+
+        write_json_header(writer, &header)?;
+        write_ipc_stream(&self.index.to_batch()?, writer)?;
+        write_ipc_stream(&codebook_to_batch(codebook)?, writer)?;
+        write_ipc_stream_batches(self.storage.to_batches()?, writer)?;
+
+        Ok(())
+    }
+
+    fn deserialize(data: &Bytes) -> Result<Self> {
+        let mut offset = 0;
+        let header: PqPartitionHeader = read_json_header(data, &mut offset)?;
+        let distance_type = u8_to_distance_type(header.distance_type)?;
+
+        let sub_index_batch =
+            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+        let codebook_batch =
+            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+        let storage_batch =
+            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+
+        let index = S::load(sub_index_batch)?;
+        let codebook = batch_to_codebook(&codebook_batch)?;
+
+        let metadata = ProductQuantizationMetadata {
+            codebook_position: 0,
+            nbits: header.nbits,
+            num_sub_vectors: header.num_sub_vectors,
+            dimension: header.dimension,
+            codebook: Some(codebook),
+            codebook_tensor: Vec::new(),
+            transposed: header.transposed,
+        };
+
+        let storage = <ProductQuantizer as Quantization>::Storage::try_from_batch(
+            storage_batch,
+            &metadata,
+            distance_type,
+            None,
+        )?;
+
+        Ok(Self { index, storage })
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Flat (Float32)
+// ---------------------------------------------------------------------------
+
+#[derive(Serialize, Deserialize)]
+struct FlatPartitionHeader {
+    distance_type: u8,
+    dim: usize,
+}
+
+impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, FlatQuantizer> {
+    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
+        let metadata = self.storage.metadata();
+        let distance_type = self.storage.distance_type();
+
+        let header = FlatPartitionHeader {
+            distance_type: distance_type_to_u8(distance_type),
+            dim: metadata.dim,
+        };
+
+        write_json_header(writer, &header)?;
+        write_ipc_stream(&self.index.to_batch()?, writer)?;
+        write_ipc_stream_batches(self.storage.to_batches()?, writer)?;
+
+        Ok(())
+    }
+
+    fn deserialize(data: &Bytes) -> Result<Self> {
+        let mut offset = 0;
+        let header: FlatPartitionHeader = read_json_header(data, &mut offset)?;
+        let distance_type = u8_to_distance_type(header.distance_type)?;
+
+        let sub_index_batch =
+            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+        let storage_batch =
+            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+
+        let index = S::load(sub_index_batch)?;
+        let metadata = FlatMetadata { dim: header.dim };
+        let storage = <FlatQuantizer as Quantization>::Storage::try_from_batch(
+            storage_batch,
+            &metadata,
+            distance_type,
+            None,
+        )?;
+
+        Ok(Self { index, storage })
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Flat (Binary / Hamming)
+// ---------------------------------------------------------------------------
+
+impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, FlatBinQuantizer> {
+    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
+        let metadata = self.storage.metadata();
+        let distance_type = self.storage.distance_type();
+
+        let header = FlatPartitionHeader {
+            distance_type: distance_type_to_u8(distance_type),
+            dim: metadata.dim,
+        };
+
+        write_json_header(writer, &header)?;
+        write_ipc_stream(&self.index.to_batch()?, writer)?;
+        write_ipc_stream_batches(self.storage.to_batches()?, writer)?;
+
+        Ok(())
+    }
+
+    fn deserialize(data: &Bytes) -> Result<Self> {
+        let mut offset = 0;
+        let header: FlatPartitionHeader = read_json_header(data, &mut offset)?;
+        let distance_type = u8_to_distance_type(header.distance_type)?;
+
+        let sub_index_batch =
+            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+        let storage_batch =
+            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+
+        let index = S::load(sub_index_batch)?;
+        let metadata = FlatMetadata { dim: header.dim };
+        let storage = <FlatBinQuantizer as Quantization>::Storage::try_from_batch(
+            storage_batch,
+            &metadata,
+            distance_type,
+            None,
+        )?;
+
+        Ok(Self { index, storage })
+    }
+}
+
+// ---------------------------------------------------------------------------
+// SQ
+// ---------------------------------------------------------------------------
+
+#[derive(Serialize, Deserialize)]
+struct SqPartitionHeader {
+    distance_type: u8,
+    num_bits: u16,
+    dim: usize,
+    bounds_start: f64,
+    bounds_end: f64,
+}
+
+impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, ScalarQuantizer> {
+    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
+        let metadata = self.storage.metadata();
+        let distance_type = self.storage.distance_type();
+
+        let header = SqPartitionHeader {
+            distance_type: distance_type_to_u8(distance_type),
+            num_bits: metadata.num_bits,
+            dim: metadata.dim,
+            bounds_start: metadata.bounds.start,
+            bounds_end: metadata.bounds.end,
+        };
+
+        write_json_header(writer, &header)?;
+        write_ipc_stream(&self.index.to_batch()?, writer)?;
+        // SQ storage may contain multiple batches; stream them all in one IPC stream.
+        write_ipc_stream_batches(self.storage.to_batches()?, writer)?;
+
+        Ok(())
+    }
+
+    fn deserialize(data: &Bytes) -> Result<Self> {
+        let mut offset = 0;
+        let header: SqPartitionHeader = read_json_header(data, &mut offset)?;
+        let distance_type = u8_to_distance_type(header.distance_type)?;
+
+        let sub_index_batch =
+            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+        let storage_batches =
+            read_ipc_stream_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+
+        let index = S::load(sub_index_batch)?;
+        let metadata = ScalarQuantizationMetadata {
+            dim: header.dim,
+            num_bits: header.num_bits,
+            bounds: header.bounds_start..header.bounds_end,
+        };
+        let storage = <ScalarQuantizer as Quantization>::Storage::try_new(
+            metadata.num_bits,
+            distance_type,
+            metadata.bounds,
+            storage_batches,
+            None,
+        )?;
+
+        Ok(Self { index, storage })
+    }
+}
+
+// ---------------------------------------------------------------------------
+// RabitQ
+// ---------------------------------------------------------------------------
+
+#[derive(Serialize, Deserialize)]
+struct RabitPartitionHeader {
+    distance_type: u8,
+    num_bits: u8,
+    code_dim: u32,
+    /// 0 = Matrix, 1 = Fast
+    rotation_type: u8,
+    /// Fast rotation signs (only set when rotation_type == Fast).
+    fast_rotation_signs: Option<Vec<u8>>,
+}
+
+impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, RabitQuantizer> {
+    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
+        let metadata = self.storage.metadata();
+        let distance_type = self.storage.distance_type();
+
+        let header = RabitPartitionHeader {
+            distance_type: distance_type_to_u8(distance_type),
+            num_bits: metadata.num_bits,
+            code_dim: metadata.code_dim,
+            rotation_type: rotation_type_to_u8(metadata.rotation_type),
+            fast_rotation_signs: metadata.fast_rotation_signs.clone(),
+        };
+
+        write_json_header(writer, &header)?;
+
+        write_ipc_stream(&self.index.to_batch()?, writer)?;
+
+        // Write the rotation matrix IPC stream only for Matrix rotation; the
+        // Fast rotation case stores its signs compactly in the JSON header.
+        if metadata.rotation_type == RQRotationType::Matrix {
+            let mat = metadata.rotate_mat.as_ref().ok_or_else(|| {
+                Error::io(
+                    "RabitQ Matrix metadata missing rotate_mat during serialization".to_string(),
+                )
+            })?;
+            write_ipc_stream(&fsl_to_batch(mat, "rotate_mat")?, writer)?;
+        }
+
+        write_ipc_stream_batches(self.storage.to_batches()?, writer)?;
+
+        Ok(())
+    }
+
+    fn deserialize(data: &Bytes) -> Result<Self> {
+        let mut offset = 0;
+        let header: RabitPartitionHeader = read_json_header(data, &mut offset)?;
+        let distance_type = u8_to_distance_type(header.distance_type)?;
+        let rotation_type = u8_to_rotation_type(header.rotation_type)?;
+
+        let sub_index_batch =
+            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+
+        let rotate_mat = if rotation_type == RQRotationType::Matrix {
+            let mat_batch = read_ipc_stream_single_at(data, &mut offset)
+                .map_err(|e| Error::io(e.to_string()))?;
+            Some(batch_to_fsl(&mat_batch)?)
+        } else {
+            None
+        };
+
+        let storage_batch =
+            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+
+        let index = S::load(sub_index_batch)?;
+        let metadata = RabitQuantizationMetadata {
+            rotate_mat,
+            rotate_mat_position: None,
+            fast_rotation_signs: header.fast_rotation_signs,
+            rotation_type,
+            code_dim: header.code_dim,
+            num_bits: header.num_bits,
+            // The storage batch already has packed codes; skip re-packing.
+            packed: true,
+        };
+        let storage = <RabitQuantizer as Quantization>::Storage::try_from_batch(
+            storage_batch,
+            &metadata,
+            distance_type,
+            None,
+        )?;
+
+        Ok(Self { index, storage })
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::sync::Arc;
+
+    use arrow_array::cast::AsArray;
+    use arrow_array::{
+        Float16Array, Float32Array, Float64Array, UInt8Array, UInt64Array,
+        types::{Float32Type, UInt8Type},
+    };
+    use arrow_schema::{DataType, Field, Schema};
+    use half::f16;
+    use lance_arrow::FixedSizeListArrayExt;
+    use lance_index::vector::bq::storage::RABIT_CODE_COLUMN;
+    use lance_index::vector::bq::transform::{ADD_FACTORS_COLUMN, SCALE_FACTORS_COLUMN};
+    use lance_index::vector::bq::{RQRotationType, builder::RabitQuantizer};
+    use lance_index::vector::flat::index::FlatIndex;
+    use lance_index::vector::flat::storage::FlatFloatStorage;
+    use lance_index::vector::sq::storage::ScalarQuantizationStorage;
+
+    // ----- PQ helpers -------------------------------------------------------
+
+    fn make_test_codebook(dim: usize, num_sub_vectors: usize) -> FixedSizeListArray {
+        let sub_dim = dim / num_sub_vectors;
+        let num_centroids = 256;
+        let total_values = num_sub_vectors * num_centroids * sub_dim;
+        let values: Vec<f32> = (0..total_values).map(|i| i as f32 * 0.01).collect();
+        let values_array = Float32Array::from(values);
+        FixedSizeListArray::try_new_from_values(values_array, sub_dim as i32).unwrap()
+    }
+
+    fn make_test_pq_storage(
+        num_rows: usize,
+        dim: usize,
+        num_sub_vectors: usize,
+    ) -> <ProductQuantizer as Quantization>::Storage {
+        let codebook = make_test_codebook(dim, num_sub_vectors);
+        let row_ids = UInt64Array::from((0..num_rows as u64).collect::<Vec<_>>());
+        let pq_codes_flat: Vec<u8> = (0..num_rows * num_sub_vectors)
+            .map(|i| (i % 256) as u8)
+            .collect();
+        let pq_codes = UInt8Array::from(pq_codes_flat);
+        let pq_codes_fsl =
+            FixedSizeListArray::try_new_from_values(pq_codes, num_sub_vectors as i32).unwrap();
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new(lance_core::ROW_ID, DataType::UInt64, false),
+            Field::new(
+                lance_index::vector::PQ_CODE_COLUMN,
+                DataType::FixedSizeList(
+                    Arc::new(Field::new("item", DataType::UInt8, true)),
+                    num_sub_vectors as i32,
+                ),
+                false,
+            ),
+        ]));
+
+        let batch =
+            RecordBatch::try_new(schema, vec![Arc::new(row_ids), Arc::new(pq_codes_fsl)]).unwrap();
+
+        <ProductQuantizer as Quantization>::Storage::new(
+            codebook,
+            batch,
+            8,
+            num_sub_vectors,
+            dim,
+            DistanceType::L2,
+            false,
+            None,
+        )
+        .unwrap()
+    }
+
+    // ----- PQ tests ---------------------------------------------------------
+
+    #[test]
+    fn test_roundtrip_flat_pq() {
+        let dim = 128;
+        let num_sub_vectors = 16;
+        let num_rows = 100;
+
+        let storage = make_test_pq_storage(num_rows, dim, num_sub_vectors);
+        let entry = PartitionEntry::<FlatIndex, ProductQuantizer> {
+            index: FlatIndex::default(),
+            storage,
+        };
+
+        let mut serialized = Vec::new();
+        entry.serialize(&mut serialized).unwrap();
+        let deserialized = PartitionEntry::<FlatIndex, ProductQuantizer>::deserialize(
+            &bytes::Bytes::from(serialized),
+        )
+        .unwrap();
+
+        assert_eq!(entry.storage, deserialized.storage);
+    }
+
+    #[test]
+    fn test_roundtrip_preserves_distance_type() {
+        for dt in [DistanceType::L2, DistanceType::Cosine, DistanceType::Dot] {
+            let dim = 32;
+            let num_sub_vectors = 4;
+            let codebook = make_test_codebook(dim, num_sub_vectors);
+            let row_ids = UInt64Array::from(vec![0u64, 1, 2]);
+            let pq_codes = UInt8Array::from(vec![0u8; 3 * num_sub_vectors]);
+            let pq_codes_fsl =
+                FixedSizeListArray::try_new_from_values(pq_codes, num_sub_vectors as i32).unwrap();
+
+            let schema = Arc::new(Schema::new(vec![
+                Field::new(lance_core::ROW_ID, DataType::UInt64, false),
+                Field::new(
+                    lance_index::vector::PQ_CODE_COLUMN,
+                    DataType::FixedSizeList(
+                        Arc::new(Field::new("item", DataType::UInt8, true)),
+                        num_sub_vectors as i32,
+                    ),
+                    false,
+                ),
+            ]));
+            let batch =
+                RecordBatch::try_new(schema, vec![Arc::new(row_ids), Arc::new(pq_codes_fsl)])
+                    .unwrap();
+
+            let storage = <ProductQuantizer as Quantization>::Storage::new(
+                codebook,
+                batch,
+                8,
+                num_sub_vectors,
+                dim,
+                dt,
+                false,
+                None,
+            )
+            .unwrap();
+
+            let entry = PartitionEntry::<FlatIndex, ProductQuantizer> {
+                index: FlatIndex::default(),
+                storage,
+            };
+
+            let mut bytes = Vec::new();
+            entry.serialize(&mut bytes).unwrap();
+            let restored = PartitionEntry::<FlatIndex, ProductQuantizer>::deserialize(
+                &bytes::Bytes::from(bytes),
+            )
+            .unwrap();
+            assert_eq!(
+                restored.storage.distance_type(),
+                entry.storage.distance_type()
+            );
+        }
+    }
+
+    #[test]
+    fn test_empty_partition() {
+        let dim = 16;
+        let num_sub_vectors = 2;
+        let storage = make_test_pq_storage(0, dim, num_sub_vectors);
+        let entry = PartitionEntry::<FlatIndex, ProductQuantizer> {
+            index: FlatIndex::default(),
+            storage,
+        };
+
+        let mut serialized = Vec::new();
+        entry.serialize(&mut serialized).unwrap();
+        let deserialized = PartitionEntry::<FlatIndex, ProductQuantizer>::deserialize(
+            &bytes::Bytes::from(serialized),
+        )
+        .unwrap();
+        assert_eq!(entry.storage, deserialized.storage);
+    }
+
+    #[test]
+    fn test_truncated_data_errors() {
+        // Serialize a valid entry, then truncate the bytes and verify that
+        // deserialization fails rather than panicking.
+        let storage = make_test_pq_storage(1, 16, 2);
+        let entry = PartitionEntry::<FlatIndex, ProductQuantizer> {
+            index: FlatIndex::default(),
+            storage,
+        };
+        let mut bytes = Vec::new();
+        entry.serialize(&mut bytes).unwrap();
+        bytes.truncate(3);
+        assert!(
+            PartitionEntry::<FlatIndex, ProductQuantizer>::deserialize(&bytes::Bytes::from(bytes))
+                .is_err()
+        );
+    }
+
+    // ----- Flat helpers -----------------------------------------------------
+
+    fn make_flat_storage(num_rows: usize, dim: usize) -> FlatFloatStorage {
+        let values: Vec<f32> = (0..num_rows * dim).map(|i| i as f32 * 0.01).collect();
+        let values_array = Float32Array::from(values);
+        let vectors = FixedSizeListArray::try_new_from_values(values_array, dim as i32).unwrap();
+        FlatFloatStorage::new(vectors, DistanceType::L2)
+    }
+
+    fn make_flat_storage_f16(num_rows: usize, dim: usize) -> FlatFloatStorage {
+        let values: Vec<f16> = (0..num_rows * dim)
+            .map(|i| f16::from_f32(i as f32 * 0.01))
+            .collect();
+        let values_array = Float16Array::from(values);
+        let vectors = FixedSizeListArray::try_new_from_values(values_array, dim as i32).unwrap();
+        FlatFloatStorage::new(vectors, DistanceType::L2)
+    }
+
+    fn make_flat_storage_f64(num_rows: usize, dim: usize) -> FlatFloatStorage {
+        let values: Vec<f64> = (0..num_rows * dim).map(|i| i as f64 * 0.01).collect();
+        let values_array = Float64Array::from(values);
+        let vectors = FixedSizeListArray::try_new_from_values(values_array, dim as i32).unwrap();
+        FlatFloatStorage::new(vectors, DistanceType::L2)
+    }
+
+    // ----- Flat tests -------------------------------------------------------
+
+    #[test]
+    fn test_roundtrip_flat_flat() {
+        let storage = make_flat_storage(50, 64);
+        let entry = PartitionEntry::<FlatIndex, FlatQuantizer> {
+            index: FlatIndex::default(),
+            storage,
+        };
+
+        let mut bytes = Vec::new();
+        entry.serialize(&mut bytes).unwrap();
+        let restored =
+            PartitionEntry::<FlatIndex, FlatQuantizer>::deserialize(&bytes::Bytes::from(bytes))
+                .unwrap();
+
+        assert_eq!(
+            restored.storage.metadata().dim,
+            entry.storage.metadata().dim
+        );
+        assert_eq!(
+            restored.storage.distance_type(),
+            entry.storage.distance_type()
+        );
+        assert_eq!(restored.storage.len(), entry.storage.len());
+        let orig_batch = entry.storage.to_batches().unwrap().next().unwrap();
+        let rest_batch = restored.storage.to_batches().unwrap().next().unwrap();
+        assert_eq!(orig_batch, rest_batch);
+    }
+
+    #[test]
+    fn test_flat_distance_types() {
+        for dt in [DistanceType::L2, DistanceType::Cosine, DistanceType::Dot] {
+            let values = Float32Array::from(vec![1.0f32; 32]);
+            let vectors = FixedSizeListArray::try_new_from_values(values, 32).unwrap();
+            let storage = FlatFloatStorage::new(vectors, dt);
+            let entry = PartitionEntry::<FlatIndex, FlatQuantizer> {
+                index: FlatIndex::default(),
+                storage,
+            };
+            let mut bytes = Vec::new();
+            entry.serialize(&mut bytes).unwrap();
+            let restored =
+                PartitionEntry::<FlatIndex, FlatQuantizer>::deserialize(&bytes::Bytes::from(bytes))
+                    .unwrap();
+            assert_eq!(restored.storage.distance_type(), dt);
+        }
+    }
+
+    #[test]
+    fn test_roundtrip_flat_flat_f16() {
+        let storage = make_flat_storage_f16(8, 16);
+        let entry = PartitionEntry::<FlatIndex, FlatQuantizer> {
+            index: FlatIndex::default(),
+            storage,
+        };
+
+        let mut bytes = Vec::new();
+        entry.serialize(&mut bytes).unwrap();
+        let restored =
+            PartitionEntry::<FlatIndex, FlatQuantizer>::deserialize(&bytes::Bytes::from(bytes))
+                .unwrap();
+
+        let restored_batch = restored.storage.to_batches().unwrap().next().unwrap();
+        let schema = restored_batch.schema();
+        let field = schema
+            .field_with_name(lance_index::vector::flat::storage::FLAT_COLUMN)
+            .unwrap();
+        let DataType::FixedSizeList(item, _) = field.data_type() else {
+            panic!("flat column should be fixed size list");
+        };
+        assert_eq!(item.data_type(), &DataType::Float16);
+    }
+
+    #[test]
+    fn test_roundtrip_flat_flat_f64() {
+        let storage = make_flat_storage_f64(8, 16);
+        let entry = PartitionEntry::<FlatIndex, FlatQuantizer> {
+            index: FlatIndex::default(),
+            storage,
+        };
+
+        let mut bytes = Vec::new();
+        entry.serialize(&mut bytes).unwrap();
+        let restored =
+            PartitionEntry::<FlatIndex, FlatQuantizer>::deserialize(&bytes::Bytes::from(bytes))
+                .unwrap();
+
+        let restored_batch = restored.storage.to_batches().unwrap().next().unwrap();
+        let schema = restored_batch.schema();
+        let field = schema
+            .field_with_name(lance_index::vector::flat::storage::FLAT_COLUMN)
+            .unwrap();
+        let DataType::FixedSizeList(item, _) = field.data_type() else {
+            panic!("flat column should be fixed size list");
+        };
+        assert_eq!(item.data_type(), &DataType::Float64);
+    }
+
+    // ----- SQ helpers -------------------------------------------------------
+
+    fn make_sq_storage(
+        num_rows: usize,
+        dim: usize,
+        distance_type: DistanceType,
+    ) -> ScalarQuantizationStorage {
+        let row_ids = UInt64Array::from_iter_values(0..num_rows as u64);
+        let sq_codes_flat: Vec<u8> = (0..num_rows * dim).map(|i| (i % 256) as u8).collect();
+        let sq_codes = UInt8Array::from(sq_codes_flat);
+        let sq_codes_fsl = FixedSizeListArray::try_new_from_values(sq_codes, dim as i32).unwrap();
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new(lance_core::ROW_ID, DataType::UInt64, false),
+            Field::new(
+                lance_index::vector::SQ_CODE_COLUMN,
+                DataType::FixedSizeList(
+                    Arc::new(Field::new("item", DataType::UInt8, true)),
+                    dim as i32,
+                ),
+                false,
+            ),
+        ]));
+        let batch =
+            RecordBatch::try_new(schema, vec![Arc::new(row_ids), Arc::new(sq_codes_fsl)]).unwrap();
+
+        ScalarQuantizationStorage::try_new(8, distance_type, -1.0..1.0, [batch], None).unwrap()
+    }
+
+    // ----- SQ tests ---------------------------------------------------------
+
+    #[test]
+    fn test_roundtrip_flat_sq() {
+        let storage = make_sq_storage(100, 64, DistanceType::L2);
+        let entry = PartitionEntry::<FlatIndex, ScalarQuantizer> {
+            index: FlatIndex::default(),
+            storage,
+        };
+
+        let mut bytes = Vec::new();
+        entry.serialize(&mut bytes).unwrap();
+        let restored =
+            PartitionEntry::<FlatIndex, ScalarQuantizer>::deserialize(&bytes::Bytes::from(bytes))
+                .unwrap();
+
+        let m = entry.storage.metadata();
+        let rm = restored.storage.metadata();
+        assert_eq!(rm.dim, m.dim);
+        assert_eq!(rm.num_bits, m.num_bits);
+        assert_eq!(rm.bounds, m.bounds);
+        assert_eq!(
+            restored.storage.distance_type(),
+            entry.storage.distance_type()
+        );
+        assert_eq!(restored.storage.len(), entry.storage.len());
+
+        let orig_ids: Vec<u64> = entry.storage.row_ids().copied().collect();
+        let rest_ids: Vec<u64> = restored.storage.row_ids().copied().collect();
+        assert_eq!(orig_ids, rest_ids);
+    }
+
+    #[test]
+    fn test_sq_distance_types() {
+        for dt in [DistanceType::L2, DistanceType::Cosine, DistanceType::Dot] {
+            let storage = make_sq_storage(10, 16, dt);
+            let entry = PartitionEntry::<FlatIndex, ScalarQuantizer> {
+                index: FlatIndex::default(),
+                storage,
+            };
+            let mut bytes = Vec::new();
+            entry.serialize(&mut bytes).unwrap();
+            let restored = PartitionEntry::<FlatIndex, ScalarQuantizer>::deserialize(
+                &bytes::Bytes::from(bytes),
+            )
+            .unwrap();
+            assert_eq!(restored.storage.distance_type(), dt);
+        }
+    }
+
+    #[test]
+    fn test_sq_multiple_chunks_no_copy() {
+        // Build SQ storage with multiple chunks by appending batches separately.
+        let dim = 16usize;
+        let make_batch = |start: u64, n: usize| {
+            let row_ids = UInt64Array::from_iter_values(start..start + n as u64);
+            let codes = UInt8Array::from(vec![0u8; n * dim]);
+            let fsl = FixedSizeListArray::try_new_from_values(codes, dim as i32).unwrap();
+            let schema = Arc::new(Schema::new(vec![
+                Field::new(lance_core::ROW_ID, DataType::UInt64, false),
+                Field::new(
+                    lance_index::vector::SQ_CODE_COLUMN,
+                    DataType::FixedSizeList(
+                        Arc::new(Field::new("item", DataType::UInt8, true)),
+                        dim as i32,
+                    ),
+                    false,
+                ),
+            ]));
+            RecordBatch::try_new(schema, vec![Arc::new(row_ids), Arc::new(fsl)]).unwrap()
+        };
+        // Three chunks with 10 rows each.
+        let storage = ScalarQuantizationStorage::try_new(
+            8,
+            DistanceType::L2,
+            -1.0..1.0,
+            [make_batch(0, 10), make_batch(10, 10), make_batch(20, 10)],
+            None,
+        )
+        .unwrap();
+        assert_eq!(storage.len(), 30);
+
+        let entry = PartitionEntry::<FlatIndex, ScalarQuantizer> {
+            index: FlatIndex::default(),
+            storage,
+        };
+        let mut bytes = Vec::new();
+        entry.serialize(&mut bytes).unwrap();
+        let restored =
+            PartitionEntry::<FlatIndex, ScalarQuantizer>::deserialize(&bytes::Bytes::from(bytes))
+                .unwrap();
+
+        assert_eq!(restored.storage.len(), 30);
+        let orig_ids: Vec<u64> = entry.storage.row_ids().copied().collect();
+        let rest_ids: Vec<u64> = restored.storage.row_ids().copied().collect();
+        assert_eq!(orig_ids, rest_ids);
+    }
+
+    // ----- RabitQ helpers ---------------------------------------------------
+
+    fn make_rabit_storage_fast(
+        num_rows: usize,
+        code_dim: usize,
+        distance_type: DistanceType,
+    ) -> <RabitQuantizer as Quantization>::Storage {
+        use lance_arrow::FixedSizeListArrayExt;
+
+        let quantizer = RabitQuantizer::new_with_rotation::<Float32Type>(
+            1,
+            code_dim as i32,
+            RQRotationType::Fast,
+        );
+        let values: Vec<f32> = (0..num_rows * code_dim)
+            .map(|i| (i % 100) as f32 / 100.0 - 0.5)
+            .collect();
+        let values_arr = Float32Array::from(values);
+        let vectors = FixedSizeListArray::try_new_from_values(values_arr, code_dim as i32).unwrap();
+        let codes = quantizer
+            .quantize(&vectors)
+            .unwrap()
+            .as_fixed_size_list()
+            .clone();
+
+        let metadata = quantizer.metadata(None);
+        let batch = RecordBatch::try_from_iter(vec![
+            (
+                lance_core::ROW_ID,
+                Arc::new(UInt64Array::from_iter_values(0..num_rows as u64))
+                    as Arc<dyn arrow_array::Array>,
+            ),
+            (
+                RABIT_CODE_COLUMN,
+                Arc::new(codes) as Arc<dyn arrow_array::Array>,
+            ),
+            (
+                ADD_FACTORS_COLUMN,
+                Arc::new(Float32Array::from_iter_values(
+                    (0..num_rows).map(|i| i as f32 * 0.1),
+                )) as Arc<dyn arrow_array::Array>,
+            ),
+            (
+                SCALE_FACTORS_COLUMN,
+                Arc::new(Float32Array::from_iter_values(
+                    (0..num_rows).map(|i| i as f32 * 0.01 + 0.5),
+                )) as Arc<dyn arrow_array::Array>,
+            ),
+        ])
+        .unwrap();
+
+        <RabitQuantizer as Quantization>::Storage::try_from_batch(
+            batch,
+            &metadata,
+            distance_type,
+            None,
+        )
+        .unwrap()
+    }
+
+    // ----- RabitQ tests -----------------------------------------------------
+
+    #[test]
+    fn test_roundtrip_flat_rabitq_fast() {
+        let num_rows = 50;
+        let code_dim = 64;
+        let storage = make_rabit_storage_fast(num_rows, code_dim, DistanceType::L2);
+        let entry = PartitionEntry::<FlatIndex, RabitQuantizer> {
+            index: FlatIndex::default(),
+            storage,
+        };
+
+        let mut bytes = Vec::new();
+        entry.serialize(&mut bytes).unwrap();
+        let restored =
+            PartitionEntry::<FlatIndex, RabitQuantizer>::deserialize(&bytes::Bytes::from(bytes))
+                .unwrap();
+
+        let m = entry.storage.metadata();
+        let rm = restored.storage.metadata();
+        assert_eq!(rm.num_bits, m.num_bits);
+        assert_eq!(rm.code_dim, m.code_dim);
+        assert_eq!(rm.rotation_type, m.rotation_type);
+        assert_eq!(rm.fast_rotation_signs, m.fast_rotation_signs);
+        assert!(rm.packed);
+        assert_eq!(
+            restored.storage.distance_type(),
+            entry.storage.distance_type()
+        );
+        assert_eq!(restored.storage.len(), entry.storage.len());
+
+        let orig_ids: Vec<u64> = entry.storage.row_ids().copied().collect();
+        let rest_ids: Vec<u64> = restored.storage.row_ids().copied().collect();
+        assert_eq!(orig_ids, rest_ids);
+
+        let orig_batch = entry.storage.to_batches().unwrap().next().unwrap();
+        let rest_batch = restored.storage.to_batches().unwrap().next().unwrap();
+        let orig_codes = orig_batch[RABIT_CODE_COLUMN].as_fixed_size_list();
+        let rest_codes = rest_batch[RABIT_CODE_COLUMN].as_fixed_size_list();
+        assert_eq!(
+            orig_codes.values().as_primitive::<UInt8Type>().values(),
+            rest_codes.values().as_primitive::<UInt8Type>().values(),
+        );
+    }
+
+    #[test]
+    fn test_rabitq_distance_types() {
+        for dt in [DistanceType::L2, DistanceType::Cosine, DistanceType::Dot] {
+            let storage = make_rabit_storage_fast(10, 32, dt);
+            let entry = PartitionEntry::<FlatIndex, RabitQuantizer> {
+                index: FlatIndex::default(),
+                storage,
+            };
+            let mut bytes = Vec::new();
+            entry.serialize(&mut bytes).unwrap();
+            let restored = PartitionEntry::<FlatIndex, RabitQuantizer>::deserialize(
+                &bytes::Bytes::from(bytes),
+            )
+            .unwrap();
+            assert_eq!(restored.storage.distance_type(), dt);
+        }
+    }
+
+    #[test]
+    fn test_ivf_index_state_roundtrip() {
+        use crate::index::vector::ivf::v2::{IvfIndexState, IvfStateEntryBox};
+        use lance_index::vector::flat::index::FlatQuantizer;
+        use lance_index::vector::ivf::storage::IvfModel;
+        use lance_index::vector::quantizer::QuantizationType;
+        use lance_index::vector::v3::subindex::SubIndexType;
+
+        // Build a minimal IvfModel (single centroid, dim=2).
+        let centroids =
+            FixedSizeListArray::try_new_from_values(Float32Array::from(vec![0.0f32, 1.0]), 2)
+                .unwrap();
+        let ivf = IvfModel::new(centroids, None);
+
+        let state = IvfIndexState::<FlatQuantizer> {
+            index_file_path: "my/index.lance".to_string(),
+            uuid: "test-uuid-1234".to_string(),
+            ivf: ivf.clone(),
+            aux_ivf: ivf,
+            distance_type: DistanceType::L2,
+            sub_index_metadata: vec!["meta1".to_string()],
+            metadata: lance_index::vector::flat::index::FlatMetadata { dim: 2 },
+            sub_index_type: SubIndexType::Flat,
+            quantization_type: QuantizationType::Flat,
+            cache_key_prefix: "prefix/".to_string(),
+            index_file_size: 1024,
+            aux_file_size: 512,
+        };
+
+        let entry = IvfStateEntryBox(Arc::new(state));
+
+        let mut bytes = Vec::new();
+        CacheCodecImpl::serialize(&entry, &mut bytes).unwrap();
+
+        let restored =
+            <IvfStateEntryBox as CacheCodecImpl>::deserialize(&bytes::Bytes::from(bytes.clone()))
+                .unwrap();
+
+        // Re-serialize the restored entry and compare bytes — a stronger check
+        // than field-by-field comparison and avoids needing to downcast.
+        let mut restored_bytes = Vec::new();
+        CacheCodecImpl::serialize(&restored, &mut restored_bytes).unwrap();
+        assert_eq!(bytes, restored_bytes);
+    }
+}
diff --git a/rust/lance/src/index/vector/ivf/v2.rs b/rust/lance/src/index/vector/ivf/v2.rs
index 96be37e18ed..7d38aa312c5 100644
--- a/rust/lance/src/index/vector/ivf/v2.rs
+++ b/rust/lance/src/index/vector/ivf/v2.rs
@@ -3,14 +3,12 @@
 
 //! IVF - Inverted File index.
 
+use std::io::Write as IoWrite;
 use std::marker::PhantomData;
 use std::{any::Any, collections::HashMap, sync::Arc};
 
 use crate::index::vector::{IndexFileVersion, builder::index_type_string};
-use crate::index::{
-    PreFilter,
-    vector::{VectorIndex, utils::PartitionLoadLock},
-};
+use crate::index::{PreFilter, vector::VectorIndex};
 use arrow::compute::concat_batches;
 use arrow_arith::numeric::sub;
 use arrow_array::{Float32Array, RecordBatch, UInt32Array};
@@ -18,23 +16,29 @@ use async_trait::async_trait;
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use deepsize::DeepSizeOf;
+use futures::future::BoxFuture;
 use futures::prelude::stream::{self, TryStreamExt};
 use futures::{StreamExt, TryFutureExt};
 use lance_arrow::RecordBatchExt;
-use lance_core::cache::{CacheKey, LanceCache, WeakLanceCache};
+use lance_arrow::ipc::write_len_prefixed_bytes;
+use lance_core::cache::{CacheCodec, CacheCodecImpl, CacheKey, LanceCache, WeakLanceCache};
 use lance_core::utils::tokio::spawn_cpu;
 use lance_core::utils::tracing::{IO_TYPE_LOAD_VECTOR_PART, TRACE_IO_EVENTS};
 use lance_core::{Error, ROW_ID, Result};
 use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
-use lance_file::reader::{FileReader, FileReaderOptions};
+use lance_file::LanceEncodingsIo;
+use lance_file::reader::{CachedFileMetadata, FileReader, FileReaderOptions};
 use lance_index::frag_reuse::FragReuseIndex;
 use lance_index::metrics::{LocalMetricsCollector, MetricsCollector, NoOpMetricsCollector};
 use lance_index::vector::VectorIndexCacheEntry;
-use lance_index::vector::flat::index::{FlatIndex, FlatQuantizer};
+use lance_index::vector::bq::builder::RabitQuantizer;
+use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex, FlatQuantizer};
 use lance_index::vector::hnsw::HNSW;
 use lance_index::vector::ivf::storage::IvfModel;
 use lance_index::vector::pq::ProductQuantizer;
-use lance_index::vector::quantizer::{QuantizationType, Quantizer};
+use lance_index::vector::quantizer::{
+    QuantizationType, Quantizer, QuantizerMetadata, QuantizerStorage,
+};
 use lance_index::vector::sq::ScalarQuantizer;
 use lance_index::vector::storage::VectorStore;
 use lance_index::vector::v3::subindex::SubIndexType;
@@ -60,6 +64,381 @@ use tracing::{info, instrument};
 
 use super::{IvfIndexPartitionStatistics, IvfIndexStatistics, centroids_to_vectors};
 
+/// Serializable state of an IVF index, sufficient to reconstruct the index
+/// without re-reading global buffers from object storage.
+///
+/// Serializable, type-specific state of an IVF index.
+///
+/// Generic over `Q` so that the parsed quantizer metadata (`Q::Metadata`) can
+/// be stored directly, avoiding repeated JSON round-trips on reconstruction.
+/// Produced by [`IVFIndex::to_state_entry`] and wrapped in [`IvfStateEntryBox`]
+/// for storage in the index cache.
+#[derive(Debug, Clone)]
+pub(crate) struct IvfIndexState<Q: Quantization> {
+    pub(crate) index_file_path: String,
+    pub(crate) uuid: String,
+    pub(crate) ivf: IvfModel,
+    /// IvfModel for the auxiliary/storage file (quantizer row layout).
+    /// The index and aux files have independent row layouts, so we must store
+    /// both to avoid using wrong row offsets during reconstruction.
+    pub(crate) aux_ivf: IvfModel,
+    pub(crate) distance_type: DistanceType,
+    pub(crate) sub_index_metadata: Vec<String>,
+    /// Parsed quantizer metadata — stored directly to avoid JSON re-parsing on
+    /// every warm-path reconstruction.
+    pub(crate) metadata: <Q::Storage as QuantizerStorage>::Metadata,
+    pub(crate) sub_index_type: SubIndexType,
+    pub(crate) quantization_type: QuantizationType,
+    /// The cache key prefix used by the original index's WeakLanceCache.
+    pub(crate) cache_key_prefix: String,
+    /// File sizes for the index and auxiliary files, used to avoid HEAD requests
+    /// when reconstructing from cache.
+    pub(crate) index_file_size: u64,
+    pub(crate) aux_file_size: u64,
+}
+
+impl<Q: Quantization> DeepSizeOf for IvfIndexState<Q> {
+    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+        self.index_file_path.deep_size_of_children(context)
+            + self.uuid.deep_size_of_children(context)
+            + self.ivf.deep_size_of_children(context)
+            + self.aux_ivf.deep_size_of_children(context)
+            + self.sub_index_metadata.deep_size_of_children(context)
+            + self.metadata.deep_size_of_children(context)
+            + self.cache_key_prefix.deep_size_of_children(context)
+    }
+}
+
+/// Serialization header for the `IvfIndexState` wire format.
+///
+/// Kept as a flat, non-generic struct so the JSON header format is stable
+/// regardless of `Q`. `quantizer_metadata_json` holds the serialized
+/// `Q::Metadata`; large blobs (PQ codebook, RQ matrix) follow as raw bytes.
+#[derive(serde::Serialize, serde::Deserialize)]
+struct IvfIndexStateHeader {
+    index_file_path: String,
+    uuid: String,
+    distance_type: String,
+    sub_index_metadata: Vec<String>,
+    sub_index_type: String,
+    quantization_type: String,
+    quantizer_metadata_json: String,
+    #[serde(default)]
+    cache_key_prefix: String,
+    #[serde(default)]
+    index_file_size: u64,
+    #[serde(default)]
+    aux_file_size: u64,
+}
+
+/// Object-safe interface for a type-erased `IvfIndexState<Q>`.
+///
+/// Stored as `Arc<dyn IvfStateEntry>` inside [`IvfStateEntryBox`], which is
+/// the concrete type held in the index cache. Splitting the trait from the
+/// wrapper lets the cache infrastructure work with a sized type while the
+/// hot paths call `reconstruct` without knowing `Q`.
+pub(crate) trait IvfStateEntry: DeepSizeOf + Send + Sync + 'static {
+    fn serialize_state(&self, writer: &mut dyn IoWrite) -> Result<()>;
+
+    fn reconstruct<'a>(
+        &'a self,
+        object_store: Arc<ObjectStore>,
+        file_metadata_cache: &'a LanceCache,
+        index_cache: LanceCache,
+    ) -> BoxFuture<'a, Result<Arc<dyn VectorIndex>>>;
+}
+
+/// Sized wrapper around `Arc<dyn IvfStateEntry>` for use as a cache value.
+///
+/// `IvfStateEntryBox` is the `CacheKey::ValueType` for `IvfIndexStateCacheKey`.
+/// `CacheCodecImpl` on this type holds the full deserialization dispatch
+/// (matching on `quantization_type`) so callers never need to branch on
+/// index type after a cache hit.
+pub(crate) struct IvfStateEntryBox(pub(crate) Arc<dyn IvfStateEntry>);
+
+impl DeepSizeOf for IvfStateEntryBox {
+    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+        self.0.deep_size_of_children(context)
+    }
+}
+
+/// Wire format (unchanged from the non-generic `IvfIndexState`):
+/// `[header_json_len: u64 LE][header JSON][ivf_pb_len: u64 LE][ivf protobuf]
+///  [extra_len: u64 LE][extra bytes][aux_ivf_pb_len: u64 LE][aux_ivf protobuf]`
+impl CacheCodecImpl for IvfStateEntryBox {
+    fn serialize(&self, writer: &mut dyn IoWrite) -> Result<()> {
+        self.0.serialize_state(writer)
+    }
+
+    fn deserialize(data: &bytes::Bytes) -> Result<Self> {
+        use lance_arrow::ipc::read_len_prefixed_bytes_at;
+
+        // Parse the common wire format, then dispatch on quantization_type to
+        // construct the right IvfIndexState<Q>.
+        let mut offset = 0;
+        let header_bytes = read_len_prefixed_bytes_at(data, &mut offset)?;
+        let header: IvfIndexStateHeader = serde_json::from_slice(&header_bytes)
+            .map_err(|e| lance_core::Error::io(format!("IvfIndexState header: {e}")))?;
+
+        let ivf_bytes = read_len_prefixed_bytes_at(data, &mut offset)?;
+        let ivf = IvfModel::try_from(
+            pb::Ivf::decode(ivf_bytes.as_ref())
+                .map_err(|e| lance_core::Error::io(format!("IvfIndexState IVF decode: {e}")))?,
+        )?;
+
+        let extra_bytes = read_len_prefixed_bytes_at(data, &mut offset)?;
+
+        // aux_ivf was added after initial deployment; fall back to ivf on
+        // clean EOF (legacy format without the field).
+        let aux_ivf = if offset + 8 <= data.len() {
+            let aux_ivf_bytes = read_len_prefixed_bytes_at(data, &mut offset)?;
+            IvfModel::try_from(pb::Ivf::decode(aux_ivf_bytes.as_ref()).map_err(|e| {
+                lance_core::Error::io(format!("IvfIndexState aux IVF decode: {e}"))
+            })?)?
+        } else {
+            ivf.clone()
+        };
+
+        let distance_type = DistanceType::try_from(header.distance_type.as_str())?;
+        let sub_index_type = SubIndexType::try_from(header.sub_index_type.as_str())?;
+        let quantization_type = header.quantization_type.parse::<QuantizationType>()?;
+
+        // Helper: parse Q::Metadata from the JSON+extra_bytes in the header,
+        // then build an IvfStateEntryBox wrapping IvfIndexState<Q>.
+        fn make_entry<Q: Quantization + 'static>(
+            header: IvfIndexStateHeader,
+            ivf: IvfModel,
+            aux_ivf: IvfModel,
+            extra_bytes: bytes::Bytes,
+            distance_type: DistanceType,
+            sub_index_type: SubIndexType,
+            quantization_type: QuantizationType,
+        ) -> Result<IvfStateEntryBox>
+        where
+            <Q::Storage as QuantizerStorage>::Metadata:
+                serde::de::DeserializeOwned + QuantizerMetadata,
+        {
+            let mut metadata: <Q::Storage as QuantizerStorage>::Metadata =
+                serde_json::from_str(&header.quantizer_metadata_json)
+                    .map_err(|e| lance_core::Error::io(format!("IvfIndexState metadata: {e}")))?;
+            if !extra_bytes.is_empty() {
+                metadata.parse_buffer(extra_bytes)?;
+            }
+            Ok(IvfStateEntryBox(Arc::new(IvfIndexState::<Q> {
+                index_file_path: header.index_file_path,
+                uuid: header.uuid,
+                ivf,
+                aux_ivf,
+                distance_type,
+                sub_index_metadata: header.sub_index_metadata,
+                metadata,
+                sub_index_type,
+                quantization_type,
+                cache_key_prefix: header.cache_key_prefix,
+                index_file_size: header.index_file_size,
+                aux_file_size: header.aux_file_size,
+            })))
+        }
+
+        match quantization_type {
+            QuantizationType::Flat => make_entry::<FlatQuantizer>(
+                header,
+                ivf,
+                aux_ivf,
+                extra_bytes,
+                distance_type,
+                sub_index_type,
+                quantization_type,
+            ),
+            QuantizationType::FlatBin => make_entry::<FlatBinQuantizer>(
+                header,
+                ivf,
+                aux_ivf,
+                extra_bytes,
+                distance_type,
+                sub_index_type,
+                quantization_type,
+            ),
+            QuantizationType::Product => make_entry::<ProductQuantizer>(
+                header,
+                ivf,
+                aux_ivf,
+                extra_bytes,
+                distance_type,
+                sub_index_type,
+                quantization_type,
+            ),
+            QuantizationType::Scalar => make_entry::<ScalarQuantizer>(
+                header,
+                ivf,
+                aux_ivf,
+                extra_bytes,
+                distance_type,
+                sub_index_type,
+                quantization_type,
+            ),
+            QuantizationType::Rabit => make_entry::<RabitQuantizer>(
+                header,
+                ivf,
+                aux_ivf,
+                extra_bytes,
+                distance_type,
+                sub_index_type,
+                quantization_type,
+            ),
+        }
+    }
+}
+
+impl<Q: Quantization + 'static> IvfStateEntry for IvfIndexState<Q> {
+    fn serialize_state(&self, writer: &mut dyn IoWrite) -> Result<()> {
+        let quantizer_metadata_json = serde_json::to_string(&self.metadata)
+            .map_err(|e| lance_core::Error::io(format!("IvfIndexState metadata: {e}")))?;
+        let extra = self.metadata.extra_metadata()?;
+        let extra = extra.as_deref().unwrap_or(&[]);
+
+        let header = IvfIndexStateHeader {
+            index_file_path: self.index_file_path.clone(),
+            uuid: self.uuid.clone(),
+            distance_type: self.distance_type.to_string(),
+            sub_index_metadata: self.sub_index_metadata.clone(),
+            sub_index_type: self.sub_index_type.to_string(),
+            quantization_type: self.quantization_type.to_string(),
+            quantizer_metadata_json,
+            cache_key_prefix: self.cache_key_prefix.clone(),
+            index_file_size: self.index_file_size,
+            aux_file_size: self.aux_file_size,
+        };
+        let header_json = serde_json::to_vec(&header)
+            .map_err(|e| lance_core::Error::io(format!("IvfIndexState header: {e}")))?;
+        let ivf_bytes = pb::Ivf::try_from(&self.ivf)?.encode_to_vec();
+        let aux_ivf_bytes = pb::Ivf::try_from(&self.aux_ivf)?.encode_to_vec();
+
+        write_len_prefixed_bytes(writer, &header_json)?;
+        write_len_prefixed_bytes(writer, &ivf_bytes)?;
+        write_len_prefixed_bytes(writer, extra)?;
+        write_len_prefixed_bytes(writer, &aux_ivf_bytes)?;
+        Ok(())
+    }
+
+    fn reconstruct<'a>(
+        &'a self,
+        object_store: Arc<ObjectStore>,
+        file_metadata_cache: &'a LanceCache,
+        index_cache: LanceCache,
+    ) -> BoxFuture<'a, Result<Arc<dyn VectorIndex>>> {
+        Box::pin(async move {
+            match self.sub_index_type {
+                SubIndexType::Flat => {
+                    reconstruct_typed::<FlatIndex, Q>(
+                        self,
+                        object_store,
+                        file_metadata_cache,
+                        index_cache,
+                    )
+                    .await
+                }
+                SubIndexType::Hnsw => {
+                    reconstruct_typed::<HNSW, Q>(
+                        self,
+                        object_store,
+                        file_metadata_cache,
+                        index_cache,
+                    )
+                    .await
+                }
+            }
+        })
+    }
+}
+
+struct FileMetadataCacheKey;
+
+impl CacheKey for FileMetadataCacheKey {
+    type ValueType = CachedFileMetadata;
+    fn type_name() -> &'static str {
+        "CachedFileMetadata"
+    }
+    fn key(&self) -> std::borrow::Cow<'_, str> {
+        "".into()
+    }
+}
+
+/// Cached open file readers for the index and aux files.
+///
+/// Stored in `file_metadata_cache` to avoid re-opening files on every reconstruction.
+/// Not serializable (no codec); a cache miss just triggers a re-open.
+struct CachedIndexReaders {
+    index_reader: Arc<FileReader>,
+    aux_reader: Arc<FileReader>,
+}
+
+impl deepsize::DeepSizeOf for CachedIndexReaders {
+    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+        // FileReader doesn't impl DeepSizeOf. We approximate by counting the
+        // fixed struct size for each reader plus the Arc<CachedFileMetadata>
+        // heap contents. The metadata Arcs are also held by FileMetadataCacheKey
+        // entries, so this may over-count across cache entries, but
+        // over-counting is safer than under-counting for eviction purposes.
+        std::mem::size_of::<FileReader>() * 2
+            + self.index_reader.metadata().deep_size_of_children(context)
+            + self.aux_reader.metadata().deep_size_of_children(context)
+    }
+}
+
+struct CachedIndexReadersKey {
+    uuid: String,
+}
+
+impl CacheKey for CachedIndexReadersKey {
+    type ValueType = CachedIndexReaders;
+    fn type_name() -> &'static str {
+        "CachedIndexReaders"
+    }
+    fn key(&self) -> std::borrow::Cow<'_, str> {
+        self.uuid.as_str().into()
+    }
+    // No codec() override → in-memory only
+}
+
+/// Open a FileReader, reusing cached file metadata if available.
+async fn open_reader_cached(
+    scheduler: &Arc<ScanScheduler>,
+    path: &Path,
+    cache: &LanceCache,
+    known_file_size: u64,
+) -> Result<FileReader> {
+    let file_cache = cache.with_key_prefix(path.as_ref());
+    // CachedFileSize::new(0) == CachedFileSize::unknown(); passing the raw
+    // hint directly is safe — the type already encodes 0 as "unknown".
+    let cached_size = CachedFileSize::new(known_file_size);
+
+    if let Some(cached_meta) = file_cache.get_with_key(&FileMetadataCacheKey).await {
+        let file_scheduler = scheduler.open_file(path, &cached_size).await?;
+        let encodings_io = Arc::new(LanceEncodingsIo::new(file_scheduler));
+        FileReader::try_open_with_file_metadata(
+            encodings_io,
+            path.clone(),
+            None,
+            Arc::<DecoderPlugins>::default(),
+            cached_meta,
+            cache,
+            FileReaderOptions::default(),
+        )
+        .await
+    } else {
+        let file_scheduler = scheduler.open_file(path, &cached_size).await?;
+        FileReader::try_open(
+            file_scheduler,
+            None,
+            Arc::<DecoderPlugins>::default(),
+            cache,
+            FileReaderOptions::default(),
+        )
+        .await
+    }
+}
+
 #[derive(Debug, DeepSizeOf)]
 pub struct PartitionEntry<S: IvfSubIndex, Q: Quantization> {
     pub index: S,
@@ -96,12 +475,26 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> CacheKey for IVFPartit
     fn key(&self) -> std::borrow::Cow<'_, str> {
         format!("ivf-{}", self.partition_id).into()
     }
+
+    fn type_name() -> &'static str {
+        // Using type_name is safe here: the impl is in the same crate as the
+        // types, so the monomorphized pointer is consistent.
+        std::any::type_name::<PartitionEntry<S, Q>>()
+    }
+
+    fn codec() -> Option<CacheCodec> {
+        super::partition_serde::partition_entry_codec::<S, Q>()
+    }
 }
 
 /// IVF Index.
 #[derive(Debug)]
 pub struct IVFIndex<S: IvfSubIndex + 'static, Q: Quantization + 'static> {
+    /// Local display path (via `to_local_path`), used for statistics.
     uri: String,
+    /// Object-store path to the index file (forward-slash separated).
+    /// Used by `cacheable_state()` for cross-platform reconstruction.
+    index_path: String,
     uuid: String,
 
     /// Ivf model
@@ -111,8 +504,6 @@ pub struct IVFIndex<S: IvfSubIndex + 'static, Q: Quantization + 'static> {
     sub_index_metadata: Vec<String>,
     storage: IvfQuantizationStorage<Q>,
 
-    partition_locks: PartitionLoadLock,
-
     distance_type: DistanceType,
 
     index_cache: WeakLanceCache,
@@ -125,6 +516,7 @@ pub struct IVFIndex<S: IvfSubIndex + 'static, Q: Quantization + 'static> {
 impl<S: IvfSubIndex, Q: Quantization> DeepSizeOf for IVFIndex<S, Q> {
     fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
         self.uri.deep_size_of_children(context)
+            + self.index_path.deep_size_of_children(context)
             + self.ivf.deep_size_of_children(context)
             + self.sub_index_metadata.deep_size_of_children(context)
             + self.uuid.deep_size_of_children(context)
@@ -142,16 +534,19 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
         frag_reuse_index: Option<Arc<FragReuseIndex>>,
         file_metadata_cache: &LanceCache,
         index_cache: LanceCache,
+        file_sizes: HashMap<String, u64>,
     ) -> Result<Self> {
         let io_parallelism = object_store.io_parallelism();
         let scheduler_config = SchedulerConfig::max_bandwidth(&object_store);
         let scheduler = ScanScheduler::new(object_store, scheduler_config);
 
         let uri = index_dir.child(uuid.as_str()).child(INDEX_FILE_NAME);
+        let cached_size = file_sizes
+            .get(INDEX_FILE_NAME)
+            .map(|&size| CachedFileSize::new(size))
+            .unwrap_or_else(CachedFileSize::unknown);
         let index_reader = FileReader::try_open(
-            scheduler
-                .open_file(&uri, &CachedFileSize::unknown())
-                .await?,
+            scheduler.open_file(&uri, &cached_size).await?,
             None,
             Arc::<DecoderPlugins>::default(),
             file_metadata_cache,
@@ -185,13 +580,17 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
             .ok_or(Error::index(format!("{} not found", S::metadata_key())))?;
         let sub_index_metadata: Vec<String> = serde_json::from_str(sub_index_metadata)?;
 
+        let aux_cached_size = file_sizes
+            .get(INDEX_AUXILIARY_FILE_NAME)
+            .map(|&size| CachedFileSize::new(size))
+            .unwrap_or_else(CachedFileSize::unknown);
         let storage_reader = FileReader::try_open(
             scheduler
                 .open_file(
                     &index_dir
                         .child(uuid.as_str())
                         .child(INDEX_AUXILIARY_FILE_NAME),
-                    &CachedFileSize::unknown(),
+                    &aux_cached_size,
                 )
                 .await?,
             None,
@@ -203,14 +602,38 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
         let storage =
             IvfQuantizationStorage::try_new(storage_reader, frag_reuse_index.clone()).await?;
 
-        let num_partitions = ivf.num_partitions();
+        // Cache file metadata so reconstructions from IvfIndexState can skip
+        // footer reads.
+        file_metadata_cache
+            .with_key_prefix(uri.as_ref())
+            .insert_with_key(&FileMetadataCacheKey, index_reader.metadata().clone())
+            .await;
+        let aux_path = index_dir
+            .child(uuid.as_str())
+            .child(INDEX_AUXILIARY_FILE_NAME);
+        file_metadata_cache
+            .with_key_prefix(aux_path.as_ref())
+            .insert_with_key(&FileMetadataCacheKey, storage.reader().metadata().clone())
+            .await;
+
+        // Cache open readers so the first reconstruction also skips file opens.
+        file_metadata_cache
+            .insert_with_key(
+                &CachedIndexReadersKey { uuid: uuid.clone() },
+                Arc::new(CachedIndexReaders {
+                    index_reader: Arc::new(index_reader.clone()),
+                    aux_reader: Arc::new(storage.reader().clone()),
+                }),
+            )
+            .await;
+
         Ok(Self {
             uri: to_local_path(&uri),
+            index_path: uri.as_ref().to_string(),
             uuid,
             ivf,
             reader: index_reader,
             storage,
-            partition_locks: PartitionLoadLock::new(num_partitions),
             sub_index_metadata,
             distance_type,
             index_cache: WeakLanceCache::from(&index_cache),
@@ -219,6 +642,35 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
         })
     }
 
+    /// Reconstruct an IVFIndex from pre-parsed state without any I/O.
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) fn from_cached_state(
+        uri: String,
+        index_path: String,
+        uuid: String,
+        ivf: IvfModel,
+        reader: FileReader,
+        storage: IvfQuantizationStorage<Q>,
+        sub_index_metadata: Vec<String>,
+        distance_type: DistanceType,
+        index_cache: LanceCache,
+        io_parallelism: usize,
+    ) -> Self {
+        Self {
+            uri,
+            index_path,
+            uuid,
+            ivf,
+            reader,
+            storage,
+            sub_index_metadata,
+            distance_type,
+            index_cache: WeakLanceCache::from(&index_cache),
+            io_parallelism,
+            _marker: PhantomData,
+        }
+    }
+
     #[instrument(level = "debug", skip(self, metrics))]
     pub async fn load_partition(
         &self,
@@ -226,71 +678,69 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
         write_cache: bool,
         metrics: &dyn MetricsCollector,
     ) -> Result<Arc<dyn VectorIndexCacheEntry>> {
+        if partition_id >= self.ivf.num_partitions() {
+            return Err(Error::index(format!(
+                "partition id {} is out of range of {} partitions",
+                partition_id,
+                self.ivf.num_partitions()
+            )));
+        }
+
         let cache_key = IVFPartitionKey::<S, Q>::new(partition_id);
-        let part_entry = if let Some(part_idx) = self.index_cache.get_with_key(&cache_key).await {
-            part_idx
+
+        if write_cache {
+            let entry = self
+                .index_cache
+                .get_or_insert_with_key(cache_key, || async {
+                    info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_VECTOR_PART, index_type="ivf", part_id=partition_id);
+                    metrics.record_part_load();
+                    self.load_partition_entry(partition_id).await
+                })
+                .await?;
+            Ok(entry as Arc<dyn VectorIndexCacheEntry>)
         } else {
-            info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_VECTOR_PART, index_type="ivf", part_id=cache_key.key().as_ref());
-            metrics.record_part_load();
-            if partition_id >= self.ivf.num_partitions() {
-                return Err(Error::index(format!(
-                    "partition id {} is out of range of {} partitions",
-                    partition_id,
-                    self.ivf.num_partitions()
-                )));
+            if let Some(part_idx) = self.index_cache.get_with_key(&cache_key).await {
+                return Ok(part_idx);
             }
+            info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_VECTOR_PART, index_type="ivf", part_id=partition_id);
+            metrics.record_part_load();
+            Ok(Arc::new(self.load_partition_entry(partition_id).await?))
+        }
+    }
 
-            let mtx = self.partition_locks.get_partition_mutex(partition_id);
-            let _guard = mtx.lock().await;
-
-            // check the cache again, as the partition may have been loaded by another
-            // thread that held the lock on loading the partition
-            if let Some(part_idx) = self.index_cache.get_with_key(&cache_key).await {
-                part_idx
-            } else {
-                let schema = Arc::new(self.reader.schema().as_ref().into());
-                let batch = match self.reader.metadata().num_rows {
-                    0 => RecordBatch::new_empty(schema),
-                    _ => {
-                        let row_range = self.ivf.row_range(partition_id);
-                        if row_range.is_empty() {
-                            RecordBatch::new_empty(schema)
-                        } else {
-                            let batches = self
-                                .reader
-                                .read_stream(
-                                    ReadBatchParams::Range(row_range),
-                                    u32::MAX,
-                                    1,
-                                    FilterExpression::no_filter(),
-                                )?
-                                .try_collect::<Vec<_>>()
-                                .await?;
-                            concat_batches(&schema, batches.iter())?
-                        }
-                    }
-                };
-                let batch = batch.add_metadata(
-                    S::metadata_key().to_owned(),
-                    self.sub_index_metadata[partition_id].clone(),
-                )?;
-                let idx = S::load(batch)?;
-                let storage = self.load_partition_storage(partition_id).await?;
-                let partition_entry = Arc::new(PartitionEntry::<S, Q> {
-                    index: idx,
-                    storage,
-                });
-                if write_cache {
-                    self.index_cache
-                        .insert_with_key(&cache_key, partition_entry.clone())
-                        .await;
+    async fn load_partition_entry(&self, partition_id: usize) -> Result<PartitionEntry<S, Q>> {
+        let schema = Arc::new(self.reader.schema().as_ref().into());
+        let batch = match self.reader.metadata().num_rows {
+            0 => RecordBatch::new_empty(schema),
+            _ => {
+                let row_range = self.ivf.row_range(partition_id);
+                if row_range.is_empty() {
+                    RecordBatch::new_empty(schema)
+                } else {
+                    let batches = self
+                        .reader
+                        .read_stream(
+                            ReadBatchParams::Range(row_range),
+                            u32::MAX,
+                            1,
+                            FilterExpression::no_filter(),
+                        )?
+                        .try_collect::<Vec<_>>()
+                        .await?;
+                    concat_batches(&schema, batches.iter())?
                 }
-
-                partition_entry
             }
         };
-
-        Ok(part_entry)
+        let batch = batch.add_metadata(
+            S::metadata_key().to_owned(),
+            self.sub_index_metadata[partition_id].clone(),
+        )?;
+        let idx = S::load(batch)?;
+        let storage = self.load_partition_storage(partition_id).await?;
+        Ok(PartitionEntry {
+            index: idx,
+            storage,
+        })
     }
 
     pub async fn load_partition_storage(&self, partition_id: usize) -> Result<Q::Storage> {
@@ -317,6 +767,25 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
             Ok(query.clone())
         }
     }
+
+    /// Export the index state needed for reconstruction from a disk cache.
+    pub(crate) fn to_state_entry(&self) -> IvfStateEntryBox {
+        let (sub_index_type, quantization_type) = self.sub_index_type();
+        IvfStateEntryBox(Arc::new(IvfIndexState::<Q> {
+            index_file_path: self.index_path.clone(),
+            uuid: self.uuid.clone(),
+            ivf: self.ivf.clone(),
+            aux_ivf: self.storage.ivf().clone(),
+            distance_type: self.distance_type,
+            sub_index_metadata: self.sub_index_metadata.clone(),
+            metadata: self.storage.metadata().clone(),
+            sub_index_type,
+            quantization_type,
+            cache_key_prefix: self.index_cache.prefix().to_string(),
+            index_file_size: self.reader.metadata().file_size(),
+            aux_file_size: self.storage.reader().metadata().file_size(),
+        }))
+    }
 }
 
 #[async_trait]
@@ -345,13 +814,15 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> Index for IVFIndex<S,
 
     fn index_type(&self) -> IndexType {
         match self.sub_index_type() {
-            (SubIndexType::Flat, QuantizationType::Flat) => IndexType::IvfFlat,
+            (SubIndexType::Flat, QuantizationType::Flat)
+            | (SubIndexType::Flat, QuantizationType::FlatBin) => IndexType::IvfFlat,
             (SubIndexType::Flat, QuantizationType::Product) => IndexType::IvfPq,
             (SubIndexType::Flat, QuantizationType::Scalar) => IndexType::IvfSq,
             (SubIndexType::Flat, QuantizationType::Rabit) => IndexType::IvfRq,
             (SubIndexType::Hnsw, QuantizationType::Product) => IndexType::IvfHnswPq,
             (SubIndexType::Hnsw, QuantizationType::Scalar) => IndexType::IvfHnswSq,
-            (SubIndexType::Hnsw, QuantizationType::Flat) => IndexType::IvfHnswFlat,
+            (SubIndexType::Hnsw, QuantizationType::Flat)
+            | (SubIndexType::Hnsw, QuantizationType::FlatBin) => IndexType::IvfHnswFlat,
             (sub_index_type, quantization_type) => {
                 unimplemented!(
                     "unsupported index type: {}, {}",
@@ -386,10 +857,12 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> Index for IVFIndex<S,
 
         sub_index_stats.append(store_stats);
         if S::name() == "FLAT" {
-            sub_index_stats.insert(
-                "index_type".to_string(),
-                Q::quantization_type().to_string().into(),
-            );
+            let qt_label = match Q::quantization_type() {
+                // FlatBin is the Hamming variant of Flat; report as "FLAT".
+                QuantizationType::FlatBin => "FLAT".to_string(),
+                other => other.to_string(),
+            };
+            sub_index_stats.insert("index_type".to_string(), qt_label.into());
         } else {
             sub_index_stats.insert("index_type".to_string(), S::name().into());
         }
@@ -596,6 +1069,84 @@ pub type IvfPq = IVFIndex<FlatIndex, ProductQuantizer>;
 pub type IvfHnswSqIndex = IVFIndex<HNSW, ScalarQuantizer>;
 pub type IvfHnswPqIndex = IVFIndex<HNSW, ProductQuantizer>;
 
+async fn reconstruct_typed<S: IvfSubIndex + 'static, Q: Quantization + 'static>(
+    state: &IvfIndexState<Q>,
+    object_store: Arc<ObjectStore>,
+    file_metadata_cache: &LanceCache,
+    index_cache: LanceCache,
+) -> Result<Arc<dyn VectorIndex>> {
+    let io_parallelism = object_store.io_parallelism();
+
+    let index_path = Path::parse(&state.index_file_path)
+        .map_err(|e| Error::io(format!("invalid index path: {e}")))?;
+
+    // Derive aux path from the index path's parent directory.
+    let mut parts: Vec<_> = index_path.parts().collect();
+    parts.pop();
+    let dir: Path = parts.into_iter().collect();
+    let aux_path = dir.child(INDEX_AUXILIARY_FILE_NAME);
+
+    let readers_key = CachedIndexReadersKey {
+        uuid: state.uuid.clone(),
+    };
+
+    let (index_reader, aux_reader) =
+        if let Some(cached) = file_metadata_cache.get_with_key(&readers_key).await {
+            // Warm path: reuse the cached readers directly, no file opens needed.
+            ((*cached.index_reader).clone(), (*cached.aux_reader).clone())
+        } else {
+            // Cold path: open files, then cache the readers for future reconstructions.
+            let scheduler_config = SchedulerConfig::max_bandwidth(&object_store);
+            let scheduler = ScanScheduler::new(object_store, scheduler_config);
+            let index_reader = open_reader_cached(
+                &scheduler,
+                &index_path,
+                file_metadata_cache,
+                state.index_file_size,
+            )
+            .await?;
+            let aux_reader = open_reader_cached(
+                &scheduler,
+                &aux_path,
+                file_metadata_cache,
+                state.aux_file_size,
+            )
+            .await?;
+            file_metadata_cache
+                .insert_with_key(
+                    &readers_key,
+                    Arc::new(CachedIndexReaders {
+                        index_reader: Arc::new(index_reader.clone()),
+                        aux_reader: Arc::new(aux_reader.clone()),
+                    }),
+                )
+                .await;
+            (index_reader, aux_reader)
+        };
+
+    let storage = IvfQuantizationStorage::from_cached(
+        aux_reader,
+        state.aux_ivf.clone(),
+        state.metadata.clone(),
+        state.distance_type,
+        None,
+    );
+
+    let index = IVFIndex::<S, Q>::from_cached_state(
+        to_local_path(&index_path),
+        index_path.to_string(),
+        state.uuid.clone(),
+        state.ivf.clone(),
+        index_reader,
+        storage,
+        state.sub_index_metadata.clone(),
+        state.distance_type,
+        index_cache,
+        io_parallelism,
+    );
+    Ok(Arc::new(index))
+}
+
 #[cfg(test)]
 mod tests {
     use std::collections::HashSet;
@@ -619,8 +1170,8 @@ mod tests {
     use lance_index::vector::storage::VectorStore;
 
     use crate::dataset::{InsertBuilder, UpdateBuilder, WriteMode, WriteParams};
+    use crate::index::DatasetIndexExt;
     use crate::index::DatasetIndexInternalExt;
-    use crate::index::vector::ivf::finalize_distributed_merge;
     use crate::index::vector::ivf::v2::IvfPq;
     use crate::utils::test::copy_test_data_to_tmp;
     use crate::{
@@ -637,6 +1188,8 @@ mod tests {
     use lance_encoding::decoder::DecoderPlugins;
     use lance_file::reader::{FileReader, FileReaderOptions};
     use lance_file::writer::FileWriter;
+    use lance_index::IndexType;
+    use lance_index::progress::IndexBuildProgress;
     use lance_index::vector::DIST_COL;
     use lance_index::vector::ivf::IvfBuildParams;
     use lance_index::vector::kmeans::{KMeansParams, train_kmeans};
@@ -644,9 +1197,10 @@ mod tests {
     use lance_index::vector::quantizer::QuantizerMetadata;
     use lance_index::vector::sq::builder::SQBuildParams;
     use lance_index::vector::{
-        pq::storage::ProductQuantizationMetadata, storage::STORAGE_METADATA_KEY,
+        pq::storage::ProductQuantizationMetadata,
+        sq::storage::{SQ_METADATA_KEY, ScalarQuantizationMetadata},
+        storage::STORAGE_METADATA_KEY,
     };
-    use lance_index::{DatasetIndexExt, IndexType};
     use lance_index::{INDEX_AUXILIARY_FILE_NAME, metrics::NoOpMetricsCollector};
     use lance_index::{optimize::OptimizeOptions, scalar::IndexReader};
     use lance_index::{scalar::IndexWriter, vector::hnsw::builder::HnswBuildParams};
@@ -657,16 +1211,17 @@ mod tests {
     };
     use lance_linalg::distance::{DistanceType, multivec_distance};
     use lance_linalg::kernels::normalize_fsl;
+    use lance_table::format::IndexMetadata;
     use lance_testing::datagen::{generate_random_array, generate_random_array_with_range};
-    use object_store::path::Path;
     use rand::distr::uniform::SampleUniform;
     use rand::{Rng, SeedableRng, rngs::StdRng};
     use rstest::rstest;
-    use uuid::Uuid;
 
     const NUM_ROWS: usize = 512;
     const DIM: usize = 32;
 
+    lance_testing::define_stage_event_progress!(RecordingProgress, IndexBuildProgress, Result<()>);
+
     async fn generate_test_dataset<T: ArrowPrimitiveType>(
         test_uri: &str,
         range: Range<T::Native>,
@@ -752,6 +1307,37 @@ mod tests {
         serde_json::from_str(&metadata_entries[0]).unwrap()
     }
 
+    async fn get_sq_metadata(
+        dataset: &Dataset,
+        scheduler: Arc<ScanScheduler>,
+        index_uuid: &str,
+    ) -> ScalarQuantizationMetadata {
+        let index_path = dataset
+            .indices_dir()
+            .child(index_uuid)
+            .child(INDEX_AUXILIARY_FILE_NAME);
+        let file_scheduler = scheduler
+            .open_file(&index_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let reader = FileReader::try_open(
+            file_scheduler,
+            None,
+            Arc::<DecoderPlugins>::default(),
+            &LanceCache::no_cache(),
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+        if let Some(metadata) = reader.schema().metadata.get(SQ_METADATA_KEY) {
+            serde_json::from_str(metadata).unwrap()
+        } else {
+            let metadata = reader.schema().metadata.get(STORAGE_METADATA_KEY).unwrap();
+            let metadata_entries: Vec<String> = serde_json::from_str(metadata).unwrap();
+            serde_json::from_str(&metadata_entries[0]).unwrap()
+        }
+    }
+
     async fn assert_rq_rotation_type(dataset: &Dataset, expected: RQRotationType) {
         let obj_store = Arc::new(ObjectStore::local());
         let scheduler = ScanScheduler::new(obj_store, SchedulerConfig::default_for_testing());
@@ -933,6 +1519,49 @@ mod tests {
         )
     }
 
+    fn make_fragment_offset_batches(
+        rows_per_fragment: usize,
+        offsets: &[f32],
+    ) -> (Arc<Schema>, Vec<RecordBatch>) {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt64, false),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(
+                    Arc::new(Field::new("item", DataType::Float32, true)),
+                    DIM as i32,
+                ),
+                false,
+            ),
+        ]));
+
+        let mut next_id = 0_u64;
+        let batches = offsets
+            .iter()
+            .map(|offset| {
+                let ids = Arc::new(UInt64Array::from_iter_values(
+                    next_id..next_id + rows_per_fragment as u64,
+                ));
+                next_id += rows_per_fragment as u64;
+
+                let mut values = Vec::with_capacity(rows_per_fragment * DIM);
+                for _ in 0..rows_per_fragment {
+                    for dim in 0..DIM {
+                        values.push(*offset + dim as f32);
+                    }
+                }
+
+                let vectors = Arc::new(
+                    FixedSizeListArray::try_new_from_values(Float32Array::from(values), DIM as i32)
+                        .unwrap(),
+                );
+                RecordBatch::try_new(schema.clone(), vec![ids, vectors]).unwrap()
+            })
+            .collect();
+
+        (schema, batches)
+    }
+
     struct VectorIndexTestContext {
         stats_json: String,
         stats: serde_json::Value,
@@ -1023,10 +1652,9 @@ mod tests {
 
         let dataset = Dataset::open(test_uri).await.unwrap();
         let final_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await;
-        assert_eq!(
-            final_ctx.num_partitions(),
-            3,
-            "Expected partition split to increase partitions from 2 to 3 for {}, got stats: {}",
+        assert!(
+            final_ctx.num_partitions() >= 3,
+            "Expected partition split to increase partitions beyond 2 for {}, got stats: {}",
             description,
             final_ctx.stats_json()
         );
@@ -1103,6 +1731,51 @@ mod tests {
         append_constant_vector_with_params(dataset, rows, template, None).await;
     }
 
+    async fn append_partition_templates(
+        dataset: &mut Dataset,
+        rows_per_template: usize,
+        templates: &[Vec<f32>],
+    ) {
+        assert!(
+            !templates.is_empty(),
+            "at least one template is required for append"
+        );
+        for template in templates {
+            assert_eq!(
+                template.len(),
+                DIM,
+                "Template vector should have {} dimensions",
+                DIM
+            );
+        }
+
+        let start_id = dataset.count_all_rows().await.unwrap() as u64;
+        let total_rows = rows_per_template * templates.len();
+        let ids = Arc::new(UInt64Array::from_iter_values(
+            start_id..start_id + total_rows as u64,
+        ));
+        let mut appended_values = Vec::with_capacity(total_rows * DIM);
+        for template in templates {
+            for _ in 0..rows_per_template {
+                appended_values.extend_from_slice(template);
+            }
+        }
+        let vectors = Arc::new(
+            FixedSizeListArray::try_new_from_values(
+                Float32Array::from(appended_values),
+                DIM as i32,
+            )
+            .unwrap(),
+        );
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt64, false),
+            Field::new("vector", vectors.data_type().clone(), false),
+        ]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![ids, vectors]).unwrap();
+        let batches = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        dataset.append(batches, None).await.unwrap();
+    }
+
     async fn append_constant_vector_with_params(
         dataset: &mut Dataset,
         rows: usize,
@@ -1281,7 +1954,6 @@ mod tests {
         .unwrap();
     }
 
-    #[allow(dead_code)]
     async fn ground_truth(
         dataset: &Dataset,
         column: &str,
@@ -1307,7 +1979,6 @@ mod tests {
             .collect()
     }
 
-    #[allow(dead_code)]
     fn multivec_ground_truth(
         vectors: &ListArray,
         query: &dyn Array,
@@ -1430,50 +2101,84 @@ mod tests {
         (ivf_params, pq_params)
     }
 
-    async fn build_ivfpq_for_fragment_groups(
-        dataset: &mut Dataset,
-        fragment_groups: Vec<Vec<u32>>, // each group is a set of fragment ids
-        ivf_params: &IvfBuildParams,
-        pq_params: &PQBuildParams,
-        index_name: &str,
-    ) {
-        let shared_uuid = Uuid::new_v4();
+    async fn prepare_global_ivf(dataset: &Dataset, vector_column: &str) -> IvfBuildParams {
+        let batch = dataset
+            .scan()
+            .project(&[vector_column.to_string()])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let vectors = batch
+            .column_by_name(vector_column)
+            .expect("vector column should exist")
+            .as_fixed_size_list();
+
+        let dim = vectors.value_length() as usize;
+        assert_eq!(dim, TWO_FRAG_DIM, "unexpected vector dimension");
+
+        let values = vectors.values().as_primitive::<Float32Type>();
+        let kmeans_params = KMeansParams::new(None, TWO_FRAG_MAX_ITERS, 1, DistanceType::L2);
+        let kmeans = train_kmeans::<Float32Type>(
+            values,
+            kmeans_params,
+            dim,
+            TWO_FRAG_NUM_PARTITIONS,
+            TWO_FRAG_SAMPLE_RATE,
+        )
+        .unwrap();
+
+        let centroids_flat = kmeans.centroids.as_primitive::<Float32Type>().clone();
+        let centroids_fsl =
+            Arc::new(FixedSizeListArray::try_new_from_values(centroids_flat, dim as i32).unwrap());
+        let mut ivf_params =
+            IvfBuildParams::try_with_centroids(TWO_FRAG_NUM_PARTITIONS, centroids_fsl).unwrap();
+        ivf_params.max_iters = TWO_FRAG_MAX_ITERS as usize;
+        ivf_params.sample_rate = TWO_FRAG_SAMPLE_RATE;
+        ivf_params
+    }
+
+    async fn build_segments_for_fragment_groups(
+        dataset: &mut Dataset,
+        fragment_groups: Vec<Vec<u32>>, // each group is a set of fragment ids
+        params: &VectorIndexParams,
+        index_name: &str,
+    ) -> Vec<IndexMetadata> {
+        let mut segments = Vec::new();
+
+        for fragments in fragment_groups {
+            let mut builder = dataset.create_index_builder(&["vector"], IndexType::Vector, params);
+            builder = builder.name(index_name.to_string()).fragments(fragments);
+            segments.push(builder.execute_uncommitted().await.unwrap());
+        }
+
+        segments
+    }
+
+    async fn build_ivfpq_for_fragment_groups(
+        dataset: &mut Dataset,
+        fragment_groups: Vec<Vec<u32>>, // each group is a set of fragment ids
+        ivf_params: &IvfBuildParams,
+        pq_params: &PQBuildParams,
+        index_name: &str,
+    ) {
         let params = VectorIndexParams::with_ivf_pq_params(
             DistanceType::L2,
             ivf_params.clone(),
             pq_params.clone(),
         );
 
-        for fragments in fragment_groups {
-            let mut builder = dataset.create_index_builder(&["vector"], IndexType::Vector, &params);
-            builder = builder
-                .name(index_name.to_string())
-                .fragments(fragments)
-                .index_uuid(shared_uuid.to_string());
-            // Build partial index shards without committing to manifest.
-            builder.execute_uncommitted().await.unwrap();
-        }
-
-        let index_dir = dataset.indices_dir().child(shared_uuid.to_string());
-        finalize_distributed_merge(dataset.object_store(), &index_dir, Some(IndexType::IvfPq))
-            .await
-            .unwrap();
-
-        dataset
-            .commit_existing_index(index_name, "vector", shared_uuid)
-            .await
-            .unwrap();
+        let segments =
+            build_segments_for_fragment_groups(dataset, fragment_groups, &params, index_name).await;
+        let committed_segments = build_distributed_segments(dataset, segments, index_name).await;
+        assert!(!committed_segments.is_empty());
     }
 
-    fn assert_ivf_layout_equal(stats_a: &serde_json::Value, stats_b: &serde_json::Value) {
-        let idx_a = &stats_a["indices"][0];
-        let idx_b = &stats_b["indices"][0];
-
-        // Centroids: same shape and values (within tolerance).
-        let centroids_a = idx_a["centroids"]
+    fn assert_centroids_equal(reference: &serde_json::Value, candidate: &serde_json::Value) {
+        let centroids_a = reference["centroids"]
             .as_array()
             .expect("centroids should be an array");
-        let centroids_b = idx_b["centroids"]
+        let centroids_b = candidate["centroids"]
             .as_array()
             .expect("centroids should be an array");
         assert_eq!(
@@ -1500,24 +2205,59 @@ mod tests {
                 );
             }
         }
+    }
+
+    fn sum_partition_sizes(indices: &[serde_json::Value]) -> Vec<u64> {
+        let mut totals = Vec::new();
+        for index in indices {
+            let partitions = index["partitions"]
+                .as_array()
+                .expect("partitions should be an array");
+            if totals.is_empty() {
+                totals.resize(partitions.len(), 0);
+            } else {
+                assert_eq!(totals.len(), partitions.len(), "num partitions mismatch");
+            }
+            for (total, partition) in totals.iter_mut().zip(partitions.iter()) {
+                *total += partition["size"].as_u64().expect("partition size");
+            }
+        }
+        totals
+    }
 
-        // Partitions sizes.
-        let parts_a = idx_a["partitions"]
+    fn assert_ivf_layout_compatible(stats_a: &serde_json::Value, stats_b: &serde_json::Value) {
+        let indices_a = stats_a["indices"]
             .as_array()
-            .expect("partitions should be an array");
-        let parts_b = idx_b["partitions"]
+            .expect("indices should be an array");
+        let indices_b = stats_b["indices"]
             .as_array()
-            .expect("partitions should be an array");
-        assert_eq!(parts_a.len(), parts_b.len(), "num partitions mismatch");
-        let sizes_a: Vec<u64> = parts_a
-            .iter()
-            .map(|p| p["size"].as_u64().expect("partition size"))
-            .collect();
-        let sizes_b: Vec<u64> = parts_b
-            .iter()
-            .map(|p| p["size"].as_u64().expect("partition size"))
-            .collect();
-        assert_eq!(sizes_a, sizes_b, "partition sizes mismatch");
+            .expect("indices should be an array");
+        assert!(
+            !indices_a.is_empty() && !indices_b.is_empty(),
+            "indices should not be empty",
+        );
+
+        let reference = &indices_a[0];
+        for index in indices_a.iter().skip(1).chain(indices_b.iter()) {
+            assert_centroids_equal(reference, index);
+        }
+
+        let sizes_a = sum_partition_sizes(indices_a);
+        let sizes_b = sum_partition_sizes(indices_b);
+        assert_eq!(sizes_a, sizes_b, "aggregated partition sizes mismatch");
+    }
+
+    /// Commit caller-defined segment groups as one logical index.
+    async fn build_distributed_segments(
+        dataset: &mut Dataset,
+        segments: Vec<IndexMetadata>,
+        index_name: &str,
+    ) -> Vec<IndexMetadata> {
+        dataset
+            .commit_existing_index_segments(index_name, "vector", segments.clone())
+            .await
+            .unwrap();
+        segments
     }
 
     #[tokio::test]
@@ -1527,7 +2267,6 @@ mod tests {
         let test_dir = TempStrDir::default();
         let base_uri = test_dir.as_str();
 
-        // Generate the data once, then write it twice to two independent dataset URIs.
         let (schema, batches) = make_two_fragment_batches();
 
         let ds_single_uri = format!("{}/single", base_uri);
@@ -1537,7 +2276,6 @@ mod tests {
             write_dataset_from_batches(&ds_single_uri, schema.clone(), batches.clone()).await;
         let mut ds_split = write_dataset_from_batches(&ds_split_uri, schema, batches).await;
 
-        // Ensure we have at least 2 fragments.
         let fragments_single = ds_single.get_fragments();
         assert!(
             fragments_single.len() >= 2,
@@ -1551,10 +2289,8 @@ mod tests {
             fragments_split.len()
         );
 
-        // Pretrain global IVF centroids and PQ codebook.
         let (ivf_params, pq_params) = prepare_global_ivf_pq(&ds_single, "vector").await;
 
-        // Build single index using two fragments in one distributed build.
         let group_single = vec![
             fragments_single[0].id() as u32,
             fragments_single[1].id() as u32,
@@ -1568,7 +2304,6 @@ mod tests {
         )
         .await;
 
-        // Build split index: one fragment per distributed build, then merge.
         let group0 = vec![fragments_split[0].id() as u32];
         let group1 = vec![fragments_split[1].id() as u32];
         build_ivfpq_for_fragment_groups(
@@ -1580,38 +2315,364 @@ mod tests {
         )
         .await;
 
-        // Compare IVF layout via index statistics.
         let stats_single_json = ds_single.index_statistics(INDEX_NAME).await.unwrap();
         let stats_split_json = ds_split.index_statistics(INDEX_NAME).await.unwrap();
         let stats_single: serde_json::Value = serde_json::from_str(&stats_single_json).unwrap();
         let stats_split: serde_json::Value = serde_json::from_str(&stats_split_json).unwrap();
-        assert_ivf_layout_equal(&stats_single, &stats_split);
-
-        // Compare row id sets per partition.
-        let ctx_single = load_vector_index_context(&ds_single, "vector", INDEX_NAME).await;
-        let ctx_split = load_vector_index_context(&ds_split, "vector", INDEX_NAME).await;
-
-        let ivf_single = ctx_single.ivf();
-        let ivf_split = ctx_split.ivf();
-        let total_partitions = ivf_single.total_partitions();
-        assert_eq!(total_partitions, ivf_split.total_partitions());
-
-        for part_id in 0..total_partitions {
-            let row_ids_single = load_partition_row_ids(ivf_single, part_id).await;
-            let row_ids_split = load_partition_row_ids(ivf_split, part_id).await;
-            let set_single: HashSet<u64> = row_ids_single.into_iter().collect();
-            let set_split: HashSet<u64> = row_ids_split.into_iter().collect();
+        assert_ivf_layout_compatible(&stats_single, &stats_split);
+        assert_eq!(
+            stats_single["num_indexed_rows"],
+            stats_split["num_indexed_rows"]
+        );
+
+        const K: usize = 10;
+        const NUM_QUERIES: usize = 10;
+
+        async fn collect_row_ids(ds: &Dataset, queries: &[Arc<dyn Array>]) -> Vec<Vec<u64>> {
+            let mut ids_per_query = Vec::with_capacity(queries.len());
+            for q in queries {
+                let result = ds
+                    .scan()
+                    .with_row_id()
+                    .project(&["_rowid"] as &[&str])
+                    .unwrap()
+                    .nearest("vector", q.as_ref(), K)
+                    .unwrap()
+                    .minimum_nprobes(TWO_FRAG_NUM_PARTITIONS)
+                    .try_into_batch()
+                    .await
+                    .unwrap();
+
+                let row_ids = result[ROW_ID]
+                    .as_primitive::<UInt64Type>()
+                    .values()
+                    .iter()
+                    .copied()
+                    .collect::<Vec<u64>>();
+                ids_per_query.push(row_ids);
+            }
+            ids_per_query
+        }
+
+        let query_batch = ds_single
+            .scan()
+            .project(&["vector"] as &[&str])
+            .unwrap()
+            .limit(Some(NUM_QUERIES as i64), None)
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let vectors = query_batch["vector"].as_fixed_size_list();
+        let queries: Vec<Arc<dyn Array>> = (0..vectors.len())
+            .map(|i| vectors.value(i) as Arc<dyn Array>)
+            .collect();
+
+        let ids_single = collect_row_ids(&ds_single, &queries).await;
+        let ids_split = collect_row_ids(&ds_split, &queries).await;
+
+        assert_eq!(
+            ids_single, ids_split,
+            "single vs split index returned different Top-K row ids",
+        );
+    }
+
+    #[rstest]
+    #[case::ivf_flat(IndexType::IvfFlat)]
+    #[case::ivf_pq(IndexType::IvfPq)]
+    #[case::ivf_sq(IndexType::IvfSq)]
+    #[case::ivf_rq(IndexType::IvfRq)]
+    #[tokio::test]
+    async fn test_distributed_vector_build_commits_multiple_segments_and_preserves_query_results(
+        #[case] index_type: IndexType,
+    ) {
+        const INDEX_NAME: &str = "vector_idx";
+        const K: usize = 10;
+        const NUM_QUERIES: usize = 10;
+
+        let test_dir = TempStrDir::default();
+        let base_uri = test_dir.as_str();
+
+        // Generate the data once, then write it twice to two independent dataset URIs.
+        let (schema, batches) = make_two_fragment_batches();
+
+        let ds_single_uri = format!("{}/single", base_uri);
+        let ds_split_uri = format!("{}/split", base_uri);
+
+        let mut ds_single =
+            write_dataset_from_batches(&ds_single_uri, schema.clone(), batches.clone()).await;
+        let mut ds_split = write_dataset_from_batches(&ds_split_uri, schema, batches).await;
+
+        // Ensure we have at least 2 fragments.
+        let fragments_single = ds_single.get_fragments();
+        assert!(
+            fragments_single.len() >= 2,
+            "expected at least 2 fragments in ds_single, got {}",
+            fragments_single.len()
+        );
+        let fragments_split = ds_split.get_fragments();
+        assert!(
+            fragments_split.len() >= 2,
+            "expected at least 2 fragments in ds_split, got {}",
+            fragments_split.len()
+        );
+
+        let distributed_params = match index_type {
+            IndexType::IvfFlat => {
+                let ivf_params = prepare_global_ivf(&ds_single, "vector").await;
+                VectorIndexParams::with_ivf_flat_params(DistanceType::L2, ivf_params)
+            }
+            IndexType::IvfPq => {
+                let (ivf_params, pq_params) = prepare_global_ivf_pq(&ds_single, "vector").await;
+                VectorIndexParams::with_ivf_pq_params(DistanceType::L2, ivf_params, pq_params)
+            }
+            IndexType::IvfSq => {
+                let ivf_params = prepare_global_ivf(&ds_single, "vector").await;
+                VectorIndexParams::with_ivf_sq_params(
+                    DistanceType::L2,
+                    ivf_params,
+                    SQBuildParams::default(),
+                )
+            }
+            IndexType::IvfRq => {
+                let ivf_params = prepare_global_ivf(&ds_single, "vector").await;
+                VectorIndexParams::with_ivf_rq_params(
+                    DistanceType::L2,
+                    ivf_params,
+                    RQBuildParams::with_rotation_type(1, RQRotationType::Fast),
+                )
+            }
+            other => panic!("unsupported test index type: {}", other),
+        };
+
+        ds_single
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some(INDEX_NAME.to_string()),
+                &distributed_params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        let fragment_groups = fragments_split
+            .iter()
+            .map(|fragment| vec![fragment.id() as u32])
+            .collect::<Vec<_>>();
+        let expected_segment_count = fragment_groups.len();
+        let segments = build_segments_for_fragment_groups(
+            &mut ds_split,
+            fragment_groups,
+            &distributed_params,
+            INDEX_NAME,
+        )
+        .await;
+        let segments = build_distributed_segments(&mut ds_split, segments, INDEX_NAME).await;
+        assert_eq!(segments.len(), expected_segment_count);
+        for segment in &segments {
+            let segment_index = ds_split
+                .indices_dir()
+                .child(segment.uuid.to_string())
+                .child(crate::index::INDEX_FILE_NAME);
+            assert!(
+                ds_split
+                    .object_store()
+                    .exists(&segment_index)
+                    .await
+                    .unwrap(),
+                "segment file should exist at {}",
+                segment_index
+            );
+        }
+
+        let committed_segments = ds_split.load_indices_by_name(INDEX_NAME).await.unwrap();
+        assert_eq!(committed_segments.len(), expected_segment_count);
+        for committed in committed_segments {
+            let covered_fragments = committed
+                .fragment_bitmap
+                .as_ref()
+                .expect("distributed segment should have fragment coverage");
+            assert_eq!(covered_fragments.len(), 1);
+        }
+
+        async fn collect_row_ids(ds: &Dataset, queries: &[Arc<dyn Array>]) -> Vec<Vec<u64>> {
+            let mut ids_per_query = Vec::with_capacity(queries.len());
+            for q in queries {
+                let result = ds
+                    .scan()
+                    .with_row_id()
+                    .project(&["_rowid"] as &[&str])
+                    .unwrap()
+                    .nearest("vector", q.as_ref(), K)
+                    .unwrap()
+                    .minimum_nprobes(TWO_FRAG_NUM_PARTITIONS)
+                    .try_into_batch()
+                    .await
+                    .unwrap();
+
+                let row_ids = result[ROW_ID]
+                    .as_primitive::<UInt64Type>()
+                    .values()
+                    .iter()
+                    .copied()
+                    .collect::<Vec<u64>>();
+                ids_per_query.push(row_ids);
+            }
+            ids_per_query
+        }
+
+        // Collect a deterministic query set from ds_single.
+        let query_batch = ds_single
+            .scan()
+            .project(&["vector"] as &[&str])
+            .unwrap()
+            .limit(Some(NUM_QUERIES as i64), None)
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let vectors = query_batch["vector"].as_fixed_size_list();
+        let queries: Vec<Arc<dyn Array>> = (0..vectors.len())
+            .map(|i| vectors.value(i) as Arc<dyn Array>)
+            .collect();
+
+        let ids_single = collect_row_ids(&ds_single, &queries).await;
+        let ids_split = collect_row_ids(&ds_split, &queries).await;
+
+        if index_type == IndexType::IvfRq {
+            for row_ids in &ids_split {
+                assert_eq!(
+                    row_ids.len(),
+                    K,
+                    "distributed IVF_RQ query should still return exactly {K} row ids",
+                );
+            }
+        } else {
             assert_eq!(
-                set_single, set_split,
-                "row id set mismatch for partition {}",
-                part_id
+                ids_single, ids_split,
+                "single vs segmented distributed index returned different Top-K row ids",
             );
         }
+    }
 
-        // Compare Top-K row ids on a deterministic set of queries.
+    #[rstest]
+    #[case::ivf_flat(IndexType::IvfFlat)]
+    #[case::ivf_pq(IndexType::IvfPq)]
+    #[case::ivf_sq(IndexType::IvfSq)]
+    #[tokio::test]
+    async fn test_distributed_vector_grouped_build_allows_concurrent_group_execution(
+        #[case] index_type: IndexType,
+    ) {
+        const INDEX_NAME: &str = "grouped_idx";
         const K: usize = 10;
         const NUM_QUERIES: usize = 10;
 
+        let test_dir = TempStrDir::default();
+        let base_uri = test_dir.as_str();
+
+        let (schema, batches) = make_two_fragment_batches();
+        let ds_single_uri = format!("{}/grouped_single", base_uri);
+        let ds_split_uri = format!("{}/grouped_split", base_uri);
+
+        let mut ds_single =
+            write_dataset_from_batches(&ds_single_uri, schema.clone(), batches.clone()).await;
+        let mut ds_split = write_dataset_from_batches(&ds_split_uri, schema, batches).await;
+
+        let distributed_params = match index_type {
+            IndexType::IvfFlat => {
+                let ivf_params = prepare_global_ivf(&ds_single, "vector").await;
+                VectorIndexParams::with_ivf_flat_params(DistanceType::L2, ivf_params)
+            }
+            IndexType::IvfPq => {
+                let (ivf_params, pq_params) = prepare_global_ivf_pq(&ds_single, "vector").await;
+                VectorIndexParams::with_ivf_pq_params(DistanceType::L2, ivf_params, pq_params)
+            }
+            IndexType::IvfSq => {
+                let ivf_params = prepare_global_ivf(&ds_single, "vector").await;
+                VectorIndexParams::with_ivf_sq_params(
+                    DistanceType::L2,
+                    ivf_params,
+                    SQBuildParams::default(),
+                )
+            }
+            other => panic!("unsupported test index type: {}", other),
+        };
+
+        ds_single
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some(INDEX_NAME.to_string()),
+                &distributed_params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        let fragment_groups = ds_split
+            .get_fragments()
+            .into_iter()
+            .map(|fragment| vec![fragment.id() as u32])
+            .collect::<Vec<_>>();
+        let segments = build_segments_for_fragment_groups(
+            &mut ds_split,
+            fragment_groups,
+            &distributed_params,
+            INDEX_NAME,
+        )
+        .await;
+
+        assert!(segments.len() >= 4);
+        let grouped_inputs = segments
+            .chunks(2)
+            .map(|group| group.to_vec())
+            .collect::<Vec<_>>();
+        let mut expected_fragment_coverage = grouped_inputs
+            .iter()
+            .map(|group| {
+                group
+                    .iter()
+                    .flat_map(|partial| {
+                        partial
+                            .fragment_bitmap
+                            .as_ref()
+                            .expect("partial shard should have fragment coverage")
+                            .iter()
+                    })
+                    .sorted()
+                    .collect::<Vec<_>>()
+            })
+            .collect::<Vec<_>>();
+        expected_fragment_coverage.sort();
+
+        let grouped_segments = futures::future::try_join_all(
+            grouped_inputs
+                .into_iter()
+                .map(|group| ds_split.merge_existing_index_segments(group)),
+        )
+        .await
+        .unwrap();
+        let grouped_segments =
+            build_distributed_segments(&mut ds_split, grouped_segments, INDEX_NAME).await;
+        assert_eq!(grouped_segments.len(), expected_fragment_coverage.len());
+        let mut actual_fragment_coverage = grouped_segments
+            .iter()
+            .map(|segment| {
+                segment
+                    .fragment_bitmap
+                    .as_ref()
+                    .unwrap()
+                    .iter()
+                    .collect::<Vec<_>>()
+            })
+            .collect::<Vec<_>>();
+        actual_fragment_coverage.sort();
+        assert_eq!(
+            actual_fragment_coverage, expected_fragment_coverage,
+            "built segment coverage should equal the union of its source partial shards",
+        );
+
         async fn collect_row_ids(ds: &Dataset, queries: &[Arc<dyn Array>]) -> Vec<Vec<u64>> {
             let mut ids_per_query = Vec::with_capacity(queries.len());
             for q in queries {
@@ -1622,22 +2683,23 @@ mod tests {
                     .unwrap()
                     .nearest("vector", q.as_ref(), K)
                     .unwrap()
+                    .minimum_nprobes(TWO_FRAG_NUM_PARTITIONS)
                     .try_into_batch()
                     .await
                     .unwrap();
 
-                let row_ids = result[ROW_ID]
-                    .as_primitive::<UInt64Type>()
-                    .values()
-                    .iter()
-                    .copied()
-                    .collect::<Vec<u64>>();
-                ids_per_query.push(row_ids);
+                ids_per_query.push(
+                    result[ROW_ID]
+                        .as_primitive::<UInt64Type>()
+                        .values()
+                        .iter()
+                        .copied()
+                        .collect(),
+                );
             }
             ids_per_query
         }
 
-        // Collect a deterministic query set from ds_single.
         let query_batch = ds_single
             .scan()
             .project(&["vector"] as &[&str])
@@ -1654,11 +2716,288 @@ mod tests {
 
         let ids_single = collect_row_ids(&ds_single, &queries).await;
         let ids_split = collect_row_ids(&ds_split, &queries).await;
+        if matches!(index_type, IndexType::IvfSq) {
+            for (single, split) in ids_single.iter().zip(ids_split.iter()) {
+                assert_eq!(single.len(), split.len());
+                let overlap = single
+                    .iter()
+                    .filter(|row_id| split.contains(row_id))
+                    .count();
+                assert!(
+                    overlap >= K / 3,
+                    "single vs segmented distributed SQ index returned too little top-k overlap",
+                );
+            }
+        } else {
+            assert_eq!(ids_single, ids_split);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_distributed_vector_plan_rejects_overlapping_fragment_coverage() {
+        let test_dir = TempStrDir::default();
+        let base_uri = test_dir.as_str();
+        let (schema, batches) = make_two_fragment_batches();
+        let dataset_uri = format!("{}/overlap_fragments", base_uri);
+        let mut dataset = write_dataset_from_batches(&dataset_uri, schema, batches).await;
+
+        let fragment = dataset.get_fragments()[0].id() as u32;
+        let params = VectorIndexParams::with_ivf_flat_params(
+            DistanceType::L2,
+            prepare_global_ivf(&dataset, "vector").await,
+        );
+        let mut segments = Vec::new();
+
+        for _ in 0..2 {
+            let segment = dataset
+                .create_index_builder(&["vector"], IndexType::Vector, &params)
+                .name("vector_idx".to_string())
+                .fragments(vec![fragment])
+                .execute_uncommitted()
+                .await
+                .unwrap();
+            segments.push(segment);
+        }
+
+        let err = dataset
+            .merge_existing_index_segments(segments)
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("overlapping fragment coverage"));
+    }
+
+    #[tokio::test]
+    async fn test_distributed_vector_build_supports_hnsw_variants() {
+        let test_dir = TempStrDir::default();
+        let base_uri = test_dir.as_str();
+        let (schema, batches) = make_two_fragment_batches();
+        let dataset_uri = format!("{}/distributed_hnsw_supported", base_uri);
+        let mut dataset = write_dataset_from_batches(&dataset_uri, schema, batches).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+        let params = VectorIndexParams::ivf_hnsw(
+            DistanceType::L2,
+            prepare_global_ivf(&dataset, "vector").await,
+            HnswBuildParams::default(),
+        );
+        let mut segments = Vec::new();
+
+        for fragment in fragments.iter().take(2) {
+            let segment = dataset
+                .create_index_builder(&["vector"], IndexType::Vector, &params)
+                .name("vector_idx".to_string())
+                .fragments(vec![fragment.id() as u32])
+                .execute_uncommitted()
+                .await
+                .unwrap();
+            segments.push(segment);
+        }
+
+        dataset
+            .commit_existing_index_segments("vector_idx", "vector", segments)
+            .await
+            .unwrap();
+
+        let query_batch = dataset
+            .scan()
+            .project(&["vector"] as &[&str])
+            .unwrap()
+            .limit(Some(4), None)
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let q = query_batch["vector"].as_fixed_size_list().value(0);
+        let result = dataset
+            .scan()
+            .project(&["_rowid"] as &[&str])
+            .unwrap()
+            .nearest("vector", q.as_ref(), 5)
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert!(result.num_rows() > 0);
+    }
+
+    #[tokio::test]
+    async fn test_merge_index_metadata_reports_progress() {
+        const INDEX_NAME: &str = "vector_idx";
+
+        let test_dir = TempStrDir::default();
+        let dataset_uri = format!("{}/progress", test_dir.as_str());
+        let (schema, batches) = make_two_fragment_batches();
+        let mut dataset = write_dataset_from_batches(&dataset_uri, schema, batches).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(
+            fragments.len() >= 2,
+            "expected at least 2 fragments, got {}",
+            fragments.len()
+        );
+        let expected_rows = fragments[0].physical_rows().await.unwrap() as u64
+            + fragments[1].physical_rows().await.unwrap() as u64;
+
+        let (ivf_params, pq_params) = prepare_global_ivf_pq(&dataset, "vector").await;
+        let params = VectorIndexParams::with_ivf_pq_params(DistanceType::L2, ivf_params, pq_params);
+        let mut segments = Vec::new();
+        for fragment in fragments.iter().take(2) {
+            segments.push(
+                dataset
+                    .create_index_builder(&["vector"], IndexType::Vector, &params)
+                    .name(INDEX_NAME.to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap(),
+            );
+        }
+
+        let progress = Arc::new(RecordingProgress::default());
+        let merged_segment = crate::index::vector::ivf::merge_segments_with_progress(
+            dataset.object_store(),
+            &dataset.indices_dir(),
+            segments,
+            progress.clone(),
+        )
+        .await
+        .unwrap();
+        dataset
+            .commit_existing_index_segments(INDEX_NAME, "vector", vec![merged_segment])
+            .await
+            .unwrap();
 
+        let events = progress.recorded_events();
+        let tags = events
+            .iter()
+            .map(|(kind, stage, _)| format!("{kind}:{stage}"))
+            .collect::<Vec<_>>();
+        let merge_total = events
+            .iter()
+            .find_map(|(kind, stage, value)| {
+                if kind == "start" && stage == "merge_partitions" {
+                    Some(*value)
+                } else {
+                    None
+                }
+            })
+            .expect("missing merge_partitions start total");
+        let merged_rows = events
+            .iter()
+            .filter_map(|(kind, stage, value)| {
+                if kind == "progress" && stage == "merge_partitions" {
+                    Some(*value)
+                } else {
+                    None
+                }
+            })
+            .next_back()
+            .unwrap_or_default();
+        let read_start = tags
+            .iter()
+            .position(|e| e == "start:read_shard_metadata")
+            .expect("missing read_shard_metadata start");
+        let read_complete = tags
+            .iter()
+            .position(|e| e == "complete:read_shard_metadata")
+            .expect("missing read_shard_metadata complete");
+        let merge_start = tags
+            .iter()
+            .position(|e| e == "start:merge_partitions")
+            .expect("missing merge_partitions start");
+        let merge_complete = tags
+            .iter()
+            .position(|e| e == "complete:merge_partitions")
+            .expect("missing merge_partitions complete");
+        let aux_start = tags
+            .iter()
+            .position(|e| e == "start:write_auxiliary_index")
+            .expect("missing write_auxiliary_index start");
+        let aux_complete = tags
+            .iter()
+            .position(|e| e == "complete:write_auxiliary_index")
+            .expect("missing write_auxiliary_index complete");
+        let root_start = tags
+            .iter()
+            .position(|e| e == "start:write_root_index")
+            .expect("missing write_root_index start");
+        let root_complete = tags
+            .iter()
+            .position(|e| e == "complete:write_root_index")
+            .expect("missing write_root_index complete");
+
+        assert!(read_start < read_complete);
+        assert!(read_complete < merge_start);
+        assert!(merge_start < merge_complete);
+        assert!(merge_complete < aux_start);
+        assert!(aux_start < aux_complete);
+        assert!(aux_complete < root_start);
+        assert!(root_start < root_complete);
         assert_eq!(
-            ids_single, ids_split,
-            "single vs split index returned different Top-K row ids",
+            merge_total, expected_rows,
+            "expected merge_partitions total rows to match dataset rows"
+        );
+        assert_eq!(
+            merged_rows, expected_rows,
+            "expected merge_partitions completed rows to match dataset rows"
+        );
+        assert!(
+            tags.iter().any(|e| e == "progress:write_root_index"),
+            "expected write_root_index progress callbacks"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_distributed_ivf_sq_worker_training_respects_fragment_filter() {
+        const ROWS_PER_FRAGMENT: usize = 64;
+        const FRAGMENT_OFFSETS: [f32; 2] = [0.0, 1000.0];
+
+        let test_dir = TempStrDir::default();
+        let dataset_uri = format!("{}/distributed_sq_fragment_filter", test_dir.as_str());
+        let (schema, batches) = make_fragment_offset_batches(ROWS_PER_FRAGMENT, &FRAGMENT_OFFSETS);
+        let batches = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
+        let mut dataset = Dataset::write(
+            batches,
+            &dataset_uri,
+            Some(WriteParams {
+                max_rows_per_file: ROWS_PER_FRAGMENT,
+                mode: WriteMode::Overwrite,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let fragments = dataset.get_fragments();
+        assert_eq!(fragments.len(), FRAGMENT_OFFSETS.len());
+
+        let ivf_params =
+            IvfBuildParams::try_with_centroids(2, build_centroids_for_offsets(&FRAGMENT_OFFSETS))
+                .unwrap();
+        let params = VectorIndexParams::with_ivf_sq_params(
+            DistanceType::L2,
+            ivf_params,
+            SQBuildParams::default(),
         );
+
+        let segment = dataset
+            .create_index_builder(&["vector"], IndexType::Vector, &params)
+            .name("sq_fragment_filter".to_string())
+            .fragments(vec![fragments[0].id() as u32])
+            .execute_uncommitted()
+            .await
+            .unwrap();
+
+        let scheduler = ScanScheduler::new(
+            Arc::new(dataset.object_store().clone()),
+            SchedulerConfig::default_for_testing(),
+        );
+        let sq_meta = get_sq_metadata(&dataset, scheduler, &segment.uuid.to_string()).await;
+
+        assert_eq!(sq_meta.bounds.start, 0.0);
+        assert_eq!(sq_meta.bounds.end, (DIM - 1) as f64);
+        assert_lt!(sq_meta.bounds.end, FRAGMENT_OFFSETS[1] as f64);
     }
 
     async fn test_index(
@@ -1682,13 +3021,7 @@ mod tests {
                 )
                 .await;
 
-                let index_type = params.index_type();
-                // *_FLAT doesn't support float16/float64
-                if !(index_type == IndexType::IvfFlat
-                    || (index_type == IndexType::IvfHnswFlat && params.stages.len() == 2)) // IVF_HNSW_FLAT
-                    && dataset.is_none()
-                // if dataset is provided, it has been created, so the data type is already determined, no need to test float64
-                {
+                if dataset.is_none() {
                     test_index_impl::<Float64Type>(
                         params,
                         nlist,
@@ -1738,10 +3071,10 @@ mod tests {
             && matches!(params.version, IndexFileVersion::V3)
             && params.index_type() == IndexType::IvfPq
         {
-            let index = dataset.load_indices().await.unwrap();
-            assert_eq!(index.len(), 1);
-            let index_path = dataset.indices_dir().child(index[0].uuid.to_string());
-            rewrite_pq_storage(index_path).await.unwrap();
+            let indices = dataset.load_indices().await.unwrap();
+            assert_eq!(indices.len(), 1);
+            let old_meta = indices[0].clone();
+            rewrite_pq_storage(&mut dataset, &old_meta).await.unwrap();
             // do the test again
             test_recall::<T>(
                 params,
@@ -1767,13 +3100,23 @@ mod tests {
                 .await;
             }
             _ => {
+                let index_type = params.index_type();
                 Box::pin(test_remap_impl::<Float32Type>(
-                    params,
+                    params.clone(),
                     nlist,
                     recall_requirement,
                     0.0..1.0,
                 ))
                 .await;
+                if matches!(index_type, IndexType::IvfFlat | IndexType::IvfHnswFlat) {
+                    Box::pin(test_remap_impl::<Float64Type>(
+                        params,
+                        nlist,
+                        recall_requirement,
+                        0.0..1.0,
+                    ))
+                    .await;
+                }
             }
         }
     }
@@ -2157,6 +3500,7 @@ mod tests {
     #[case(4, DistanceType::L2, 0.9)]
     #[case(4, DistanceType::Cosine, 0.9)]
     #[case(4, DistanceType::Dot, 0.85)]
+    #[case(4, DistanceType::Hamming, 0.9)]
     #[tokio::test]
     async fn test_create_ivf_hnsw_flat(
         #[case] nlist: usize,
@@ -2367,6 +3711,7 @@ mod tests {
             Some((dataset.clone(), vectors.clone())),
         )
         .await;
+        dataset.checkout_latest().await.unwrap();
         // retest with v3 params on the same dataset
         test_index(
             v3_params,
@@ -2713,17 +4058,36 @@ mod tests {
         );
     }
 
-    async fn rewrite_pq_storage(dir: Path) -> Result<()> {
+    /// Rewrite the auxiliary storage file to the legacy PQ format (codebook
+    /// embedded in schema metadata rather than stored as a global buffer), then
+    /// commit a `CreateIndex` transaction so the manifest records the correct
+    /// new file size.
+    /// Rewrite the auxiliary PQ storage file with the codebook inlined into
+    /// schema metadata (legacy format). Uses a new UUID to avoid cache key
+    /// collisions with the original index.
+    async fn rewrite_pq_storage(dataset: &mut Dataset, old_meta: &IndexMetadata) -> Result<()> {
+        use crate::dataset::transaction::{Operation, Transaction};
+
         let obj_store = Arc::new(ObjectStore::local());
-        let store_path = dir.child(INDEX_AUXILIARY_FILE_NAME);
-        let copied_path = dir.child(format!("{}.original", INDEX_AUXILIARY_FILE_NAME));
-        obj_store.copy(&store_path, &copied_path).await?;
-        obj_store.delete(&store_path).await?;
+        let old_dir = dataset.indices_dir().child(old_meta.uuid.to_string());
+        let new_uuid = uuid::Uuid::new_v4();
+        let new_dir = dataset.indices_dir().child(new_uuid.to_string());
+
+        // Copy the main index file to the new directory unchanged.
+        obj_store
+            .copy(
+                &old_dir.child(super::INDEX_FILE_NAME),
+                &new_dir.child(super::INDEX_FILE_NAME),
+            )
+            .await?;
+
+        // Read the original auxiliary file.
+        let old_aux_path = old_dir.child(INDEX_AUXILIARY_FILE_NAME);
         let scheduler =
             ScanScheduler::new(obj_store.clone(), SchedulerConfig::default_for_testing());
         let reader = FileReader::try_open(
             scheduler
-                .open_file(&copied_path, &CachedFileSize::unknown())
+                .open_file(&old_aux_path, &CachedFileSize::unknown())
                 .await?,
             None,
             Arc::<DecoderPlugins>::default(),
@@ -2732,21 +4096,21 @@ mod tests {
         )
         .await?;
 
+        // Rewrite auxiliary file with PQ codebook inlined into schema metadata.
         let mut metadata = reader.schema().metadata.clone();
         let batch = reader
             .read_range(0..reader.num_rows() as usize, None)
             .await?;
+        let new_aux_path = new_dir.child(INDEX_AUXILIARY_FILE_NAME);
         let mut writer = FileWriter::try_new(
-            obj_store.create(&store_path).await?,
+            obj_store.create(&new_aux_path).await?,
             batch.schema_ref().as_ref().try_into()?,
             Default::default(),
         )?;
         writer.write_batch(&batch).await?;
-        // write the IVF
         writer
             .add_global_buffer(reader.read_global_buffer(1).await?)
             .await?;
-        // rewrite the PQ to legacy format
         let codebook = reader.read_global_buffer(2).await?;
         let pq_metadata: Vec<String> = serde_json::from_str(&metadata[STORAGE_METADATA_KEY])?;
         let mut pq_metadata: ProductQuantizationMetadata = serde_json::from_str(&pq_metadata[0])?;
@@ -2758,7 +4122,26 @@ mod tests {
             serde_json::to_string(&vec![pq_metadata])?,
         );
         writer.finish_with_metadata(metadata).await?;
-        obj_store.delete(&copied_path).await?;
+
+        // Build new IndexMetadata with the new UUID and file sizes.
+        let new_files =
+            lance_table::format::list_index_files_with_sizes(&obj_store, &new_dir).await?;
+        let mut new_meta = old_meta.clone();
+        new_meta.uuid = new_uuid;
+        new_meta.files = Some(new_files);
+
+        let transaction = Transaction::new(
+            dataset.manifest.version,
+            Operation::CreateIndex {
+                new_indices: vec![new_meta],
+                removed_indices: vec![old_meta.clone()],
+            },
+            None,
+        );
+        dataset
+            .apply_commit(transaction, &Default::default(), &Default::default())
+            .await?;
+
         Ok(())
     }
 
@@ -3212,55 +4595,168 @@ mod tests {
         .await;
         expected_rows += rows;
 
-        // Append #2: triggers split and merge.
-        expected_partitions += 1;
-        let rows = SECOND_APPEND_ROWS;
-        append_and_verify_append_phase(
-            &mut dataset,
-            INDEX_NAME,
-            &template_values,
-            rows,
-            expected_partitions,
-            expected_rows + rows,
-            1,
-            true,
-        )
-        .await;
-        expected_rows += rows;
+        // Append #2: triggers split and merge.
+        expected_partitions += 1;
+        let rows = SECOND_APPEND_ROWS;
+        append_and_verify_append_phase(
+            &mut dataset,
+            INDEX_NAME,
+            &template_values,
+            rows,
+            expected_partitions,
+            expected_rows + rows,
+            1,
+            true,
+        )
+        .await;
+        expected_rows += rows;
+
+        // Append #3: triggers another split, remains a single merged index.
+        expected_partitions += 1;
+        let rows = THIRD_APPEND_ROWS;
+        append_and_verify_append_phase(
+            &mut dataset,
+            INDEX_NAME,
+            &template_values,
+            rows,
+            expected_partitions,
+            expected_rows + rows,
+            1,
+            true,
+        )
+        .await;
+    }
+
+    #[tokio::test]
+    async fn test_partition_split_on_append_multivec() {
+        // This test verifies that when we append enough multivector data to a partition
+        // such that it exceeds MAX_PARTITION_SIZE_FACTOR * target_partition_size,
+        // the partition will be split into 2 partitions.
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        // Create initial dataset with multivector data
+        let (dataset, _) = generate_multivec_test_dataset::<Float32Type>(test_uri, 0.0..1.0).await;
+
+        // Create an IVF-PQ index with 2 partitions
+        // For IvfPq, target_partition_size = 8192
+        // Split triggers when partition_size > 4 * 8192 = 32,768
+        let params = VectorIndexParams::ivf_pq(2, 8, DIM / 8, DistanceType::Cosine, 50);
+        verify_partition_split_after_append(dataset, test_uri, params, "multivector data").await;
+    }
+
+    #[tokio::test]
+    async fn test_split_multiple_partitions_in_one_optimize() {
+        const INDEX_NAME: &str = "vector_idx";
+        const BASE_ROWS_PER_PARTITION: usize = 512;
+        const APPEND_ROWS_PER_PARTITION: usize = 40_000;
+        let offsets = [-50.0, 50.0];
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        let (batch, schema) = generate_clustered_batch(BASE_ROWS_PER_PARTITION, offsets);
+        let batches = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let mut dataset = Dataset::write(
+            batches,
+            test_uri,
+            Some(WriteParams {
+                mode: WriteMode::Overwrite,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let centroids = build_centroids_for_offsets(&offsets);
+        let ivf_params = IvfBuildParams::try_with_centroids(2, centroids).unwrap();
+        let params = VectorIndexParams::with_ivf_pq_params(
+            DistanceType::L2,
+            ivf_params,
+            PQBuildParams::default(),
+        );
+        dataset
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some(INDEX_NAME.to_string()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        let initial_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await;
+        assert_eq!(initial_ctx.num_partitions(), 2);
+        let mut templates = Vec::with_capacity(2);
+        for partition_idx in 0..2 {
+            let row_ids = load_partition_row_ids(initial_ctx.ivf(), partition_idx).await;
+            let template_batch = dataset
+                .take_rows(&[row_ids[0]], dataset.schema().clone())
+                .await
+                .unwrap();
+            templates.push(
+                template_batch["vector"]
+                    .as_fixed_size_list()
+                    .value(0)
+                    .as_primitive::<Float32Type>()
+                    .values()
+                    .to_vec(),
+            );
+        }
 
-        // Append #3: triggers another split, remains a single merged index.
-        expected_partitions += 1;
-        let rows = THIRD_APPEND_ROWS;
-        append_and_verify_append_phase(
-            &mut dataset,
-            INDEX_NAME,
-            &template_values,
-            rows,
-            expected_partitions,
-            expected_rows + rows,
-            1,
-            true,
-        )
-        .await;
-    }
+        append_partition_templates(&mut dataset, APPEND_ROWS_PER_PARTITION, &templates).await;
 
-    #[tokio::test]
-    async fn test_partition_split_on_append_multivec() {
-        // This test verifies that when we append enough multivector data to a partition
-        // such that it exceeds MAX_PARTITION_SIZE_FACTOR * target_partition_size,
-        // the partition will be split into 2 partitions.
+        dataset
+            .optimize_indices(&OptimizeOptions::new())
+            .await
+            .unwrap();
+        dataset.validate().await.unwrap();
 
-        let test_dir = TempStrDir::default();
-        let test_uri = test_dir.as_str();
+        let final_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await;
+        assert_eq!(
+            final_ctx.num_partitions(),
+            4,
+            "Expected both original partitions to split in one optimize, stats: {}",
+            final_ctx.stats_json()
+        );
 
-        // Create initial dataset with multivector data
-        let (dataset, _) = generate_multivec_test_dataset::<Float32Type>(test_uri, 0.0..1.0).await;
+        let indices = final_ctx.stats()["indices"]
+            .as_array()
+            .expect("indices should be present");
+        assert_eq!(
+            indices.len(),
+            1,
+            "Expected split optimize to merge into one index, stats: {}",
+            final_ctx.stats_json()
+        );
 
-        // Create an IVF-PQ index with 2 partitions
-        // For IvfPq, target_partition_size = 8192
-        // Split triggers when partition_size > 4 * 8192 = 32,768
-        let params = VectorIndexParams::ivf_pq(2, 8, DIM / 8, DistanceType::Cosine, 50);
-        verify_partition_split_after_append(dataset, test_uri, params, "multivector data").await;
+        let partitions = indices[0]["partitions"]
+            .as_array()
+            .expect("partitions should be present");
+        assert_eq!(partitions.len(), 4);
+        let expected_rows = 2 * BASE_ROWS_PER_PARTITION + 2 * APPEND_ROWS_PER_PARTITION;
+        let total_partition_rows = partitions
+            .iter()
+            .map(|part| part["size"].as_u64().unwrap() as usize)
+            .sum::<usize>();
+        assert_eq!(total_partition_rows, expected_rows);
+        assert_eq!(dataset.count_all_rows().await.unwrap(), expected_rows);
+
+        let nearest = dataset
+            .scan()
+            .with_row_id()
+            .nearest("vector", &Float32Array::from(templates[0].clone()), 10)
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let ids = nearest[ROW_ID].as_primitive::<UInt64Type>();
+        let mut seen = HashSet::new();
+        for row_id in ids.values() {
+            assert!(seen.insert(*row_id), "Duplicate row id found: {}", row_id);
+        }
     }
 
     #[tokio::test]
@@ -3510,4 +5006,320 @@ mod tests {
         let stats = dataset.object_store().io_stats_incremental();
         assert_io_eq!(stats, read_iops, 0, "second prewarm should not perform IO");
     }
+
+    #[tokio::test]
+    async fn test_prewarm_ivf_pq_multiple_deltas() {
+        use lance_io::assert_io_eq;
+
+        const INDEX_NAME: &str = "my_idx";
+        const BASE_ROWS_PER_PARTITION: usize = 3_000;
+        const SMALL_APPEND_ROWS: usize = 64;
+        let offsets = [-50.0, 50.0];
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        let (batch, schema) = generate_clustered_batch(BASE_ROWS_PER_PARTITION, offsets);
+        let batches = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let mut dataset = Dataset::write(
+            batches,
+            test_uri,
+            Some(WriteParams {
+                mode: WriteMode::Overwrite,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let centroids = build_centroids_for_offsets(&offsets);
+        let ivf_params = IvfBuildParams::try_with_centroids(2, centroids).unwrap();
+        let params = VectorIndexParams::with_ivf_pq_params(
+            DistanceType::L2,
+            ivf_params,
+            PQBuildParams::default(),
+        );
+        dataset
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some(INDEX_NAME.to_string()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        let template_batch = dataset
+            .take_rows(&[0], dataset.schema().clone())
+            .await
+            .unwrap();
+        let template_values = template_batch["vector"]
+            .as_fixed_size_list()
+            .value(0)
+            .as_primitive::<Float32Type>()
+            .values()
+            .to_vec();
+        let mut append_params = WriteParams {
+            max_rows_per_file: 32,
+            max_rows_per_group: 32,
+            ..Default::default()
+        };
+        append_params.mode = WriteMode::Append;
+        append_constant_vector_with_params(
+            &mut dataset,
+            SMALL_APPEND_ROWS,
+            &template_values,
+            Some(append_params),
+        )
+        .await;
+
+        dataset
+            .optimize_indices(&OptimizeOptions::new())
+            .await
+            .unwrap();
+
+        // Reopen dataset to avoid carrying index state in-memory from index creation.
+        let dataset = Dataset::open(test_uri).await.unwrap();
+        let indices = dataset.load_indices_by_name(INDEX_NAME).await.unwrap();
+        assert_eq!(indices.len(), 2, "expected two index deltas for my_idx");
+        let unique_uuids: HashSet<_> = indices.iter().map(|meta| meta.uuid).collect();
+        assert_eq!(unique_uuids.len(), 2, "expected two unique index UUIDs");
+
+        // Reset IO stats after index creation
+        dataset.object_store().io_stats_incremental();
+
+        // Prewarm should perform IO to load all index deltas into cache
+        dataset.prewarm_index(INDEX_NAME).await.unwrap();
+        let stats = dataset.object_store().io_stats_incremental();
+        assert!(
+            stats.read_iops > 0,
+            "prewarm should have read from disk, but read_iops was 0"
+        );
+
+        // Query should not perform IO after prewarm of all deltas
+        let q = Float32Array::from(template_values.clone());
+        dataset
+            .scan()
+            .nearest("vector", &q, 10)
+            .unwrap()
+            .project(&["_rowid"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let stats = dataset.object_store().io_stats_incremental();
+        assert_io_eq!(
+            stats,
+            read_iops,
+            0,
+            "query should not perform IO after prewarm"
+        );
+
+        // Second prewarm should not need IO (already cached)
+        dataset.prewarm_index(INDEX_NAME).await.unwrap();
+        let stats = dataset.object_store().io_stats_incremental();
+        assert_io_eq!(stats, read_iops, 0, "second prewarm should not perform IO");
+    }
+
+    type SerializedEntry = (Vec<u8>, lance_core::cache::CacheCodec, usize);
+
+    #[derive(Debug)]
+    struct SerializingBackend {
+        /// Serialized entries: key -> (bytes, codec, size).
+        serialized: tokio::sync::Mutex<
+            std::collections::HashMap<lance_core::cache::InternalCacheKey, SerializedEntry>,
+        >,
+        /// Fallback for entries without a codec.
+        passthrough: lance_core::cache::MokaCacheBackend,
+    }
+
+    impl SerializingBackend {
+        fn new() -> Self {
+            Self {
+                serialized: tokio::sync::Mutex::new(std::collections::HashMap::new()),
+                passthrough: lance_core::cache::MokaCacheBackend::with_capacity(256 * 1024 * 1024),
+            }
+        }
+
+        async fn serialized_entry_count(&self) -> usize {
+            self.serialized.lock().await.len()
+        }
+
+        async fn passthrough_entry_count(&self) -> usize {
+            use lance_core::cache::CacheBackend;
+            self.passthrough.num_entries().await
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl lance_core::cache::CacheBackend for SerializingBackend {
+        async fn get(
+            &self,
+            key: &lance_core::cache::InternalCacheKey,
+            codec: Option<lance_core::cache::CacheCodec>,
+        ) -> Option<lance_core::cache::CacheEntry> {
+            // Try serialized store first
+            let guard = self.serialized.lock().await;
+            if let Some((bytes, stored_codec, _)) = guard.get(key) {
+                return Some(
+                    stored_codec
+                        .deserialize(&bytes::Bytes::copy_from_slice(bytes))
+                        .expect("deserialization should succeed"),
+                );
+            }
+            drop(guard);
+            // Fall through to passthrough
+            self.passthrough.get(key, codec).await
+        }
+
+        async fn insert(
+            &self,
+            key: &lance_core::cache::InternalCacheKey,
+            entry: lance_core::cache::CacheEntry,
+            size_bytes: usize,
+            codec: Option<lance_core::cache::CacheCodec>,
+        ) {
+            if let Some(codec) = codec {
+                let mut bytes = Vec::new();
+                codec
+                    .serialize(&entry, &mut bytes)
+                    .expect("serialization should succeed");
+                self.serialized
+                    .lock()
+                    .await
+                    .insert(key.clone(), (bytes, codec, size_bytes));
+            } else {
+                self.passthrough.insert(key, entry, size_bytes, None).await;
+            }
+        }
+
+        async fn get_or_insert<'a>(
+            &self,
+            key: &lance_core::cache::InternalCacheKey,
+            loader: std::pin::Pin<
+                Box<
+                    dyn futures::Future<Output = Result<(lance_core::cache::CacheEntry, usize)>>
+                        + Send
+                        + 'a,
+                >,
+            >,
+            codec: Option<lance_core::cache::CacheCodec>,
+        ) -> Result<(lance_core::cache::CacheEntry, bool)> {
+            if let Some(entry) = self.get(key, codec).await {
+                return Ok((entry, true));
+            }
+            let (entry, size) = loader.await?;
+            self.insert(key, entry.clone(), size, codec).await;
+            Ok((entry, false))
+        }
+
+        async fn invalidate_prefix(&self, prefix: &str) {
+            self.serialized
+                .lock()
+                .await
+                .retain(|k, _| !k.starts_with(prefix));
+            self.passthrough.invalidate_prefix(prefix).await;
+        }
+
+        async fn clear(&self) {
+            self.serialized.lock().await.clear();
+            self.passthrough.clear().await;
+        }
+
+        async fn num_entries(&self) -> usize {
+            self.serialized.lock().await.len() + self.passthrough.num_entries().await
+        }
+
+        async fn size_bytes(&self) -> usize {
+            let serialized: usize = self
+                .serialized
+                .lock()
+                .await
+                .values()
+                .map(|(_, _, s)| *s)
+                .sum();
+            serialized + self.passthrough.size_bytes().await
+        }
+    }
+
+    /// Integration test: create a vector index, prewarm it through a
+    /// serializing cache backend, then query. Verifies that entries are
+    /// serialized to bytes and that queries produce correct results after
+    /// deserialization.
+    #[tokio::test]
+    async fn test_prewarm_and_query_with_serializing_backend() {
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        // Create dataset with vector index using default cache
+        let (mut dataset, _) = generate_test_dataset::<Float32Type>(test_uri, 0.0..1.0).await;
+        let params = VectorIndexParams::with_ivf_pq_params(
+            DistanceType::L2,
+            IvfBuildParams::new(4),
+            PQBuildParams::default(),
+        );
+        dataset
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some("serde_idx".to_owned()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Re-open with the serializing backend
+        let backend = Arc::new(SerializingBackend::new());
+        let session = Arc::new(crate::session::Session::with_index_cache_backend(
+            backend.clone(),
+            128 * 1024 * 1024,
+            Arc::new(lance_io::object_store::ObjectStoreRegistry::default()),
+        ));
+        let dataset = crate::DatasetBuilder::from_uri(test_uri)
+            .with_session(session)
+            .load()
+            .await
+            .unwrap();
+
+        // Prewarm — this should serialize entries into the backend
+        dataset.prewarm_index("serde_idx").await.unwrap();
+        let serialized = backend.serialized_entry_count().await;
+        let passthrough = backend.passthrough_entry_count().await;
+        assert!(
+            serialized > 0,
+            "prewarm should have serialized entries into the backend"
+        );
+        assert_eq!(
+            passthrough, 0,
+            "all index cache entries should have codecs (nothing in passthrough), \
+             but found {passthrough} passthrough entries"
+        );
+
+        // Query — the backend will deserialize entries from bytes.
+        // After prewarm, all entries are in serialized form, so every
+        // cache hit involves a deserialization round-trip.
+        let q = Float32Array::from_iter_values(repeat_n(0.5, DIM));
+        let results = dataset
+            .scan()
+            .nearest("vector", &q, 10)
+            .unwrap()
+            .nprobes(4)
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(results.num_rows(), 10, "should return 10 nearest neighbors");
+
+        // Verify distances are sorted (ascending for L2)
+        let distances: Vec<f32> = results
+            .column_by_name("_distance")
+            .unwrap()
+            .as_primitive::<Float32Type>()
+            .values()
+            .to_vec();
+        for w in distances.windows(2) {
+            assert!(w[1] >= w[0], "distances should be sorted ascending");
+        }
+    }
 }
diff --git a/rust/lance/src/index/vector/pq.rs b/rust/lance/src/index/vector/pq.rs
index d89999d921c..615f1b9c829 100644
--- a/rust/lance/src/index/vector/pq.rs
+++ b/rust/lance/src/index/vector/pq.rs
@@ -502,6 +502,18 @@ pub async fn build_pq_model(
     metric_type: MetricType,
     params: &PQBuildParams,
     ivf: Option<&IvfModel>,
+) -> Result<ProductQuantizer> {
+    build_pq_model_in_fragments(dataset, column, dim, metric_type, params, ivf, None).await
+}
+
+pub async fn build_pq_model_in_fragments(
+    dataset: &Dataset,
+    column: &str,
+    dim: usize,
+    metric_type: MetricType,
+    params: &PQBuildParams,
+    ivf: Option<&IvfModel>,
+    fragment_ids: Option<&[u32]>,
 ) -> Result<ProductQuantizer> {
     let num_codes = 2_usize.pow(params.num_bits as u32);
 
@@ -542,7 +554,7 @@ pub async fn build_pq_model(
     );
     let start = std::time::Instant::now();
     let mut training_data =
-        maybe_sample_training_data(dataset, column, expected_sample_size).await?;
+        maybe_sample_training_data(dataset, column, expected_sample_size, fragment_ids).await?;
     info!(
         "Finished loading training data in {:02} seconds",
         start.elapsed().as_secs_f32()
@@ -712,6 +724,7 @@ mod tests {
             DIM,
             MetricType::Cosine,
             &ivf_params,
+            None,
             lance_index::progress::noop_progress(),
         )
         .await
diff --git a/rust/lance/src/index/vector/utils.rs b/rust/lance/src/index/vector/utils.rs
index 83e010dc1a4..19156ac8eed 100644
--- a/rust/lance/src/index/vector/utils.rs
+++ b/rust/lance/src/index/vector/utils.rs
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use std::collections::HashSet;
+use std::pin::Pin;
 use std::sync::Arc;
 
 use arrow::array::ArrayData;
@@ -8,7 +10,7 @@ use arrow::datatypes::DataType;
 use arrow_array::new_empty_array;
 use arrow_array::{Array, ArrayRef, FixedSizeListArray, RecordBatch, cast::AsArray};
 use arrow_buffer::{Buffer, MutableBuffer};
-use futures::StreamExt;
+use futures::{Stream, StreamExt, stream};
 use lance_arrow::DataTypeExt;
 use lance_core::datatypes::Schema;
 use lance_linalg::distance::DistanceType;
@@ -18,7 +20,7 @@ use rand::seq::{IteratorRandom, SliceRandom};
 use rand::{Rng, SeedableRng};
 use tokio::sync::Mutex;
 
-use crate::dataset::Dataset;
+use crate::dataset::{Dataset, ProjectionRequest, TakeBuilder, row_offsets_to_row_addresses};
 use crate::{Error, Result};
 
 /// Helper function to extract a column from a RecordBatch, supporting nested field paths.
@@ -86,6 +88,7 @@ async fn estimate_multivector_vectors_per_row(
     dataset: &Dataset,
     column: &str,
     num_rows: usize,
+    fragments: Option<&[u32]>,
 ) -> Result<usize> {
     if num_rows == 0 {
         return Ok(1030);
@@ -96,7 +99,9 @@ async fn estimate_multivector_vectors_per_row(
     // Try a few random samples first (fast path).
     let sample_batch_size = std::cmp::min(64, num_rows);
     for _ in 0..8 {
-        let batch = dataset.sample(sample_batch_size, &projection).await?;
+        let batch = dataset
+            .sample(sample_batch_size, &projection, fragments)
+            .await?;
         let array = get_column_from_batch(&batch, column)?;
         let list_array = array.as_list::<i32>();
         for i in 0..list_array.len() {
@@ -114,6 +119,9 @@ async fn estimate_multivector_vectors_per_row(
     // flakiness when values are extremely sparse.
     let mut scanner = dataset.scan();
     scanner.project(&[column])?;
+    if let Some(fragments) = fragments {
+        scanner.with_fragments(resolve_scan_fragments(dataset, fragments)?);
+    }
     let column_expr = lance_datafusion::logical_expr::field_path_to_expr(column)?;
     scanner.filter_expr(column_expr.is_not_null());
     scanner.limit(Some(std::cmp::min(num_rows, 1024) as i64), None)?;
@@ -261,8 +269,15 @@ pub async fn maybe_sample_training_data(
     dataset: &Dataset,
     column: &str,
     sample_size_hint: usize,
+    fragment_ids: Option<&[u32]>,
 ) -> Result<FixedSizeListArray> {
-    let num_rows = dataset.count_rows(None).await?;
+    let num_rows = if let Some(fragment_ids) = fragment_ids {
+        let mut scanner = dataset.scan();
+        scanner.with_fragments(resolve_scan_fragments(dataset, fragment_ids)?);
+        scanner.count_rows().await? as usize
+    } else {
+        dataset.count_rows(None).await?
+    };
 
     let vector_field = dataset.schema().field(column).ok_or(Error::index(format!(
         "Sample training data: column {} does not exist in schema",
@@ -291,7 +306,8 @@ pub async fn maybe_sample_training_data(
             // Set a minimum sample size of 128 to avoid too small samples,
             // it's not a problem because 128 multivectors is just about 64 MiB
             let vectors_per_row =
-                estimate_multivector_vectors_per_row(dataset, column, num_rows).await?;
+                estimate_multivector_vectors_per_row(dataset, column, num_rows, fragment_ids)
+                    .await?;
             sample_size_hint.div_ceil(vectors_per_row).max(128)
         }
         _ => sample_size_hint,
@@ -306,11 +322,12 @@ pub async fn maybe_sample_training_data(
             num_rows,
             vector_field,
             is_nullable,
+            fragment_ids,
         )
         .await
     } else {
         // too small to require sampling
-        let batch = scan_all_training_data(dataset, column, is_nullable).await?;
+        let batch = scan_all_training_data(dataset, column, is_nullable, fragment_ids).await?;
         vector_column_to_fsl(&batch, column)
     }
 }
@@ -378,9 +395,13 @@ async fn scan_all_training_data(
     dataset: &Dataset,
     column: &str,
     is_nullable: bool,
+    fragment_ids: Option<&[u32]>,
 ) -> Result<RecordBatch> {
     let mut scanner = dataset.scan();
     scanner.project(&[column])?;
+    if let Some(fragment_ids) = fragment_ids {
+        scanner.with_fragments(resolve_scan_fragments(dataset, fragment_ids)?);
+    }
     if is_nullable {
         let column_expr = lance_datafusion::logical_expr::field_path_to_expr(column)?;
         scanner.filter_expr(column_expr.is_not_null());
@@ -406,14 +427,38 @@ async fn sample_training_data(
     num_rows: usize,
     vector_field: &lance_core::datatypes::Field,
     is_nullable: bool,
+    fragment_ids: Option<&[u32]>,
 ) -> Result<FixedSizeListArray> {
     let byte_width = vector_field
         .data_type()
         .byte_width_opt()
         .unwrap_or(4 * 1024);
 
+    if let Some(fragment_ids) = fragment_ids {
+        if !is_nullable {
+            let projection = dataset.schema().project(&[column])?;
+            let batch = dataset
+                .sample(sample_size_hint, &projection, Some(fragment_ids))
+                .await?;
+            return vector_column_to_fsl(&batch, column);
+        }
+        let scan = sample_training_data_scan_from_fragments(
+            dataset,
+            column,
+            sample_size_hint,
+            num_rows,
+            fragment_ids,
+        )?;
+        return match vector_field.data_type() {
+            DataType::FixedSizeList(_, _) => {
+                sample_nullable_fsl(column, sample_size_hint, byte_width, vector_field, scan).await
+            }
+            _ => sample_nullable_fallback(column, sample_size_hint, is_nullable, scan).await,
+        };
+    }
+
     match vector_field.data_type() {
-        DataType::FixedSizeList(_, _) if !is_nullable => {
+        DataType::FixedSizeList(_, _) if !is_nullable && fragment_ids.is_none() => {
             sample_fsl_uniform(
                 dataset,
                 column,
@@ -454,6 +499,127 @@ fn sample_training_data_scan(
     ))
 }
 
+/// Build a batch stream over fragment-limited random samples.
+///
+/// This is the only extra sampling helper we keep for ENT-1099. The existing
+/// range-based scan only works for dataset-wide offsets, while fragment-limited
+/// sampling must first map random offsets within the selected fragments to row
+/// addresses and then `take` those rows. Both nullable FSL and multivector
+/// paths reuse this stream to avoid duplicating fragment sampling logic.
+fn sample_training_data_scan_from_fragments(
+    dataset: &Dataset,
+    column: &str,
+    sample_size_hint: usize,
+    num_rows: usize,
+    fragment_ids: &[u32],
+) -> Result<Pin<Box<dyn Stream<Item = Result<RecordBatch>> + Send>>> {
+    if fragment_ids.is_empty() {
+        return Err(Error::invalid_input(
+            "Training fragment filter must not be empty".to_string(),
+        ));
+    }
+
+    let mut ordered_ids = fragment_ids.to_vec();
+    ordered_ids.sort_unstable();
+    ordered_ids.dedup();
+    let selected_fragments = dataset
+        .get_frags_from_ordered_ids(&ordered_ids)
+        .into_iter()
+        .zip(ordered_ids.iter())
+        .map(|(fragment, fragment_id)| {
+            fragment.ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "Unknown fragment id {fragment_id} in training fragment filter"
+                ))
+            })
+        })
+        .collect::<Result<Vec<_>>>()?;
+    let dataset = Arc::new(dataset.clone());
+    let projection = Arc::new(
+        ProjectionRequest::from(dataset.schema().project(&[column])?)
+            .into_projection_plan(dataset.clone())?,
+    );
+
+    let stream = stream::try_unfold(
+        (
+            dataset,
+            projection,
+            selected_fragments,
+            HashSet::<u64>::with_capacity(sample_size_hint.min(num_rows)),
+            SmallRng::from_os_rng(),
+        ),
+        move |(dataset, projection, selected_fragments, mut seen_offsets, mut rng)| async move {
+            if seen_offsets.len() >= num_rows {
+                return Ok(None);
+            }
+
+            let remaining = num_rows.saturating_sub(seen_offsets.len());
+            let target = sample_size_hint.saturating_mul(2).min(remaining);
+            let mut sampled_offsets = if remaining <= target.saturating_mul(4) {
+                let mut unseen_indices = (0..num_rows as u64)
+                    .filter(|index| !seen_offsets.contains(index))
+                    .collect::<Vec<_>>();
+                unseen_indices.shuffle(&mut rng);
+                unseen_indices.truncate(target);
+                seen_offsets.extend(unseen_indices.iter().copied());
+                unseen_indices
+            } else {
+                let mut sampled_offsets = Vec::with_capacity(target);
+                while sampled_offsets.len() < target {
+                    let index = rng.random_range(0..num_rows as u64);
+                    if seen_offsets.insert(index) {
+                        sampled_offsets.push(index);
+                    }
+                }
+                sampled_offsets
+            };
+            if sampled_offsets.is_empty() {
+                return Ok(None);
+            }
+            sampled_offsets.sort_unstable();
+
+            let mut row_addrs =
+                row_offsets_to_row_addresses(&selected_fragments, &sampled_offsets).await?;
+            row_addrs.sort_unstable();
+
+            let batch = TakeBuilder::try_new_from_addresses(
+                dataset.clone(),
+                row_addrs,
+                projection.clone(),
+            )?
+            .execute()
+            .await?;
+            Ok(Some((
+                batch,
+                (dataset, projection, selected_fragments, seen_offsets, rng),
+            )))
+        },
+    );
+    Ok(Box::pin(stream))
+}
+
+fn resolve_scan_fragments(
+    dataset: &Dataset,
+    fragment_ids: &[u32],
+) -> Result<Vec<lance_table::format::Fragment>> {
+    let mut ordered_ids = fragment_ids.to_vec();
+    ordered_ids.sort_unstable();
+    let fragments = dataset.get_frags_from_ordered_ids(&ordered_ids);
+    if let Some(missing_id) = fragments
+        .iter()
+        .zip(ordered_ids.iter())
+        .find_map(|(fragment, fragment_id)| fragment.is_none().then_some(*fragment_id))
+    {
+        return Err(Error::invalid_input(format!(
+            "Unknown fragment id {missing_id} in training fragment filter"
+        )));
+    }
+    Ok(fragments
+        .into_iter()
+        .map(|fragment| fragment.unwrap().metadata().clone())
+        .collect())
+}
+
 /// Build a FixedSizeListArray from raw flat value bytes.
 fn fsl_values_to_array(
     field: &lance_core::datatypes::Field,
@@ -505,13 +671,16 @@ fn fsl_values_to_array(
 /// accumulate non-null vector bytes directly into a flat buffer, dropping
 /// each source batch immediately. This keeps peak memory proportional to the
 /// output sample rather than the input scan.
-async fn sample_nullable_fsl(
+async fn sample_nullable_fsl<S>(
     column: &str,
     sample_size_hint: usize,
     byte_width: usize,
     vector_field: &lance_core::datatypes::Field,
-    mut scan: crate::dataset::scanner::DatasetRecordBatchStream,
-) -> Result<FixedSizeListArray> {
+    mut scan: S,
+) -> Result<FixedSizeListArray>
+where
+    S: Stream<Item = Result<RecordBatch>> + Unpin,
+{
     let mut values_buf = MutableBuffer::with_capacity(sample_size_hint * byte_width);
     let mut num_non_null: usize = 0;
 
@@ -537,7 +706,6 @@ async fn sample_nullable_fsl(
 
     fsl_values_to_array(vector_field, values_buf, num_rows_out)
 }
-
 /// True uniform random sampling for non-nullable FixedSizeList columns.
 ///
 /// Generates truly random row indices, sorts them, and fetches via
@@ -614,12 +782,15 @@ fn accumulate_fsl_values(
 /// Fallback sampling for non-FixedSizeList columns (e.g. multivector List
 /// columns). Collects batches and concatenates them. When `is_nullable` is
 /// true, filters null rows from each batch.
-async fn sample_nullable_fallback(
+async fn sample_nullable_fallback<S>(
     column: &str,
     sample_size_hint: usize,
     is_nullable: bool,
-    mut scan: crate::dataset::scanner::DatasetRecordBatchStream,
-) -> Result<FixedSizeListArray> {
+    mut scan: S,
+) -> Result<FixedSizeListArray>
+where
+    S: Stream<Item = Result<RecordBatch>> + Unpin,
+{
     let mut schema = None;
     let mut filtered = Vec::new();
     let mut num_non_null: usize = 0;
@@ -769,13 +940,12 @@ fn random_ranges(
 mod tests {
     use super::*;
 
-    use arrow_array::{Float32Array, types::Float32Type};
+    use crate::dataset::InsertBuilder;
+    use arrow_array::{ArrayRef, Float32Array, types::Float32Type};
     use arrow_schema::{DataType, Field};
     use lance_arrow::FixedSizeListArrayExt;
     use lance_datagen::{ArrayGeneratorExt, Dimension, RowCount, array, gen_batch};
 
-    use crate::dataset::InsertBuilder;
-
     #[rstest::rstest]
     #[test]
     fn test_random_ranges(
@@ -821,7 +991,7 @@ mod tests {
             .await
             .unwrap();
 
-        let training_data = maybe_sample_training_data(&dataset, "mv", 1000)
+        let training_data = maybe_sample_training_data(&dataset, "mv", 1000, None)
             .await
             .unwrap();
         assert_eq!(training_data.len(), 1000);
@@ -897,7 +1067,7 @@ mod tests {
             .await
             .unwrap();
 
-        let training_data = maybe_sample_training_data(&dataset, "vec", sample_size)
+        let training_data = maybe_sample_training_data(&dataset, "vec", sample_size, None)
             .await
             .unwrap();
 
@@ -983,7 +1153,7 @@ mod tests {
             .await
             .unwrap();
 
-        let n = estimate_multivector_vectors_per_row(&dataset, "mv", nrows)
+        let n = estimate_multivector_vectors_per_row(&dataset, "mv", nrows, None)
             .await
             .unwrap();
         assert_eq!(n, 1030);
diff --git a/rust/lance/src/io/commit.rs b/rust/lance/src/io/commit.rs
index ebab7eb6b4d..9e8db7caf0f 100644
--- a/rust/lance/src/io/commit.rs
+++ b/rust/lance/src/io/commit.rs
@@ -32,7 +32,7 @@ use lance_index::metrics::NoOpMetricsCollector;
 use lance_io::utils::CachedFileSize;
 use lance_table::format::{
     DETACHED_VERSION_MASK, DataStorageFormat, DeletionFile, Fragment, IndexMetadata, Manifest,
-    WriterVersion, is_detached_version, pb,
+    WriterVersion, is_detached_version, list_index_files_with_sizes, pb,
 };
 use lance_table::io::commit::{
     CommitConfig, CommitError, CommitHandler, ManifestLocation, ManifestNamingScheme,
@@ -48,6 +48,7 @@ use crate::dataset::{
     ManifestWriteConfig, NewTransactionResult, TRANSACTIONS_DIR, load_new_transactions,
     write_manifest_file,
 };
+use crate::index::DatasetIndexExt;
 use crate::index::DatasetIndexInternalExt;
 use crate::io::deletion::read_dataset_deletion_file;
 use crate::session::Session;
@@ -56,7 +57,7 @@ use crate::session::index_caches::IndexMetadataKey;
 use futures::future::Either;
 use futures::{StreamExt, TryFutureExt, TryStreamExt};
 use lance_core::{Error, Result};
-use lance_index::{DatasetIndexExt, is_system_index};
+use lance_index::is_system_index;
 use lance_io::object_store::ObjectStoreRegistry;
 use log;
 use object_store::path::Path;
@@ -72,7 +73,7 @@ pub mod namespace_manifest;
 mod s3_test;
 
 /// Read the transaction data from a transaction file.
-#[allow(dead_code)]
+#[cfg(test)]
 pub(crate) async fn read_transaction_file(
     object_store: &ObjectStore,
     base_path: &Path,
@@ -85,6 +86,29 @@ pub(crate) async fn read_transaction_file(
     transaction.try_into()
 }
 
+/// Best-effort delete of a transaction file that is no longer needed.
+///
+/// Logs a warning on failure rather than propagating the error, since the
+/// primary operation has already failed and the orphaned file will eventually
+/// be removed by GC.
+async fn cleanup_transaction_file(
+    object_store: &ObjectStore,
+    base_path: &Path,
+    transaction_file: &str,
+) {
+    if transaction_file.is_empty() {
+        return;
+    }
+    let path = base_path.child(TRANSACTIONS_DIR).child(transaction_file);
+    if let Err(e) = object_store.delete(&path).await {
+        log::warn!(
+            "Failed to clean up orphaned transaction file '{}': {}",
+            transaction_file,
+            e
+        );
+    }
+}
+
 /// Write a transaction to a file and return the relative path.
 pub(crate) async fn write_transaction_file(
     object_store: &ObjectStore,
@@ -152,7 +176,7 @@ async fn do_commit_new_dataset(
                 ref_path.clone(),
                 new_base_id,
                 branch_name.clone(),
-                transaction_file,
+                transaction_file.clone(),
             );
 
             let updated_indices = if let Some(index_section_pos) = source_manifest.index_section {
@@ -251,9 +275,13 @@ async fn do_commit_new_dataset(
             Ok((manifest, manifest_location))
         }
         Err(CommitError::CommitConflict) => {
+            cleanup_transaction_file(object_store, base_path, &transaction_file).await;
             Err(crate::Error::dataset_already_exists(base_path.to_string()))
         }
-        Err(CommitError::OtherError(err)) => Err(err),
+        Err(CommitError::OtherError(err)) => {
+            cleanup_transaction_file(object_store, base_path, &transaction_file).await;
+            Err(err)
+        }
     }
 }
 
@@ -361,6 +389,67 @@ fn check_storage_version(manifest: &mut Manifest) -> Result<()> {
     Ok(())
 }
 
+fn check_column_indices(manifest: &Manifest) -> Result<()> {
+    let data_storage_version = manifest.data_storage_format.lance_file_version()?;
+    if data_storage_version < LanceFileVersion::V2_1 {
+        return Ok(());
+    }
+
+    for fragment in manifest.fragments.iter() {
+        for data_file in &fragment.files {
+            if data_file.is_legacy_file() || data_file.column_indices.is_empty() {
+                continue;
+            }
+            if data_file.fields.len() != data_file.column_indices.len() {
+                return Err(Error::invalid_input(format!(
+                    "Data file '{}' (fragment {}) has {} field ids but {} column indices. \
+                     These must be the same length.",
+                    data_file.path,
+                    fragment.id,
+                    data_file.fields.len(),
+                    data_file.column_indices.len()
+                )));
+            }
+            let file_version = LanceFileVersion::try_from_major_minor(
+                data_file.file_major_version,
+                data_file.file_minor_version,
+            )?;
+            if file_version < LanceFileVersion::V2_1 {
+                continue;
+            }
+            for (field_id, column_index) in
+                data_file.fields.iter().zip(data_file.column_indices.iter())
+            {
+                // Field ids may not exist in the current schema after schema
+                // evolution (e.g. cast/drop column). Skip those.
+                let Some(field) = manifest.schema.field_by_id(*field_id) else {
+                    continue;
+                };
+                let needs_column = field.is_leaf() || field.is_packed_struct() || field.is_blob();
+                if needs_column && *column_index == -1 {
+                    return Err(Error::invalid_input(format!(
+                        "Field '{}' (id={}) in data file '{}' (fragment {}) \
+                         has column_index=-1, but leaf fields, packed structs, \
+                         and blob fields must have a valid column index in \
+                         file format 2.1+.",
+                        field.name, field_id, data_file.path, fragment.id
+                    )));
+                }
+                if !needs_column && *column_index != -1 {
+                    return Err(Error::invalid_input(format!(
+                        "Non-leaf field '{}' (id={}) in data file '{}' (fragment {}) \
+                         has column_index={}, but non-leaf fields should have \
+                         column_index=-1 in file format 2.1+. Only leaf fields, \
+                         packed structs, and blob fields should have column indices.",
+                        field.name, field_id, data_file.path, fragment.id, column_index
+                    )));
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
 /// Fix schema in case of duplicate field ids.
 ///
 /// See test dataset v0.10.5/corrupt_schema
@@ -403,17 +492,22 @@ fn fix_schema(manifest: &mut Manifest) -> Result<()> {
     // We iterate over files in reverse order so that we only map the last field id
     seen_fields.clear();
     for fragment in fragments.iter_mut() {
-        for field_id in fragment
-            .files
-            .iter_mut()
-            .rev()
-            .flat_map(|file| file.fields.iter_mut())
-        {
-            if let Some(new_field_id) = old_field_id_mapping.get(field_id)
-                && seen_fields.insert(*field_id)
-            {
-                *field_id = *new_field_id;
-            }
+        for file in fragment.files.iter_mut().rev() {
+            let new_fields: Arc<[i32]> = file
+                .fields
+                .iter()
+                .map(|field_id| {
+                    if let Some(new_field_id) = old_field_id_mapping.get(field_id)
+                        && seen_fields.insert(*field_id)
+                    {
+                        *new_field_id
+                    } else {
+                        *field_id
+                    }
+                })
+                .collect::<Vec<_>>()
+                .into();
+            file.fields = new_fields;
         }
         seen_fields.clear();
     }
@@ -564,6 +658,7 @@ fn must_recalculate_fragment_bitmap(
 /// Update indices with new fields.
 ///
 /// Indices might be missing `fragment_bitmap`, so this function will add it.
+/// Indices might also be missing `files` (file sizes), so this function will collect them.
 async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Result<()> {
     let needs_recalculating = match detect_overlapping_fragments(indices) {
         Ok(()) => vec![],
@@ -571,7 +666,7 @@ async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Re
             bad_indices.into_iter().map(|(name, _)| name).collect()
         }
     };
-    for index in indices {
+    for index in indices.iter_mut() {
         if needs_recalculating.contains(&index.name)
             || must_recalculate_fragment_bitmap(index, dataset.manifest.writer_version.as_ref())
                 && !is_system_index(index)
@@ -596,6 +691,38 @@ async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Re
                 index.uuid
             );
         }
+
+        // Migrate file sizes for indices that don't have them.
+        // Use indice_files_dir to handle shallow-cloned indices with base_id.
+        if index.files.is_none() && !is_system_index(index) {
+            let result = async {
+                let index_dir = dataset
+                    .indice_files_dir(index)?
+                    .child(index.uuid.to_string());
+                list_index_files_with_sizes(&dataset.object_store, &index_dir).await
+            }
+            .await;
+            match result {
+                Ok(files) => {
+                    log::debug!(
+                        "Migrated file sizes for index {} (uuid: {}): {} files",
+                        index.name,
+                        index.uuid,
+                        files.len()
+                    );
+                    index.files = Some(files);
+                }
+                Err(e) => {
+                    // Log but don't fail - file sizes are optional
+                    log::debug!(
+                        "Could not collect file sizes for index {} (uuid: {}): {}",
+                        index.name,
+                        index.uuid,
+                        e
+                    );
+                }
+            }
+        }
     }
 
     Ok(())
@@ -686,6 +813,7 @@ pub(crate) async fn do_commit_detached_transaction(
         // fix_schema and check_storage_version are just for sanity-checking and consistency
         fix_schema(&mut manifest)?;
         check_storage_version(&mut manifest)?;
+        check_column_indices(&manifest)?;
         migrate_indices(dataset, &mut indices).await?;
 
         // Try to commit the manifest
@@ -716,6 +844,7 @@ pub(crate) async fn do_commit_detached_transaction(
             }
             Err(CommitError::OtherError(err)) => {
                 // If other error, return
+                cleanup_transaction_file(object_store, &dataset.base, &transaction_file).await;
                 return Err(err);
             }
         }
@@ -723,6 +852,7 @@ pub(crate) async fn do_commit_detached_transaction(
 
     // This should be extremely unlikely.  There should not be *that* many detached commits.  If
     // this happens then it seems more likely there is a bug in our random u64 generation.
+    cleanup_transaction_file(object_store, &dataset.base, &transaction_file).await;
     Err(crate::Error::commit_conflict_source(
         0,
         format!(
@@ -808,6 +938,9 @@ pub(crate) async fn commit_transaction(
     // Other transactions that may have been committed since the read_version.
     // We keep pair of (version, transaction). No other transactions to check initially
     let mut other_transactions: Vec<(u64, Arc<Transaction>)>;
+    // Track the transaction file written in the current loop iteration so we can
+    // delete it if the commit ultimately fails.
+    let mut current_transaction_file = String::new();
 
     while backoff.attempt() < num_attempts {
         // We are pessimistic here and assume there may be other transactions
@@ -835,11 +968,12 @@ pub(crate) async fn commit_transaction(
             transaction = rebase.finish(&dataset).await?;
         }
 
-        let transaction_file = if !write_config.disable_transaction_file() {
+        current_transaction_file = if !write_config.disable_transaction_file() {
             write_transaction_file(object_store, &dataset.base, &transaction).await?
         } else {
             String::new()
         };
+        let transaction_file = current_transaction_file.as_str();
 
         target_version = dataset.manifest.version + 1;
         if is_detached_version(target_version) {
@@ -856,7 +990,7 @@ pub(crate) async fn commit_transaction(
                     &dataset.base,
                     version,
                     write_config,
-                    &transaction_file,
+                    transaction_file,
                     &dataset.manifest,
                 )
                 .await?
@@ -864,7 +998,7 @@ pub(crate) async fn commit_transaction(
             _ => transaction.build_manifest(
                 Some(dataset.manifest.as_ref()),
                 dataset.load_indices().await?.as_ref().clone(),
-                &transaction_file,
+                transaction_file,
                 write_config,
             )?,
         };
@@ -883,6 +1017,7 @@ pub(crate) async fn commit_transaction(
         fix_schema(&mut manifest)?;
 
         check_storage_version(&mut manifest)?;
+        check_column_indices(&manifest)?;
 
         migrate_indices(&dataset, &mut indices).await?;
 
@@ -956,6 +1091,14 @@ pub(crate) async fn commit_transaction(
                 }
 
                 if next_attempt_i < num_attempts {
+                    // The transaction file from this attempt is now stale; clean it up
+                    // before the next attempt writes a new one (possibly rebased).
+                    cleanup_transaction_file(
+                        object_store,
+                        &dataset.base,
+                        &current_transaction_file,
+                    )
+                    .await;
                     tokio::time::sleep(backoff.next_backoff()).await;
                     continue;
                 } else {
@@ -963,12 +1106,14 @@ pub(crate) async fn commit_transaction(
                 }
             }
             Err(CommitError::OtherError(err)) => {
-                // If other error, return
+                cleanup_transaction_file(object_store, &dataset.base, &current_transaction_file)
+                    .await;
                 return Err(err);
             }
         }
     }
 
+    cleanup_transaction_file(object_store, &dataset.base, &current_transaction_file).await;
     Err(crate::Error::commit_conflict_source(
         target_version,
         format!(
@@ -995,7 +1140,7 @@ mod tests {
     use lance_linalg::distance::MetricType;
     use lance_table::format::{DataFile, DataStorageFormat};
     use lance_table::io::commit::{
-        CommitLease, CommitLock, RenameCommitHandler, UnsafeCommitHandler,
+        CommitLease, CommitLock, ManifestWriter, RenameCommitHandler, UnsafeCommitHandler,
     };
     use lance_testing::datagen::generate_random_array;
 
@@ -1216,9 +1361,16 @@ mod tests {
             .collect();
 
         let results = join_all(futures).await;
-        for result in results {
-            assert!(matches!(result, Ok(Ok(_))), "{:?}", result);
-        }
+        let success_count = results
+            .iter()
+            .filter(|result| matches!(result, Ok(Ok(_))))
+            .count();
+        let retryable_count = results
+            .iter()
+            .filter(|result| matches!(result, Ok(Err(Error::RetryableCommitConflict { .. }))))
+            .count();
+        assert_eq!(success_count, 2, "{results:?}");
+        assert_eq!(retryable_count, 1, "{results:?}");
 
         // Validate that each version has the anticipated number of indexes
         let dataset = dataset.checkout_version(1).await.unwrap();
@@ -1241,12 +1393,7 @@ mod tests {
             assert_eq!(indices[0].fields, vec![0]);
         }
 
-        let dataset = dataset.checkout_version(4).await.unwrap();
-        let indices = dataset.load_indices().await.unwrap();
-        assert_eq!(indices.len(), 2);
-        let mut fields: Vec<i32> = indices.iter().flat_map(|i| i.fields.clone()).collect();
-        fields.sort();
-        assert_eq!(fields, vec![0, 1]);
+        assert!(dataset.checkout_version(4).await.is_err());
     }
 
     #[tokio::test]
@@ -1600,4 +1747,287 @@ mod tests {
         ];
         assert_eq!(manifest.fragments.as_ref(), &expected_fragments);
     }
+
+    /// A CommitHandler that always fails with OtherError, used to simulate
+    /// a manifest write failure so we can verify orphaned transaction files
+    /// are cleaned up.
+    #[derive(Debug)]
+    struct FailingCommitHandler;
+
+    #[async_trait::async_trait]
+    impl CommitHandler for FailingCommitHandler {
+        async fn commit(
+            &self,
+            _manifest: &mut Manifest,
+            _indices: Option<Vec<IndexMetadata>>,
+            _base_path: &Path,
+            _object_store: &ObjectStore,
+            _manifest_writer: ManifestWriter,
+            _naming_scheme: ManifestNamingScheme,
+            _transaction: Option<lance_table::format::Transaction>,
+        ) -> std::result::Result<ManifestLocation, CommitError> {
+            Err(CommitError::OtherError(lance_core::Error::io(
+                "simulated commit failure",
+            )))
+        }
+    }
+
+    fn count_txn_files(uri: &str) -> usize {
+        let tx_dir = std::path::Path::new(uri).join("_transactions");
+        std::fs::read_dir(&tx_dir)
+            .map(|rd| rd.filter_map(|e| e.ok()).count())
+            .unwrap_or(0)
+    }
+
+    #[tokio::test]
+    async fn test_transaction_file_cleanup_on_commit_failure() {
+        let tmp = TempStrDir::default();
+        let uri = tmp.as_str();
+
+        // Create initial dataset with a normal commit handler.
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "x",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], schema.clone());
+        Dataset::write(reader, uri, None).await.unwrap();
+
+        let txn_files_before = count_txn_files(uri);
+
+        // Attempt to append with a commit handler that always fails.
+        let params = WriteParams {
+            mode: WriteMode::Append,
+            commit_handler: Some(Arc::new(FailingCommitHandler)),
+            ..Default::default()
+        };
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let result = Dataset::write(reader, uri, Some(params)).await;
+        assert!(result.is_err(), "expected commit to fail");
+
+        // The failed commit must not leave any extra transaction files behind.
+        let txn_files_after = count_txn_files(uri);
+        assert_eq!(
+            txn_files_after,
+            txn_files_before,
+            "failed commit left {extra} orphaned transaction file(s)",
+            extra = txn_files_after.saturating_sub(txn_files_before),
+        );
+    }
+    /// Helper to build a simple manifest for check_column_indices tests.
+    fn make_manifest_with_file(
+        schema: Schema,
+        data_file: DataFile,
+        data_storage_version: LanceFileVersion,
+    ) -> Manifest {
+        let fragment = Fragment {
+            id: 0,
+            files: vec![data_file],
+            deletion_file: None,
+            row_id_meta: None,
+            physical_rows: Some(100),
+            last_updated_at_version_meta: None,
+            created_at_version_meta: None,
+        };
+        Manifest::new(
+            schema,
+            Arc::new(vec![fragment]),
+            DataStorageFormat::new(data_storage_version),
+            HashMap::new(),
+        )
+    }
+
+    #[test]
+    fn test_check_column_indices_rejects_struct_with_column() {
+        // Struct (non-leaf) field with column_index=0 in v2.1 should be rejected.
+        let mut struct_field = Field::try_from(ArrowField::new(
+            "s",
+            DataType::Struct(vec![ArrowField::new("x", DataType::Int32, false)].into()),
+            false,
+        ))
+        .unwrap();
+        struct_field.set_id(-1, &mut 0);
+
+        let schema = Schema {
+            fields: vec![struct_field],
+            metadata: Default::default(),
+        };
+
+        // field ids: struct=0, leaf=1; give struct a real column_index (wrong)
+        let data_file = DataFile::new("data.lance", vec![0, 1], vec![0, 1], 2, 1, None, None);
+        let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1);
+        let result = check_column_indices(&manifest);
+        assert!(
+            result.is_err(),
+            "Expected error for struct with column_index=0"
+        );
+        let msg = result.unwrap_err().to_string();
+        assert!(msg.contains("Non-leaf field"), "{msg}");
+    }
+
+    #[test]
+    fn test_check_column_indices_rejects_list_with_column() {
+        // List (non-leaf) field with column_index=0 in v2.1 should be rejected.
+        let mut list_field = Field::try_from(ArrowField::new(
+            "l",
+            DataType::List(Arc::new(ArrowField::new("item", DataType::Int32, true))),
+            false,
+        ))
+        .unwrap();
+        list_field.set_id(-1, &mut 0);
+
+        let schema = Schema {
+            fields: vec![list_field],
+            metadata: Default::default(),
+        };
+
+        // field ids: list=0, item=1; give list a real column_index (wrong)
+        let data_file = DataFile::new("data.lance", vec![0, 1], vec![0, 1], 2, 1, None, None);
+        let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1);
+        let result = check_column_indices(&manifest);
+        assert!(
+            result.is_err(),
+            "Expected error for list with column_index=0"
+        );
+        let msg = result.unwrap_err().to_string();
+        assert!(msg.contains("Non-leaf field"), "{msg}");
+    }
+
+    #[test]
+    fn test_check_column_indices_allows_correct_v21() {
+        // Non-leaf with column_index=-1 and leaf with column_index>=0 should pass.
+        let mut struct_field = Field::try_from(ArrowField::new(
+            "s",
+            DataType::Struct(vec![ArrowField::new("x", DataType::Int32, false)].into()),
+            false,
+        ))
+        .unwrap();
+        struct_field.set_id(-1, &mut 0);
+
+        let schema = Schema {
+            fields: vec![struct_field],
+            metadata: Default::default(),
+        };
+
+        // struct=-1 (correct), leaf=0 (correct)
+        let data_file = DataFile::new("data.lance", vec![0, 1], vec![-1, 0], 2, 1, None, None);
+        let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1);
+        assert!(check_column_indices(&manifest).is_ok());
+    }
+
+    #[test]
+    fn test_check_column_indices_allows_packed_struct() {
+        // Packed struct with a real column_index in v2.1 should be allowed.
+        let mut struct_field = Field::try_from(ArrowField::new(
+            "s",
+            DataType::Struct(vec![ArrowField::new("x", DataType::Int32, false)].into()),
+            false,
+        ))
+        .unwrap();
+        struct_field.set_id(-1, &mut 0);
+        struct_field
+            .metadata
+            .insert("lance-encoding:packed".to_string(), "true".to_string());
+
+        let schema = Schema {
+            fields: vec![struct_field],
+            metadata: Default::default(),
+        };
+
+        // packed struct=0 (allowed), leaf=1
+        let data_file = DataFile::new("data.lance", vec![0, 1], vec![0, 1], 2, 1, None, None);
+        let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1);
+        assert!(check_column_indices(&manifest).is_ok());
+    }
+
+    #[test]
+    fn test_check_column_indices_skips_v20() {
+        // Non-leaf with column_index>=0 in v2.0 should be allowed (no validation).
+        let mut struct_field = Field::try_from(ArrowField::new(
+            "s",
+            DataType::Struct(vec![ArrowField::new("x", DataType::Int32, false)].into()),
+            false,
+        ))
+        .unwrap();
+        struct_field.set_id(-1, &mut 0);
+
+        let schema = Schema {
+            fields: vec![struct_field],
+            metadata: Default::default(),
+        };
+
+        let data_file = DataFile::new("data.lance", vec![0, 1], vec![0, 1], 2, 0, None, None);
+        let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_0);
+        assert!(check_column_indices(&manifest).is_ok());
+    }
+
+    #[test]
+    fn test_check_column_indices_rejects_mismatched_lengths() {
+        // fields and column_indices must have the same length.
+        let mut leaf_field = Field::try_from(ArrowField::new("x", DataType::Int32, false)).unwrap();
+        leaf_field.set_id(-1, &mut 0);
+
+        let schema = Schema {
+            fields: vec![leaf_field],
+            metadata: Default::default(),
+        };
+
+        // 1 field id but 2 column indices
+        let data_file = DataFile::new("data.lance", vec![0], vec![0, 1], 2, 1, None, None);
+        let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1);
+        let result = check_column_indices(&manifest);
+        assert!(result.is_err(), "Expected error for mismatched lengths");
+        let msg = result.unwrap_err().to_string();
+        assert!(msg.contains("1 field ids but 2 column indices"), "{msg}");
+    }
+
+    #[test]
+    fn test_check_column_indices_skips_unknown_field_id() {
+        // A field id not present in the schema is skipped (schema evolution).
+        let mut leaf_field = Field::try_from(ArrowField::new("x", DataType::Int32, false)).unwrap();
+        leaf_field.set_id(-1, &mut 0);
+
+        let schema = Schema {
+            fields: vec![leaf_field],
+            metadata: Default::default(),
+        };
+
+        // field id 99 does not exist in the schema — should be skipped
+        let data_file = DataFile::new("data.lance", vec![0, 99], vec![0, 1], 2, 1, None, None);
+        let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1);
+        assert!(check_column_indices(&manifest).is_ok());
+    }
+
+    #[test]
+    fn test_check_column_indices_rejects_leaf_with_negative_one() {
+        // A leaf field with column_index=-1 in v2.1 should be rejected.
+        let mut struct_field = Field::try_from(ArrowField::new(
+            "s",
+            DataType::Struct(vec![ArrowField::new("x", DataType::Int32, false)].into()),
+            false,
+        ))
+        .unwrap();
+        struct_field.set_id(-1, &mut 0);
+
+        let schema = Schema {
+            fields: vec![struct_field],
+            metadata: Default::default(),
+        };
+
+        // struct=-1 (correct), but leaf=-1 (wrong — leaf must have a real column)
+        let data_file = DataFile::new("data.lance", vec![0, 1], vec![-1, -1], 2, 1, None, None);
+        let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1);
+        let result = check_column_indices(&manifest);
+        assert!(
+            result.is_err(),
+            "Expected error for leaf with column_index=-1"
+        );
+        let msg = result.unwrap_err().to_string();
+        assert!(msg.contains("must have a valid column index"), "{msg}");
+    }
 }
diff --git a/rust/lance/src/io/commit/conflict_resolver.rs b/rust/lance/src/io/commit/conflict_resolver.rs
index 450bd03a964..ec03ba596db 100644
--- a/rust/lance/src/io/commit/conflict_resolver.rs
+++ b/rust/lance/src/io/commit/conflict_resolver.rs
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use crate::index::DatasetIndexExt;
 use crate::index::frag_reuse::{build_frag_reuse_index_metadata, load_frag_reuse_index_details};
 use crate::index::mem_wal::{load_mem_wal_index_details, new_mem_wal_index_meta};
 use crate::io::deletion::read_dataset_deletion_file;
@@ -501,8 +502,6 @@ impl<'a> TransactionRebase<'a> {
                 Operation::Append { .. }
                 | Operation::Clone { .. }
                 | Operation::UpdateBases { .. } => Ok(()),
-                // Indices are identified by UUIDs, so they shouldn't conflict.
-                // unless it is the same frag reuse index or MemWAL index
                 Operation::CreateIndex {
                     new_indices: created_indices,
                     ..
@@ -518,9 +517,20 @@ impl<'a> TransactionRebase<'a> {
                     let other_has_mem_wal = created_indices
                         .iter()
                         .any(|idx| idx.name == MEM_WAL_INDEX_NAME);
+                    let has_regular_name_conflict = new_indices
+                        .iter()
+                        .filter(|idx| {
+                            idx.name != FRAG_REUSE_INDEX_NAME && idx.name != MEM_WAL_INDEX_NAME
+                        })
+                        .any(|new_index| {
+                            created_indices
+                                .iter()
+                                .any(|created_index| created_index.name == new_index.name)
+                        });
 
                     if (self_has_frag_reuse && other_has_frag_reuse)
                         || (self_has_mem_wal && other_has_mem_wal)
+                        || has_regular_name_conflict
                     {
                         Err(self.retryable_conflict_err(other_transaction, other_version))
                     } else {
@@ -597,7 +607,7 @@ impl<'a> TransactionRebase<'a> {
                         .flat_map(|idx| idx.fields.iter())
                         .collect::<HashSet<_>>();
                     for replacement in replacements {
-                        for field in &replacement.1.fields {
+                        for field in replacement.1.fields.iter() {
                             if newly_indexed_fields.contains(&field) {
                                 return Err(
                                     self.retryable_conflict_err(other_transaction, other_version)
@@ -888,7 +898,7 @@ impl<'a> TransactionRebase<'a> {
                         .flat_map(|idx| idx.fields.iter())
                         .collect::<HashSet<_>>();
                     for replacement in replacements {
-                        for field in &replacement.1.fields {
+                        for field in replacement.1.fields.iter() {
                             if newly_indexed_fields.contains(&field) {
                                 return Err(
                                     self.retryable_conflict_err(other_transaction, other_version)
@@ -923,7 +933,7 @@ impl<'a> TransactionRebase<'a> {
                                 continue;
                             }
 
-                            for field in &replacement.1.fields {
+                            for field in replacement.1.fields.iter() {
                                 if other_replacement.1.fields.contains(field) {
                                     return Err(self
                                         .retryable_conflict_err(other_transaction, other_version));
@@ -1085,7 +1095,7 @@ impl<'a> TransactionRebase<'a> {
                         .transaction
                         .operation
                         .upsert_key_conflict(&other_transaction.operation)
-                        | self
+                        || self
                             .transaction
                             .operation
                             .modifies_same_metadata(&other_transaction.operation)
@@ -1128,7 +1138,7 @@ impl<'a> TransactionRebase<'a> {
                     merged_generations: other_merged_generations,
                 } => {
                     // Two UpdateMemWalState transactions conflict if they're updating
-                    // the same region's merged_generation
+                    // the same shard's merged_generation
                     self.check_merged_generations_conflict(
                         other_merged_generations,
                         self_merged_generations,
@@ -1235,11 +1245,11 @@ impl<'a> TransactionRebase<'a> {
         other_transaction: &Transaction,
         other_version: u64,
     ) -> Result<()> {
-        // Check if any region has conflicting updates
+        // Check if any shard has conflicting updates
         for committed_mg in committed {
             for to_commit_mg in to_commit {
-                if committed_mg.region_id == to_commit_mg.region_id {
-                    // Same region being updated
+                if committed_mg.shard_id == to_commit_mg.shard_id {
+                    // Same shard being updated
                     // If committed >= to_commit, data already merged or superseded - abort without retry
                     // If committed < to_commit, can retry with new state
                     if committed_mg.generation >= to_commit_mg.generation {
@@ -1440,7 +1450,11 @@ impl<'a> TransactionRebase<'a> {
     }
 
     async fn finish_create_index(mut self, dataset: &Dataset) -> Result<Transaction> {
-        if let Operation::CreateIndex { new_indices, .. } = &mut self.transaction.operation {
+        if let Operation::CreateIndex {
+            new_indices,
+            removed_indices,
+        } = &mut self.transaction.operation
+        {
             // Handle FRAG_REUSE_INDEX rebasing
             let has_frag_reuse = new_indices
                 .iter()
@@ -1502,13 +1516,13 @@ impl<'a> TransactionRebase<'a> {
                 let current_meta = new_indices.remove(pos);
                 let mut details = load_mem_wal_index_details(current_meta)?;
 
-                // Merge conflicting merged_generations - for each region, keep higher generation
+                // Merge conflicting merged_generations - for each shard, keep higher generation
                 // We own self so we can consume conflicting_mem_wal_merged_gens directly
                 for new_mg in self.conflicting_mem_wal_merged_gens {
                     if let Some(existing) = details
                         .merged_generations
                         .iter_mut()
-                        .find(|mg| mg.region_id == new_mg.region_id)
+                        .find(|mg| mg.shard_id == new_mg.shard_id)
                     {
                         if new_mg.generation > existing.generation {
                             existing.generation = new_mg.generation;
@@ -1522,6 +1536,25 @@ impl<'a> TransactionRebase<'a> {
                 new_indices.push(new_meta);
             }
 
+            for singleton_name in [FRAG_REUSE_INDEX_NAME, MEM_WAL_INDEX_NAME] {
+                if new_indices.iter().any(|idx| idx.name == singleton_name) {
+                    for existing_idx in dataset
+                        .load_indices()
+                        .await?
+                        .iter()
+                        .filter(|idx| idx.name == singleton_name)
+                        .cloned()
+                    {
+                        if !removed_indices
+                            .iter()
+                            .any(|removed_idx| removed_idx.uuid == existing_idx.uuid)
+                        {
+                            removed_indices.push(existing_idx);
+                        }
+                    }
+                }
+            }
+
             Ok(self.transaction)
         } else {
             Err(wrong_operation_err(&self.transaction.operation))
@@ -2142,6 +2175,7 @@ mod tests {
             index_version: 0,
             created_at: None, // Test index, not setting timestamp
             base_id: None,
+            files: None,
         };
         let fragment0 = Fragment::new(0);
         let fragment1 = Fragment::new(1);
@@ -2296,10 +2330,10 @@ mod tests {
                     new_indices: vec![index0.clone()],
                     removed_indices: vec![index0],
                 },
-                // Will only conflict with operations that modify row ids.
+                // Conflicts with row-id-changing operations and same-name CreateIndex.
                 [
                     Compatible,    // append
-                    Compatible,    // create index
+                    Retryable,     // create index
                     Compatible,    // delete
                     Compatible,    // merge
                     NotCompatible, // overwrite
@@ -2629,6 +2663,104 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_create_index_conflicts_only_on_same_name() {
+        let index0 = IndexMetadata {
+            uuid: uuid::Uuid::new_v4(),
+            name: "test".to_string(),
+            fields: vec![0],
+            dataset_version: 1,
+            fragment_bitmap: None,
+            index_details: None,
+            index_version: 0,
+            created_at: None,
+            base_id: None,
+            files: None,
+        };
+        let index1 = IndexMetadata {
+            uuid: uuid::Uuid::new_v4(),
+            name: "other".to_string(),
+            ..index0.clone()
+        };
+
+        let txn = Transaction::new(
+            0,
+            Operation::CreateIndex {
+                new_indices: vec![index0.clone()],
+                removed_indices: vec![],
+            },
+            None,
+        );
+        let mut rebase = TransactionRebase {
+            transaction: txn,
+            initial_fragments: HashMap::new(),
+            modified_fragment_ids: HashSet::new(),
+            affected_rows: None,
+            conflicting_frag_reuse_indices: Vec::new(),
+            conflicting_mem_wal_merged_gens: Vec::new(),
+        };
+
+        let same_name = Transaction::new(
+            0,
+            Operation::CreateIndex {
+                new_indices: vec![IndexMetadata {
+                    uuid: uuid::Uuid::new_v4(),
+                    ..index0
+                }],
+                removed_indices: vec![],
+            },
+            None,
+        );
+        let different_name = Transaction::new(
+            0,
+            Operation::CreateIndex {
+                new_indices: vec![index1],
+                removed_indices: vec![],
+            },
+            None,
+        );
+
+        let same_name_result = rebase.check_txn(&same_name, 1);
+        assert!(
+            matches!(same_name_result, Err(Error::RetryableCommitConflict { .. })),
+            "Expected retryable conflict for same-name CreateIndex, got {:?}",
+            same_name_result
+        );
+
+        let mut rebase = TransactionRebase {
+            transaction: Transaction::new(
+                0,
+                Operation::CreateIndex {
+                    new_indices: vec![IndexMetadata {
+                        uuid: uuid::Uuid::new_v4(),
+                        name: "test".to_string(),
+                        fields: vec![0],
+                        dataset_version: 1,
+                        fragment_bitmap: None,
+                        index_details: None,
+                        index_version: 0,
+                        created_at: None,
+                        base_id: None,
+                        files: None,
+                    }],
+                    removed_indices: vec![],
+                },
+                None,
+            ),
+            initial_fragments: HashMap::new(),
+            modified_fragment_ids: HashSet::new(),
+            affected_rows: None,
+            conflicting_frag_reuse_indices: Vec::new(),
+            conflicting_mem_wal_merged_gens: Vec::new(),
+        };
+        let different_name_result = rebase.check_txn(&different_name, 1);
+        assert!(
+            different_name_result.is_ok(),
+            "Expected compatibility for different-name CreateIndex, got {:?}",
+            different_name_result
+        );
+    }
+
     #[tokio::test]
     async fn test_add_bases_non_conflicting() {
         let dataset = test_dataset(10, 2).await;
@@ -3143,13 +3275,13 @@ mod tests {
     #[test]
     fn test_merged_generations_conflict_lower_generation_fails() {
         // Test: committed generation >= to_commit generation should be incompatible (no retry)
-        let region = Uuid::new_v4();
+        let shard = Uuid::new_v4();
 
         // Committed has generation 10, we're trying to commit generation 5
         let committed_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 10)],
+                merged_generations: vec![MergedGeneration::new(shard, 10)],
             },
             None,
         );
@@ -3157,7 +3289,7 @@ mod tests {
         let to_commit_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 5)],
+                merged_generations: vec![MergedGeneration::new(shard, 5)],
             },
             None,
         );
@@ -3182,12 +3314,12 @@ mod tests {
     #[test]
     fn test_merged_generations_conflict_equal_generation_fails() {
         // Test: committed generation == to_commit generation should be incompatible (no retry)
-        let region = Uuid::new_v4();
+        let shard = Uuid::new_v4();
 
         let committed_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 10)],
+                merged_generations: vec![MergedGeneration::new(shard, 10)],
             },
             None,
         );
@@ -3195,7 +3327,7 @@ mod tests {
         let to_commit_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 10)],
+                merged_generations: vec![MergedGeneration::new(shard, 10)],
             },
             None,
         );
@@ -3220,13 +3352,13 @@ mod tests {
     #[test]
     fn test_merged_generations_conflict_higher_generation_retryable() {
         // Test: committed generation < to_commit generation should be retryable
-        let region = Uuid::new_v4();
+        let shard = Uuid::new_v4();
 
         // Committed has generation 5, we're trying to commit generation 10
         let committed_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 5)],
+                merged_generations: vec![MergedGeneration::new(shard, 5)],
             },
             None,
         );
@@ -3234,7 +3366,7 @@ mod tests {
         let to_commit_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 10)],
+                merged_generations: vec![MergedGeneration::new(shard, 10)],
             },
             None,
         );
@@ -3257,15 +3389,15 @@ mod tests {
     }
 
     #[test]
-    fn test_merged_generations_different_regions_ok() {
-        // Test: different regions should not conflict
-        let region1 = Uuid::new_v4();
-        let region2 = Uuid::new_v4();
+    fn test_merged_generations_different_shards_ok() {
+        // Test: different shards should not conflict
+        let shard1 = Uuid::new_v4();
+        let shard2 = Uuid::new_v4();
 
         let committed_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region1, 10)],
+                merged_generations: vec![MergedGeneration::new(shard1, 10)],
             },
             None,
         );
@@ -3273,7 +3405,7 @@ mod tests {
         let to_commit_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region2, 5)],
+                merged_generations: vec![MergedGeneration::new(shard2, 5)],
             },
             None,
         );
@@ -3290,7 +3422,7 @@ mod tests {
         let result = rebase.check_txn(&committed_txn, 1);
         assert!(
             result.is_ok(),
-            "Expected OK for different regions, got {:?}",
+            "Expected OK for different shards, got {:?}",
             result
         );
     }
@@ -3300,11 +3432,11 @@ mod tests {
         use crate::index::mem_wal::new_mem_wal_index_meta;
         use lance_index::mem_wal::MemWalIndexDetails;
 
-        let region = Uuid::new_v4();
+        let shard = Uuid::new_v4();
 
         // Create a MemWalIndex with merged_generations
         let details = MemWalIndexDetails {
-            merged_generations: vec![MergedGeneration::new(region, 10)],
+            merged_generations: vec![MergedGeneration::new(shard, 10)],
             ..Default::default()
         };
         let mem_wal_index = new_mem_wal_index_meta(1, details).unwrap();
@@ -3323,7 +3455,7 @@ mod tests {
         let to_commit_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 5)],
+                merged_generations: vec![MergedGeneration::new(shard, 5)],
             },
             None,
         );
@@ -3348,7 +3480,7 @@ mod tests {
         let to_commit_txn_higher = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 15)],
+                merged_generations: vec![MergedGeneration::new(shard, 15)],
             },
             None,
         );
@@ -3375,7 +3507,7 @@ mod tests {
         use crate::index::mem_wal::new_mem_wal_index_meta;
         use lance_index::mem_wal::MemWalIndexDetails;
 
-        let region = Uuid::new_v4();
+        let shard = Uuid::new_v4();
 
         // CreateIndex with MemWalIndex (no merged_generations initially)
         let details = MemWalIndexDetails::default();
@@ -3394,7 +3526,7 @@ mod tests {
         let committed_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(region, 10)],
+                merged_generations: vec![MergedGeneration::new(shard, 10)],
             },
             None,
         );
@@ -3419,7 +3551,7 @@ mod tests {
 
         // Verify that merged_generations were collected
         assert_eq!(rebase.conflicting_mem_wal_merged_gens.len(), 1);
-        assert_eq!(rebase.conflicting_mem_wal_merged_gens[0].region_id, region);
+        assert_eq!(rebase.conflicting_mem_wal_merged_gens[0].shard_id, shard);
         assert_eq!(rebase.conflicting_mem_wal_merged_gens[0].generation, 10);
     }
 
diff --git a/rust/lance/src/io/commit/namespace_manifest.rs b/rust/lance/src/io/commit/namespace_manifest.rs
index 2593ad89dc6..5b3de301886 100644
--- a/rust/lance/src/io/commit/namespace_manifest.rs
+++ b/rust/lance/src/io/commit/namespace_manifest.rs
@@ -16,14 +16,14 @@ use object_store::path::Path;
 
 #[derive(Debug)]
 pub struct LanceNamespaceExternalManifestStore {
-    namespace: Arc<dyn LanceNamespace>,
+    namespace_client: Arc<dyn LanceNamespace>,
     table_id: Vec<String>,
 }
 
 impl LanceNamespaceExternalManifestStore {
-    pub fn new(namespace: Arc<dyn LanceNamespace>, table_id: Vec<String>) -> Self {
+    pub fn new(namespace_client: Arc<dyn LanceNamespace>, table_id: Vec<String>) -> Self {
         Self {
-            namespace,
+            namespace_client,
             table_id,
         }
     }
@@ -38,7 +38,10 @@ impl ExternalManifestStore for LanceNamespaceExternalManifestStore {
             ..Default::default()
         };
 
-        let response = self.namespace.describe_table_version(request).await?;
+        let response = self
+            .namespace_client
+            .describe_table_version(request)
+            .await?;
 
         // Namespace returns full path (relative to object store root)
         Ok(response.version.manifest_path)
@@ -52,7 +55,7 @@ impl ExternalManifestStore for LanceNamespaceExternalManifestStore {
             ..Default::default()
         };
 
-        let response = self.namespace.list_table_versions(request).await?;
+        let response = self.namespace_client.list_table_versions(request).await?;
 
         if response.versions.is_empty() {
             return Ok(None);
@@ -94,7 +97,7 @@ impl ExternalManifestStore for LanceNamespaceExternalManifestStore {
             ..Default::default()
         };
 
-        let response = self.namespace.create_table_version(request).await?;
+        let response = self.namespace_client.create_table_version(request).await?;
 
         // Get version info from response
         let version_info = response.version.ok_or_else(|| {
diff --git a/rust/lance/src/io/exec.rs b/rust/lance/src/io/exec.rs
index ae62214857f..0b93e3c2834 100644
--- a/rust/lance/src/io/exec.rs
+++ b/rust/lance/src/io/exec.rs
@@ -5,6 +5,8 @@
 //!
 //! WARNING: Internal API with no stability guarantees.
 
+#[cfg(feature = "substrait")]
+pub mod ann_proto;
 mod filter;
 pub mod filtered_read;
 #[cfg(feature = "substrait")]
@@ -17,6 +19,8 @@ mod pushdown_scan;
 mod rowids;
 pub mod scalar_index;
 mod scan;
+#[cfg(feature = "substrait")]
+pub mod table_identifier;
 mod take;
 #[cfg(test)]
 pub mod testing;
diff --git a/rust/lance/src/io/exec/ann_proto.rs b/rust/lance/src/io/exec/ann_proto.rs
new file mode 100644
index 00000000000..493fc1a6a0a
--- /dev/null
+++ b/rust/lance/src/io/exec/ann_proto.rs
@@ -0,0 +1,401 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Protobuf serialization for [`ANNIvfSubIndexExec`].
+//!
+//! Proto message definitions live in `crate::pb` (compiled from `ann.proto`).
+//! Conversion functions live here because they need access to `ANNIvfSubIndexExec`
+//! and `Dataset`, which are defined in this crate.
+//!
+//! A DataFusion `PhysicalExtensionCodec` can call these functions in `try_encode`
+//! and `try_decode` to support distributed execution (planner → executor).
+
+use std::sync::Arc;
+
+use arrow_array::RecordBatch;
+use arrow_schema::{Field, Schema as ArrowSchema};
+use lance_core::{Error, Result};
+use lance_index::pb as index_pb;
+use lance_index::vector::Query;
+use lance_linalg::distance::DistanceType;
+use lance_table::format::IndexMetadata;
+use lance_table::format::pb as table_pb;
+
+use crate::Dataset;
+use crate::pb;
+
+use super::knn::ANNIvfSubIndexExec;
+use super::table_identifier::{resolve_dataset, table_identifier_from_dataset};
+use super::utils::PreFilterSource;
+
+// =============================================================================
+// VectorQueryProto helpers
+// =============================================================================
+
+/// Serialize a query vector array to IPC file-format bytes.
+///
+/// Wraps the array in a single-column RecordBatch so that the IPC format
+/// preserves the full data type (Float16, Float32, Float64, UInt8, etc.).
+fn query_vector_to_ipc_bytes(array: &dyn arrow_array::Array) -> Result<Vec<u8>> {
+    let field = Field::new("query_vector", array.data_type().clone(), true);
+    let schema = Arc::new(ArrowSchema::new(vec![field]));
+    let batch = RecordBatch::try_new(schema, vec![arrow_array::make_array(array.to_data())])
+        .map_err(|e| {
+            Error::internal(format!(
+                "Failed to create RecordBatch for query vector: {e}"
+            ))
+        })?;
+
+    let mut buf = Vec::new();
+    {
+        let mut writer = arrow_ipc::writer::FileWriter::try_new(&mut buf, &batch.schema())
+            .map_err(|e| Error::internal(format!("Failed to create IPC writer: {e}")))?;
+        writer
+            .write(&batch)
+            .map_err(|e| Error::internal(format!("Failed to write IPC batch: {e}")))?;
+        writer
+            .finish()
+            .map_err(|e| Error::internal(format!("Failed to finish IPC writer: {e}")))?;
+    }
+    Ok(buf)
+}
+
+/// Deserialize a query vector array from IPC file-format bytes.
+fn query_vector_from_ipc_bytes(bytes: &[u8]) -> Result<arrow_array::ArrayRef> {
+    let cursor = std::io::Cursor::new(bytes);
+    let reader = arrow_ipc::reader::FileReader::try_new(cursor, None)
+        .map_err(|e| Error::internal(format!("Failed to create IPC reader: {e}")))?;
+
+    let batches: Vec<RecordBatch> = reader
+        .collect::<std::result::Result<Vec<_>, _>>()
+        .map_err(|e| Error::internal(format!("Failed to read IPC batches: {e}")))?;
+
+    if batches.is_empty() || batches[0].num_columns() == 0 {
+        return Err(Error::internal(
+            "IPC bytes contain no data for query vector".to_string(),
+        ));
+    }
+
+    Ok(batches[0].column(0).clone())
+}
+
+pub fn query_to_proto(query: &Query) -> Result<pb::VectorQueryProto> {
+    let query_vector_arrow_ipc = query_vector_to_ipc_bytes(query.key.as_ref())?;
+
+    let metric_type = query
+        .metric_type
+        .map(|dt| index_pb::VectorMetricType::from(dt) as i32);
+
+    Ok(pb::VectorQueryProto {
+        query_vector_arrow_ipc,
+        column: query.column.clone(),
+        k: query.k as u32,
+        lower_bound: query.lower_bound,
+        upper_bound: query.upper_bound,
+        minimum_nprobes: Some(query.minimum_nprobes as u32),
+        maximum_nprobes: query.maximum_nprobes.map(|n| n as u32),
+        ef: query.ef.map(|n| n as u32),
+        refine_factor: query.refine_factor,
+        metric_type,
+        use_index: query.use_index,
+        dist_q_c: Some(query.dist_q_c),
+    })
+}
+
+pub fn query_from_proto(proto: pb::VectorQueryProto) -> Result<Query> {
+    let key = query_vector_from_ipc_bytes(&proto.query_vector_arrow_ipc)?;
+
+    let metric_type = proto
+        .metric_type
+        .map(|v| {
+            index_pb::VectorMetricType::try_from(v)
+                .map(DistanceType::from)
+                .map_err(|_| Error::internal(format!("Invalid VectorMetricType value: {v}")))
+        })
+        .transpose()?;
+
+    Ok(Query {
+        column: proto.column,
+        key,
+        k: proto.k as usize,
+        lower_bound: proto.lower_bound,
+        upper_bound: proto.upper_bound,
+        minimum_nprobes: proto.minimum_nprobes.unwrap_or(1) as usize,
+        maximum_nprobes: proto.maximum_nprobes.map(|n| n as usize),
+        ef: proto.ef.map(|n| n as usize),
+        refine_factor: proto.refine_factor,
+        metric_type,
+        use_index: proto.use_index,
+        dist_q_c: proto.dist_q_c.unwrap_or(0.0),
+    })
+}
+
+// =============================================================================
+// ANNIvfSubIndexExec <-> Proto
+// =============================================================================
+
+/// Convert an [`ANNIvfSubIndexExec`] to proto for serialization.
+pub async fn ann_ivf_sub_index_exec_to_proto(
+    exec: &ANNIvfSubIndexExec,
+) -> Result<pb::AnnIvfSubIndexExecProto> {
+    let table = table_identifier_from_dataset(exec.dataset()).await?;
+    let query = query_to_proto(exec.query())?;
+
+    let indices: Vec<table_pb::IndexMetadata> =
+        exec.indices().iter().map(|idx| idx.into()).collect();
+
+    Ok(pb::AnnIvfSubIndexExecProto {
+        query: Some(query),
+        table: Some(table),
+        indices,
+    })
+}
+
+/// Reconstruct an [`ANNIvfSubIndexExec`] from proto.
+///
+/// The caller (codec) is responsible for extracting child inputs:
+/// - `input`: the child execution plan (e.g. partition exec)
+/// - `prefilter_source`: optional prefilter input
+pub async fn ann_ivf_sub_index_exec_from_proto(
+    proto: pb::AnnIvfSubIndexExecProto,
+    dataset: Option<Arc<Dataset>>,
+    input: Arc<dyn datafusion::physical_plan::ExecutionPlan>,
+    prefilter_source: PreFilterSource,
+) -> Result<ANNIvfSubIndexExec> {
+    let dataset = resolve_dataset(dataset, proto.table.as_ref()).await?;
+
+    let query_proto = proto.query.ok_or_else(|| {
+        Error::invalid_input_source("Missing VectorQueryProto in ANNIvfSubIndexExecProto".into())
+    })?;
+    let query = query_from_proto(query_proto)?;
+
+    let indices: Vec<IndexMetadata> = proto
+        .indices
+        .into_iter()
+        .map(IndexMetadata::try_from)
+        .collect::<Result<Vec<_>>>()?;
+
+    if indices.is_empty() {
+        return Err(Error::invalid_input_source(
+            "ANNIvfSubIndexExecProto contains no indices".into(),
+        ));
+    }
+
+    ANNIvfSubIndexExec::try_new(input, dataset, indices, query, prefilter_source)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow_array::types::{Float32Type, UInt32Type};
+    use arrow_array::{ArrayRef, Float32Array, Float64Array};
+    use half::f16;
+    use lance_datagen::{array, gen_batch};
+
+    use crate::index::DatasetIndexExt;
+    use crate::index::vector::VectorIndexParams;
+    use datafusion_physical_plan::test::TestMemoryExec;
+    use lance_index::IndexType;
+    use lance_index::vector::ivf::IvfBuildParams;
+    use lance_index::vector::pq::PQBuildParams;
+
+    #[test]
+    fn test_query_vector_ipc_roundtrip_f32() {
+        let arr: ArrayRef = Arc::new(Float32Array::from(vec![1.0, 2.0, 3.0]));
+        let bytes = query_vector_to_ipc_bytes(arr.as_ref()).unwrap();
+        let back = query_vector_from_ipc_bytes(&bytes).unwrap();
+        assert_eq!(arr.data_type(), back.data_type());
+        assert_eq!(arr.len(), back.len());
+    }
+
+    #[test]
+    fn test_query_vector_ipc_roundtrip_f64() {
+        let arr: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]));
+        let bytes = query_vector_to_ipc_bytes(arr.as_ref()).unwrap();
+        let back = query_vector_from_ipc_bytes(&bytes).unwrap();
+        assert_eq!(arr.data_type(), back.data_type());
+        assert_eq!(&*arr, &*back);
+    }
+
+    #[test]
+    fn test_query_vector_ipc_roundtrip_f16() {
+        let arr: ArrayRef = Arc::new(arrow_array::Float16Array::from(vec![
+            f16::from_f32(1.0),
+            f16::from_f32(2.0),
+        ]));
+        let bytes = query_vector_to_ipc_bytes(arr.as_ref()).unwrap();
+        let back = query_vector_from_ipc_bytes(&bytes).unwrap();
+        assert_eq!(arr.data_type(), back.data_type());
+        assert_eq!(arr.len(), back.len());
+    }
+
+    #[test]
+    fn test_query_roundtrip() {
+        let key: ArrayRef = Arc::new(Float32Array::from(vec![0.1, 0.2, 0.3]));
+        let query = Query {
+            column: "vector".to_string(),
+            key,
+            k: 10,
+            lower_bound: Some(0.5),
+            upper_bound: Some(1.5),
+            minimum_nprobes: 4,
+            maximum_nprobes: Some(16),
+            ef: Some(64),
+            refine_factor: Some(2),
+            metric_type: Some(DistanceType::Cosine),
+            use_index: true,
+            dist_q_c: 0.42,
+        };
+
+        let proto = query_to_proto(&query).unwrap();
+        let back = query_from_proto(proto).unwrap();
+
+        assert_eq!(query.column, back.column);
+        assert_eq!(query.k, back.k);
+        assert_eq!(query.lower_bound, back.lower_bound);
+        assert_eq!(query.upper_bound, back.upper_bound);
+        assert_eq!(query.minimum_nprobes, back.minimum_nprobes);
+        assert_eq!(query.maximum_nprobes, back.maximum_nprobes);
+        assert_eq!(query.ef, back.ef);
+        assert_eq!(query.refine_factor, back.refine_factor);
+        assert_eq!(query.metric_type, back.metric_type);
+        assert_eq!(query.use_index, back.use_index);
+        assert_eq!(query.dist_q_c, back.dist_q_c);
+        assert_eq!(query.key.len(), back.key.len());
+        assert_eq!(query.key.data_type(), back.key.data_type());
+    }
+
+    #[test]
+    fn test_query_roundtrip_none_metric() {
+        let key: ArrayRef = Arc::new(Float32Array::from(vec![1.0]));
+        let query = Query {
+            column: "v".to_string(),
+            key,
+            k: 5,
+            lower_bound: None,
+            upper_bound: None,
+            minimum_nprobes: 1,
+            maximum_nprobes: None,
+            ef: None,
+            refine_factor: None,
+            metric_type: None,
+            use_index: false,
+            dist_q_c: 0.0,
+        };
+
+        let proto = query_to_proto(&query).unwrap();
+        let back = query_from_proto(proto).unwrap();
+        assert!(back.metric_type.is_none());
+        assert!(!back.use_index);
+    }
+
+    async fn make_vector_dataset() -> (Arc<Dataset>, tempfile::TempDir) {
+        let dir = tempfile::tempdir().unwrap();
+        let batch = gen_batch()
+            .col("id", array::step::<UInt32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<Float32Type>(lance_datagen::Dimension::from(128)),
+            )
+            .into_batch_rows(lance_datagen::RowCount::from(256))
+            .unwrap();
+        let path = dir.path().join("test_ann.lance");
+        let ds = Dataset::write(
+            arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()),
+            path.to_str().unwrap(),
+            None,
+        )
+        .await
+        .unwrap();
+        (Arc::new(ds), dir)
+    }
+
+    async fn make_indexed_dataset() -> (Arc<Dataset>, tempfile::TempDir) {
+        let (_dataset, dir) = make_vector_dataset().await;
+        let mut ds = Dataset::open(dir.path().join("test_ann.lance").to_str().unwrap())
+            .await
+            .unwrap();
+
+        let ivf_params = IvfBuildParams::new(2);
+        let pq_params = PQBuildParams::default();
+        let index_params =
+            VectorIndexParams::with_ivf_pq_params(DistanceType::L2, ivf_params, pq_params);
+
+        ds.create_index(&["vector"], IndexType::Vector, None, &index_params, false)
+            .await
+            .unwrap();
+        let ds = Dataset::open(dir.path().join("test_ann.lance").to_str().unwrap())
+            .await
+            .unwrap();
+        (Arc::new(ds), dir)
+    }
+
+    #[tokio::test]
+    async fn test_ann_ivf_sub_index_proto_roundtrip() {
+        let (dataset, _dir) = make_indexed_dataset().await;
+
+        // Get real index metadata from the dataset
+        let indices = dataset.load_indices_by_name("vector_idx").await.unwrap();
+        assert!(!indices.is_empty());
+
+        let key: ArrayRef = Arc::new(Float32Array::from(vec![0.1f32; 128]));
+        let query = Query {
+            column: "vector".to_string(),
+            key,
+            k: 10,
+            lower_bound: None,
+            upper_bound: None,
+            minimum_nprobes: 2,
+            maximum_nprobes: Some(4),
+            ef: None,
+            refine_factor: Some(2),
+            metric_type: Some(DistanceType::L2),
+            use_index: true,
+            dist_q_c: 0.0,
+        };
+
+        // Use a TestMemoryExec as a mock input child (provides the KNN_PARTITION_SCHEMA)
+        let input: Arc<dyn datafusion::physical_plan::ExecutionPlan> =
+            TestMemoryExec::try_new_exec(
+                &[],
+                super::super::knn::KNN_PARTITION_SCHEMA.clone(),
+                None,
+            )
+            .unwrap();
+
+        let exec = ANNIvfSubIndexExec::try_new(
+            input.clone(),
+            dataset.clone(),
+            indices.clone(),
+            query,
+            PreFilterSource::None,
+        )
+        .unwrap();
+
+        // Encode
+        let proto = ann_ivf_sub_index_exec_to_proto(&exec).await.unwrap();
+        assert_eq!(proto.indices.len(), indices.len());
+
+        // Decode
+        let back = ann_ivf_sub_index_exec_from_proto(
+            proto,
+            Some(dataset.clone()),
+            input,
+            PreFilterSource::None,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(back.query().column, "vector");
+        assert_eq!(back.query().k, 10);
+        assert_eq!(back.query().minimum_nprobes, 2);
+        assert_eq!(back.query().refine_factor, Some(2));
+        assert_eq!(back.indices().len(), indices.len());
+        for (original, decoded) in indices.iter().zip(back.indices().iter()) {
+            assert_eq!(original.uuid, decoded.uuid);
+            assert_eq!(original.name, decoded.name);
+            assert_eq!(original.dataset_version, decoded.dataset_version);
+            assert_eq!(original.fields, decoded.fields);
+        }
+    }
+}
diff --git a/rust/lance/src/io/exec/filtered_read.rs b/rust/lance/src/io/exec/filtered_read.rs
index 7d22120c3c9..881497e9d0b 100644
--- a/rust/lance/src/io/exec/filtered_read.rs
+++ b/rust/lance/src/io/exec/filtered_read.rs
@@ -42,6 +42,7 @@ use lance_datafusion::utils::{
     ExecutionPlanMetricsSetExt, FRAGMENTS_SCANNED_METRIC, RANGES_SCANNED_METRIC,
     ROWS_SCANNED_METRIC, TASK_WAIT_TIME_METRIC,
 };
+use lance_file::reader::FileReaderOptions;
 use lance_index::scalar::expression::{FilterPlan, IndexExprResult};
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
 use lance_table::format::Fragment;
@@ -118,6 +119,7 @@ struct ScopedFragmentRead {
     projection: Arc<Projection>,
     with_deleted_rows: bool,
     batch_size: u32,
+    file_reader_options: Option<FileReaderOptions>,
     // An in-memory filter to apply after reading the fragment (whatever couldn't be
     // pushed down into the index query)
     filter: Option<Expr>,
@@ -127,13 +129,17 @@ struct ScopedFragmentRead {
 
 impl ScopedFragmentRead {
     fn frag_read_config(&self) -> FragReadConfig {
-        FragReadConfig::default()
+        let mut config = FragReadConfig::default()
             .with_row_id(self.with_deleted_rows || self.projection.with_row_id)
             .with_row_address(self.projection.with_row_addr)
             .with_row_last_updated_at_version(self.projection.with_row_last_updated_at_version)
             .with_row_created_at_version(self.projection.with_row_created_at_version)
             .with_scan_scheduler(self.scan_scheduler.clone())
-            .with_reader_priority(self.priority)
+            .with_reader_priority(self.priority);
+        if let Some(file_reader_options) = &self.file_reader_options {
+            config = config.with_file_reader_options(file_reader_options.clone());
+        }
+        config
     }
 }
 
@@ -669,6 +675,7 @@ impl FilteredReadStream {
                     projection: projection.clone(),
                     with_deleted_rows: options.with_deleted_rows,
                     batch_size: default_batch_size,
+                    file_reader_options: options.file_reader_options.clone(),
                     filter,
                     priority: priority as u32,
                     scan_scheduler: scan_scheduler.clone(),
@@ -1243,6 +1250,8 @@ pub struct FilteredReadOptions {
     pub with_deleted_rows: bool,
     /// The maximum number of rows per batch
     pub batch_size: Option<u32>,
+    /// File reader options to use when reading data files.
+    pub file_reader_options: Option<FileReaderOptions>,
     /// Controls how many fragments to read ahead
     pub fragment_readahead: Option<usize>,
     /// The fragments to read
@@ -1281,6 +1290,7 @@ impl FilteredReadOptions {
             scan_range_after_filter: None,
             with_deleted_rows: false,
             batch_size: None,
+            file_reader_options: None,
             fragment_readahead: None,
             fragments: None,
             projection,
@@ -1369,6 +1379,12 @@ impl FilteredReadOptions {
         self
     }
 
+    /// Specify the file reader options to use when reading data files.
+    pub fn with_file_reader_options(mut self, file_reader_options: FileReaderOptions) -> Self {
+        self.file_reader_options = Some(file_reader_options);
+        self
+    }
+
     /// Controls how many fragments to read ahead.
     ///
     /// If not set, the default will be 2 * the I/O parallelism.  Generally, reading ahead
@@ -1691,16 +1707,18 @@ impl FilteredReadExec {
         let running_stream_lock = self.running_stream.clone();
         let dataset = self.dataset.clone();
         let options = self.options.clone();
+        let batch_size_bytes = options
+            .file_reader_options
+            .as_ref()
+            .and_then(|o| o.batch_size_bytes);
         let metrics = self.metrics.clone();
         let index_input = self.index_input.clone();
         let plan_cell = self.plan.clone();
 
         let stream = futures::stream::once(async move {
             let mut running_stream = running_stream_lock.lock().await;
-            if let Some(running_stream) = &*running_stream {
-                DataFusionResult::<SendableRecordBatchStream>::Ok(
-                    running_stream.get_stream(&metrics, partition),
-                )
+            let inner = if let Some(running_stream) = &*running_stream {
+                running_stream.get_stream(&metrics, partition)
             } else {
                 let plan = Self::get_or_create_plan_impl(
                     &plan_cell,
@@ -1710,13 +1728,32 @@ impl FilteredReadExec {
                     partition,
                     context.clone(),
                 )
-                .await?;
+                .await
+                .map_err(|e| DataFusionError::External(e.into()))?;
                 let new_running_stream =
-                    FilteredReadStream::try_new(dataset, options, &metrics, plan.clone()).await?;
+                    FilteredReadStream::try_new(dataset, options, &metrics, plan.clone())
+                        .await
+                        .map_err(|e| DataFusionError::External(e.into()))?;
                 let first_stream = new_running_stream.get_stream(&metrics, partition);
                 *running_stream = Some(new_running_stream);
-                DataFusionResult::Ok(first_stream)
-            }
+                first_stream
+            };
+            let stream: SendableRecordBatchStream = match batch_size_bytes {
+                Some(target) => {
+                    let schema = inner.schema();
+                    Box::pin(RecordBatchStreamAdapter::new(
+                        schema.clone(),
+                        lance_arrow::stream::rechunk_stream_by_size(
+                            inner,
+                            schema,
+                            0,
+                            target as usize,
+                        ),
+                    ))
+                }
+                None => inner,
+            };
+            DataFusionResult::<SendableRecordBatchStream>::Ok(stream)
         })
         .try_flatten();
 
@@ -1854,7 +1891,7 @@ impl ExecutionPlan for FilteredReadExec {
 
         let total_rows: u64 = fragments.iter().map(|f| f.num_rows().unwrap() as u64).sum();
 
-        if self.options.full_filter.is_none() {
+        let Some(filter) = self.options.full_filter.as_ref() else {
             // If there is no filter, we just return the total number of rows (sans any before-filter range)
             // divided by the number of partitions.
             let total_rows =
@@ -1876,83 +1913,81 @@ impl ExecutionPlan for FilteredReadExec {
                 total_rows
             };
 
-            Ok(Statistics {
+            return Ok(Statistics {
                 num_rows: Precision::Exact(total_rows as usize),
                 ..datafusion::physical_plan::Statistics::new_unknown(self.schema().as_ref())
-            })
-        } else {
-            // We could evaluate the indexed filter here but this is still during the planning
-            // phase so we want to avoid that.
-            //
-            // Instead, we create a mock input which is the filtered read (without the filter)
-            // and then use DF's FilterExec logic to calculate the statistics (which uses column
-            // stats and basic filter shape)
-            let filter = self.options.full_filter.as_ref().unwrap();
-
-            // Need to add in filter columns even though they aren't part of the projection
-            let filter_columns = Planner::column_names_in_expr(filter);
-            let read_projection = self
-                .options
-                .projection
-                .clone()
-                .union_columns(filter_columns, OnMissing::Error)?;
-
-            let read_schema = Arc::new(read_projection.to_arrow_schema());
-
-            let planner = Arc::new(Planner::new(read_schema.clone()));
-            let physical_filter = planner.create_physical_expr(filter)?;
-
-            let mock_input = Arc::new(Self::try_new(
-                self.dataset.clone(),
-                FilteredReadOptions {
-                    scan_range_after_filter: None,
-                    refine_filter: None,
-                    full_filter: None,
-                    projection: read_projection,
-                    ..self.options.clone()
-                },
-                None,
-            )?);
-            let df_filter_exec = FilterExec::try_new(physical_filter, mock_input)?;
-            let mut df_stats = df_filter_exec.partition_statistics(partition)?;
-
-            // If we have an after-filter range, we should apply it to the stats (the before-filter range
-            // is applied in the mock input)
-            let total_rows = if let Some(scan_range_after_filter) =
-                &self.options.scan_range_after_filter
-            {
+            });
+        };
+
+        // We could evaluate the indexed filter here but this is still during the planning
+        // phase so we want to avoid that.
+        //
+        // Instead, we create a mock input which is the filtered read (without the filter)
+        // and then use DF's FilterExec logic to calculate the statistics (which uses column
+        // stats and basic filter shape)
+
+        // Need to add in filter columns even though they aren't part of the projection
+        let filter_columns = Planner::column_names_in_expr(filter);
+        let read_projection = self
+            .options
+            .projection
+            .clone()
+            .union_columns(filter_columns, OnMissing::Error)?;
+
+        let read_schema = Arc::new(read_projection.to_arrow_schema());
+
+        let planner = Arc::new(Planner::new(read_schema.clone()));
+        let physical_filter = planner.create_physical_expr(filter)?;
+
+        let mock_input = Arc::new(Self::try_new(
+            self.dataset.clone(),
+            FilteredReadOptions {
+                scan_range_after_filter: None,
+                refine_filter: None,
+                full_filter: None,
+                projection: read_projection,
+                ..self.options.clone()
+            },
+            None,
+        )?);
+        let df_filter_exec = FilterExec::try_new(physical_filter, mock_input)?;
+        let mut df_stats = df_filter_exec.partition_statistics(partition)?;
+
+        // If we have an after-filter range, we should apply it to the stats (the before-filter range
+        // is applied in the mock input)
+        let total_rows =
+            if let Some(scan_range_after_filter) = &self.options.scan_range_after_filter {
                 df_stats.num_rows.min(&Precision::Exact(
                     scan_range_after_filter.end as usize - scan_range_after_filter.start as usize,
                 ))
             } else {
                 df_stats.num_rows
             };
-            df_stats.num_rows = total_rows;
-
-            let schema = self.schema();
-
-            // We might have added some columns to the schema so the filter compiles but we drop this
-            // columns during the filtered read and they aren't part of the output.  So we need to make
-            // sure and drop them from the column stats as well.
-            assert_eq!(read_schema.fields.len(), df_stats.column_statistics.len());
-            let mut proj_iter = schema.fields.iter().peekable();
-            let mut stats_iter = read_schema.fields.iter();
-            df_stats.column_statistics.retain(|_| {
-                let stats_field = stats_iter.next().unwrap();
-                if let Some(proj_field) = proj_iter.peek() {
-                    if proj_field.name() == stats_field.name() {
-                        proj_iter.next();
-                        true
-                    } else {
-                        false
-                    }
+        df_stats.num_rows = total_rows;
+
+        let schema = self.schema();
+
+        // We might have added some columns to the schema so the filter compiles but we drop this
+        // columns during the filtered read and they aren't part of the output.  So we need to make
+        // sure and drop them from the column stats as well.
+        assert_eq!(read_schema.fields.len(), df_stats.column_statistics.len());
+        let mut proj_iter = schema.fields.iter().peekable();
+        let mut stats_iter = read_schema.fields.iter();
+        df_stats.column_statistics.retain(|_| {
+            let stats_field = stats_iter.next().unwrap();
+            if let Some(proj_field) = proj_iter.peek() {
+                if proj_field.name() == stats_field.name() {
+                    proj_iter.next();
+                    true
                 } else {
                     false
                 }
-            });
+            } else {
+                false
+            }
+        });
 
-            Ok(df_stats)
-        }
+        Ok(df_stats)
     }
 
     fn with_new_children(
@@ -2054,6 +2089,7 @@ impl ExecutionPlan for FilteredReadExec {
 mod tests {
     use std::collections::HashSet;
 
+    use crate::index::DatasetIndexExt;
     use arrow::{
         compute::concat_batches,
         datatypes::{Float32Type, UInt32Type, UInt64Type},
@@ -2066,7 +2102,7 @@ mod tests {
     use lance_core::utils::tempfile::TempStrDir;
     use lance_datagen::{BatchCount, Dimension, RowCount, array, gen_batch};
     use lance_index::{
-        DatasetIndexExt, IndexType,
+        IndexType,
         optimize::OptimizeOptions,
         scalar::{ScalarIndexParams, expression::PlannerIndexExt},
     };
diff --git a/rust/lance/src/io/exec/filtered_read_proto.rs b/rust/lance/src/io/exec/filtered_read_proto.rs
index 26225d748ea..c373a691f04 100644
--- a/rust/lance/src/io/exec/filtered_read_proto.rs
+++ b/rust/lance/src/io/exec/filtered_read_proto.rs
@@ -24,73 +24,14 @@ use lance_core::utils::mask::RowAddrTreeMap;
 use lance_core::{Error, Result};
 use lance_datafusion::pb;
 use lance_datafusion::substrait::{encode_substrait, parse_substrait, prune_schema_for_substrait};
-use lance_io::object_store::StorageOptions;
 use lance_table::format::Fragment;
-use prost::Message;
 
 use crate::Dataset;
-use crate::dataset::builder::DatasetBuilder;
 
 use super::filtered_read::{
     FilteredReadExec, FilteredReadOptions, FilteredReadPlan, FilteredReadThreadingMode,
 };
-
-// =============================================================================
-// TableIdentifier helpers (reusable by other execs)
-// =============================================================================
-
-/// Build a [`TableIdentifier`] from a [`Dataset`].
-///
-/// Default: lightweight mode (uri + version + etag only, no serialized manifest).
-/// Includes the dataset's latest storage options (if any) so the remote executor
-/// can open or cache the dataset with the correct storage configuration.
-pub async fn table_identifier_from_dataset(dataset: &Dataset) -> Result<pb::TableIdentifier> {
-    Ok(pb::TableIdentifier {
-        uri: dataset.uri().to_string(),
-        version: dataset.manifest.version,
-        manifest_etag: dataset.manifest_location.e_tag.clone(),
-        serialized_manifest: None,
-        storage_options: dataset
-            .latest_storage_options()
-            .await?
-            .map(|StorageOptions(m)| m)
-            .unwrap_or_default(),
-    })
-}
-
-/// Build a [`TableIdentifier`] with serialized manifest bytes included.
-///
-/// Fast path: remote executor skips manifest read from storage.
-pub async fn table_identifier_from_dataset_with_manifest(
-    dataset: &Dataset,
-) -> Result<pb::TableIdentifier> {
-    let manifest_proto = lance_table::format::pb::Manifest::from(dataset.manifest.as_ref());
-    Ok(pb::TableIdentifier {
-        uri: dataset.uri().to_string(),
-        version: dataset.manifest.version,
-        manifest_etag: dataset.manifest_location.e_tag.clone(),
-        serialized_manifest: Some(manifest_proto.encode_to_vec()),
-        storage_options: dataset
-            .latest_storage_options()
-            .await?
-            .map(|StorageOptions(m)| m)
-            .unwrap_or_default(),
-    })
-}
-
-/// Open a dataset from a table identifier proto
-pub async fn open_dataset_from_table_identifier(
-    table_id: &pb::TableIdentifier,
-) -> Result<Arc<Dataset>> {
-    let mut builder = DatasetBuilder::from_uri(&table_id.uri).with_version(table_id.version);
-    if let Some(manifest_bytes) = &table_id.serialized_manifest {
-        builder = builder.with_serialized_manifest(manifest_bytes)?;
-    }
-    if !table_id.storage_options.is_empty() {
-        builder = builder.with_storage_options(table_id.storage_options.clone());
-    }
-    Ok(Arc::new(builder.load().await?))
-}
+use super::table_identifier::{resolve_dataset, table_identifier_from_dataset};
 
 // =============================================================================
 // FilteredReadExec <-> Proto
@@ -131,17 +72,7 @@ pub async fn filtered_read_exec_from_proto(
     index_input: Option<Arc<dyn ExecutionPlan>>,
     state: &SessionState,
 ) -> Result<FilteredReadExec> {
-    let dataset = match dataset {
-        Some(ds) => ds, // dataset could be opened or cached by the caller
-        None => {
-            let table_id = proto.table.as_ref().ok_or_else(|| {
-                Error::invalid_input_source(
-                    "Missing table identifier in FilteredReadExecProto".into(),
-                )
-            })?;
-            open_dataset_from_table_identifier(table_id).await?
-        }
-    };
+    let dataset = resolve_dataset(dataset, proto.table.as_ref()).await?;
 
     let options_proto = proto.options.ok_or_else(|| {
         Error::invalid_input_source("Missing options in FilteredReadExecProto".into())
@@ -559,6 +490,7 @@ mod tests {
     use lance_core::utils::mask::RowAddrTreeMap;
     use lance_datagen::{array, gen_batch};
     use roaring::RoaringBitmap;
+    use std::collections::HashMap;
     use std::collections::HashSet;
 
     use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount};
@@ -632,23 +564,6 @@ mod tests {
         assert_eq!(projection.blob_handling, back.blob_handling);
     }
 
-    #[test]
-    fn test_table_identifier_without_manifest() {
-        let id = pb::TableIdentifier {
-            uri: "s3://bucket/table.lance".to_string(),
-            version: 42,
-            manifest_etag: Some("etag123".to_string()),
-            serialized_manifest: None,
-            storage_options: HashMap::new(),
-        };
-        let bytes = id.encode_to_vec();
-        let back = pb::TableIdentifier::decode(bytes.as_slice()).unwrap();
-        assert_eq!(id.uri, back.uri);
-        assert_eq!(id.version, back.version);
-        assert_eq!(id.manifest_etag, back.manifest_etag);
-        assert!(back.serialized_manifest.is_none());
-    }
-
     #[test]
     fn test_row_addr_tree_map_roundtrip_in_plan_proto() {
         let mut rows = RowAddrTreeMap::new();
@@ -812,23 +727,6 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    async fn test_table_identifier_with_manifest() {
-        let dataset = make_test_dataset().await;
-
-        let id = table_identifier_from_dataset_with_manifest(&dataset)
-            .await
-            .unwrap();
-        assert_eq!(id.uri, dataset.uri());
-        assert_eq!(id.version, dataset.manifest.version);
-        assert!(id.serialized_manifest.is_some());
-
-        // Verify the serialized manifest bytes decode
-        let manifest_bytes = id.serialized_manifest.unwrap();
-        let _manifest_proto =
-            lance_table::format::pb::Manifest::decode(manifest_bytes.as_slice()).unwrap();
-    }
-
     #[tokio::test]
     async fn test_plan_proto_roundtrip() {
         let dataset = make_test_dataset().await;
diff --git a/rust/lance/src/io/exec/fts.rs b/rust/lance/src/io/exec/fts.rs
index 91d46b9d495..1cf8eddc3ec 100644
--- a/rust/lance/src/io/exec/fts.rs
+++ b/rust/lance/src/io/exec/fts.rs
@@ -25,20 +25,22 @@ use lance_datafusion::utils::{ExecutionPlanMetricsSetExt, MetricsExt, PARTITIONS
 
 use super::PreFilterSource;
 use super::utils::{IndexMetrics, InstrumentedRecordBatchStreamAdapter, build_prefilter};
+use crate::index::DatasetIndexExt;
 use crate::{Dataset, index::DatasetIndexInternalExt};
+use lance_index::IndexCriteria;
 use lance_index::metrics::MetricsCollector;
 use lance_index::scalar::inverted::builder::document_input;
-use lance_index::scalar::inverted::lance_tokenizer::{DocType, JsonTokenizer, LanceTokenizer};
+use lance_index::scalar::inverted::document_tokenizer::{DocType, JsonTokenizer, LanceTokenizer};
 use lance_index::scalar::inverted::query::{
     BoostQuery, FtsSearchParams, MatchQuery, PhraseQuery, Tokens, collect_query_tokens,
     has_query_token,
 };
-use lance_index::scalar::inverted::tokenizer::lance_tokenizer::TextTokenizer;
+use lance_index::scalar::inverted::tokenizer::document_tokenizer::TextTokenizer;
 use lance_index::scalar::inverted::{
     FTS_SCHEMA, InvertedIndex, SCORE_COL, flat_bm25_search_stream,
 };
-use lance_index::{DatasetIndexExt, IndexCriteria};
 use lance_index::{prefilter::PreFilter, scalar::inverted::query::BooleanQuery};
+use lance_tokenizer::{SimpleTokenizer, TextAnalyzer};
 use tracing::instrument;
 
 pub struct FtsIndexMetrics {
@@ -242,7 +244,7 @@ impl ExecutionPlan for MatchQueryExec {
                 .open_generic_index(&column, &uuid, &metrics.index_metrics)
                 .await?;
 
-            let pre_filter = build_prefilter(
+            let mut pre_filter = build_prefilter(
                 context.clone(),
                 partition,
                 &prefilter_source,
@@ -259,6 +261,11 @@ impl ExecutionPlan for MatchQueryExec {
                         column,
                     ))
                 })?;
+            if !inverted_idx.deleted_fragments().is_empty() {
+                Arc::get_mut(&mut pre_filter)
+                    .expect("prefilter just created")
+                    .set_deleted_fragments(inverted_idx.deleted_fragments().clone());
+            }
             metrics.record_parts_searched(inverted_idx.partition_count());
 
             let is_fuzzy = matches!(query.fuzziness, Some(n) if n != 0);
@@ -269,9 +276,7 @@ impl ExecutionPlan for MatchQueryExec {
             let mut tokenizer = match is_fuzzy {
                 false => inverted_idx.tokenizer(),
                 true => {
-                    let tokenizer = tantivy::tokenizer::TextAnalyzer::from(
-                        tantivy::tokenizer::SimpleTokenizer::default(),
-                    );
+                    let tokenizer = TextAnalyzer::from(SimpleTokenizer::default());
                     match inverted_idx.tokenizer().doc_type() {
                         DocType::Text => {
                             Box::new(TextTokenizer::new(tokenizer)) as Box<dyn LanceTokenizer>
@@ -391,10 +396,7 @@ impl FlatMatchFilterExec {
         } // Else, no index, use text tokenzier
 
         Ok(Box::new(TextTokenizer::new(
-            tantivy::tokenizer::TextAnalyzer::builder(
-                tantivy::tokenizer::SimpleTokenizer::default(),
-            )
-            .build(),
+            TextAnalyzer::builder(SimpleTokenizer::default()).build(),
         )))
     }
 
@@ -442,7 +444,7 @@ impl FlatMatchFilterExec {
             )))?;
         let mut tokenizer = Self::load_tokenizer(&dataset, column, &metrics.index_metrics).await?;
         let query_tokens = Arc::new(collect_query_tokens(&query.terms, &mut tokenizer));
-        let column = column.to_string();
+        let column = column.clone();
 
         Ok(input.map(move |batch| -> DataFusionResult<_> {
             let batch = batch?;
@@ -872,10 +874,13 @@ impl ExecutionPlan for PhraseQueryExec {
         let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition));
         let stream = stream::once(async move {
             let _timer = metrics.baseline_metrics.elapsed_compute().timer();
-            let column = query.column.ok_or(DataFusionError::Execution(format!(
-                "column not set for PhraseQuery {}",
-                query.terms
-            )))?;
+            let column = query
+                .column
+                .clone()
+                .ok_or(DataFusionError::Execution(format!(
+                    "column not set for PhraseQuery {}",
+                    query.terms
+                )))?;
             let index_meta = ds
                 .load_scalar_index(IndexCriteria::default().for_column(&column).supports_fts())
                 .await?
@@ -888,11 +893,11 @@ impl ExecutionPlan for PhraseQueryExec {
                 .open_generic_index(&column, &uuid, &metrics.index_metrics)
                 .await?;
 
-            let pre_filter = build_prefilter(
+            let mut pre_filter = build_prefilter(
                 context.clone(),
                 partition,
                 &prefilter_source,
-                ds,
+                ds.clone(),
                 &[index_meta],
             )?;
 
@@ -905,6 +910,11 @@ impl ExecutionPlan for PhraseQueryExec {
                         column,
                     ))
                 })?;
+            if !index.deleted_fragments().is_empty() {
+                Arc::get_mut(&mut pre_filter)
+                    .expect("prefilter just created")
+                    .set_deleted_fragments(index.deleted_fragments().clone());
+            }
             metrics.record_parts_searched(index.partition_count());
 
             let mut tokenizer = index.tokenizer();
@@ -1381,9 +1391,10 @@ impl ExecutionPlan for BooleanQueryExec {
 }
 
 #[cfg(test)]
-pub mod tests {
+mod tests {
     use std::sync::{Arc, Mutex};
 
+    use crate::index::DatasetIndexExt;
     use datafusion::{execution::TaskContext, physical_plan::ExecutionPlan};
     use lance_datafusion::datagen::DatafusionDatagenExt;
     use lance_datafusion::exec::{ExecutionStatsCallback, ExecutionSummaryCounts};
@@ -1396,7 +1407,7 @@ pub mod tests {
         PhraseQuery,
     };
     use lance_index::scalar::{FullTextSearchQuery, InvertedIndexParams};
-    use lance_index::{DatasetIndexExt, IndexCriteria, IndexType};
+    use lance_index::{IndexCriteria, IndexType};
 
     use crate::{
         index::DatasetIndexInternalExt,
diff --git a/rust/lance/src/io/exec/knn.rs b/rust/lance/src/io/exec/knn.rs
index ab7589ce1a8..daec6140b0b 100644
--- a/rust/lance/src/io/exec/knn.rs
+++ b/rust/lance/src/io/exec/knn.rs
@@ -32,15 +32,15 @@ use datafusion::{
     physical_plan::metrics::MetricsSet,
 };
 use datafusion_physical_expr::{Distribution, EquivalenceProperties};
-use datafusion_physical_plan::metrics::{BaselineMetrics, Count};
+use datafusion_physical_plan::metrics::{BaselineMetrics, Count, Time};
 use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt, future, stream};
 use itertools::Itertools;
 use lance_core::ROW_ID;
 use lance_core::utils::futures::FinallyStreamExt;
 use lance_core::{ROW_ID_FIELD, utils::tokio::get_num_compute_intensive_cpus};
 use lance_datafusion::utils::{
-    DELTAS_SEARCHED_METRIC, ExecutionPlanMetricsSetExt, PARTITIONS_RANKED_METRIC,
-    PARTITIONS_SEARCHED_METRIC,
+    DELTAS_SEARCHED_METRIC, ExecutionPlanMetricsSetExt, FIND_PARTITIONS_ELAPSED_METRIC,
+    PARTITIONS_RANKED_METRIC, PARTITIONS_SEARCHED_METRIC,
 };
 use lance_index::prefilter::PreFilter;
 use lance_index::vector::{
@@ -68,6 +68,7 @@ pub struct AnnPartitionMetrics {
     index_metrics: IndexMetrics,
     partitions_ranked: Count,
     deltas_searched: Count,
+    find_partitions_elapsed: Time,
     baseline_metrics: BaselineMetrics,
 }
 
@@ -77,6 +78,7 @@ impl AnnPartitionMetrics {
             index_metrics: IndexMetrics::new(metrics, partition),
             partitions_ranked: metrics.new_count(PARTITIONS_RANKED_METRIC, partition),
             deltas_searched: metrics.new_count(DELTAS_SEARCHED_METRIC, partition),
+            find_partitions_elapsed: metrics.new_time(FIND_PARTITIONS_ELAPSED_METRIC, partition),
             baseline_metrics: BaselineMetrics::new(metrics, partition),
         }
     }
@@ -509,9 +511,12 @@ impl ExecutionPlan for ANNIvfPartitionExec {
 
                     metrics.partitions_ranked.add(index.total_partitions());
 
-                    let (partitions, dist_q_c) = index.find_partitions(&query).map_err(|e| {
-                        DataFusionError::Execution(format!("Failed to find partitions: {}", e))
-                    })?;
+                    let (partitions, dist_q_c) = {
+                        let _timer = metrics.find_partitions_elapsed.timer();
+                        index.find_partitions(&query).map_err(|e| {
+                            DataFusionError::Execution(format!("Failed to find partitions: {}", e))
+                        })?
+                    };
 
                     let mut part_list_builder = ListBuilder::new(UInt32Builder::new())
                         .with_field(Field::new("item", DataType::UInt32, false));
@@ -634,6 +639,26 @@ impl ANNIvfSubIndexExec {
             metrics: ExecutionPlanMetricsSet::new(),
         })
     }
+
+    /// Returns a reference to the vector query.
+    pub fn query(&self) -> &Query {
+        &self.query
+    }
+
+    /// Returns a reference to the dataset.
+    pub fn dataset(&self) -> &Arc<Dataset> {
+        &self.dataset
+    }
+
+    /// Returns a reference to the index metadata.
+    pub fn indices(&self) -> &[IndexMetadata] {
+        &self.indices
+    }
+
+    /// Returns a reference to the prefilter source.
+    pub fn prefilter_source(&self) -> &PreFilterSource {
+        &self.prefilter_source
+    }
 }
 
 impl DisplayAs for ANNIvfSubIndexExec {
@@ -1344,6 +1369,7 @@ impl ExecutionPlan for MultivectorScoringExec {
 mod tests {
     use super::*;
 
+    use crate::index::DatasetIndexExt;
     use arrow::compute::{concat_batches, sort_to_indices, take_record_batch};
     use arrow::datatypes::Float32Type;
     use arrow_array::{
@@ -1352,11 +1378,12 @@ mod tests {
     use arrow_schema::{Field as ArrowField, Schema as ArrowSchema};
     use lance_core::utils::tempfile::TempStrDir;
     use lance_datafusion::exec::{ExecutionStatsCallback, ExecutionSummaryCounts};
+    use lance_datafusion::utils::FIND_PARTITIONS_ELAPSED_METRIC;
     use lance_datagen::{BatchCount, RowCount, array};
+    use lance_index::IndexType;
     use lance_index::optimize::OptimizeOptions;
     use lance_index::vector::ivf::IvfBuildParams;
     use lance_index::vector::pq::PQBuildParams;
-    use lance_index::{DatasetIndexExt, IndexType};
     use lance_linalg::distance::MetricType;
     use lance_testing::datagen::generate_random_array;
     use rstest::rstest;
@@ -1731,6 +1758,17 @@ mod tests {
         }
     }
 
+    fn assert_find_partitions_elapsed_recorded(stats: &ExecutionSummaryCounts) {
+        assert!(
+            stats
+                .all_times
+                .get(FIND_PARTITIONS_ELAPSED_METRIC)
+                .copied()
+                .unwrap_or_default()
+                > 0
+        );
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_no_max_nprobes(#[values(1, 20)] num_deltas: usize) {
@@ -1766,6 +1804,7 @@ mod tests {
         if get_num_compute_intensive_cpus() <= 32 {
             assert!(*stats.all_counts.get(PARTITIONS_SEARCHED_METRIC).unwrap() < 100 * num_deltas);
         }
+        assert_find_partitions_elapsed_recorded(&stats);
     }
 
     #[rstest]
@@ -1802,6 +1841,7 @@ mod tests {
             stats.all_counts.get(PARTITIONS_SEARCHED_METRIC).unwrap(),
             &(10 * num_deltas)
         );
+        assert_find_partitions_elapsed_recorded(&stats);
     }
 
     #[rstest]
@@ -1841,6 +1881,7 @@ mod tests {
                 stats.all_counts.get(PARTITIONS_RANKED_METRIC).unwrap(),
                 &(100 * num_deltas)
             );
+            assert_find_partitions_elapsed_recorded(&stats);
         }
     }
 
@@ -1876,6 +1917,7 @@ mod tests {
             stats.all_counts.get(PARTITIONS_SEARCHED_METRIC).unwrap(),
             &(10 * num_deltas)
         );
+        assert_find_partitions_elapsed_recorded(&stats);
         assert_eq!(results.num_rows(), 20);
 
         // 15 of the results come from beyond the closest 10 partitions and these will have infinite
@@ -1947,6 +1989,7 @@ mod tests {
             stats.all_counts.get(PARTITIONS_SEARCHED_METRIC).unwrap(),
             &(100 * num_deltas)
         );
+        assert_find_partitions_elapsed_recorded(&stats);
         assert_eq!(results.num_rows(), 10000);
     }
 }
diff --git a/rust/lance/src/io/exec/scalar_index.rs b/rust/lance/src/io/exec/scalar_index.rs
index aafd3c8d2eb..f587ec22a91 100644
--- a/rust/lance/src/io/exec/scalar_index.rs
+++ b/rust/lance/src/io/exec/scalar_index.rs
@@ -7,7 +7,7 @@ use super::utils::{IndexMetrics, InstrumentedRecordBatchStreamAdapter};
 use crate::{
     Dataset,
     dataset::rowids::load_row_id_sequences,
-    index::{DatasetIndexInternalExt, prefilter::DatasetPreFilter},
+    index::{DatasetIndexExt, DatasetIndexInternalExt, prefilter::DatasetPreFilter},
 };
 use arrow_array::{Array, RecordBatch, UInt64Array};
 use arrow_schema::{Schema, SchemaRef};
@@ -40,7 +40,7 @@ use lance_datafusion::{
     },
 };
 use lance_index::{
-    DatasetIndexExt, IndexCriteria,
+    IndexCriteria,
     metrics::MetricsCollector,
     scalar::{
         SargableQuery, ScalarIndex,
@@ -733,6 +733,7 @@ impl ExecutionPlan for MaterializeIndexExec {
 mod tests {
     use std::{ops::Bound, sync::Arc};
 
+    use crate::index::DatasetIndexExt;
     use arrow::datatypes::UInt64Type;
     use datafusion::{
         execution::TaskContext, physical_plan::ExecutionPlan, prelude::SessionConfig,
@@ -742,7 +743,7 @@ mod tests {
     use lance_core::utils::tempfile::TempStrDir;
     use lance_datagen::gen_batch;
     use lance_index::{
-        DatasetIndexExt, IndexType,
+        IndexType,
         scalar::{
             SargableQuery, ScalarIndexParams,
             expression::{ScalarIndexExpr, ScalarIndexSearch},
diff --git a/rust/lance/src/io/exec/scan.rs b/rust/lance/src/io/exec/scan.rs
index 030016c9d78..c7736eccfbc 100644
--- a/rust/lance/src/io/exec/scan.rs
+++ b/rust/lance/src/io/exec/scan.rs
@@ -27,6 +27,7 @@ use lance_arrow::SchemaExt;
 use lance_core::utils::tokio::get_num_compute_intensive_cpus;
 use lance_core::utils::tracing::StreamTracingExt;
 use lance_core::{Error, ROW_ADDR_FIELD, ROW_ID_FIELD};
+use lance_file::reader::FileReaderOptions;
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
 use lance_table::format::Fragment;
 use log::debug;
@@ -274,25 +275,31 @@ impl LanceStream {
 
         let scan_scheduler_clone = scan_scheduler.clone();
 
+        let config_for_stream = config.clone();
         let batches = stream::iter(file_fragments.into_iter().enumerate())
             .map(move |(priority, file_fragment)| {
                 let project_schema = project_schema.clone();
                 let scan_scheduler = scan_scheduler.clone();
+                let config = config_for_stream.clone();
                 #[allow(clippy::type_complexity)]
                 let frag_task: BoxFuture<
                     Result<BoxStream<Result<BoxFuture<Result<RecordBatch>>>>>,
                 > = tokio::spawn(
                     (async move {
+                        let mut frag_config = FragReadConfig::default()
+                            .with_row_id(config.with_row_id)
+                            .with_row_address(config.with_row_address)
+                            .with_row_last_updated_at_version(
+                                config.with_row_last_updated_at_version,
+                            )
+                            .with_row_created_at_version(config.with_row_created_at_version);
+                        if let Some(file_reader_options) = config.file_reader_options {
+                            frag_config = frag_config.with_file_reader_options(file_reader_options);
+                        }
                         let reader = open_file(
                             file_fragment.fragment,
                             project_schema,
-                            FragReadConfig::default()
-                                .with_row_id(config.with_row_id)
-                                .with_row_address(config.with_row_address)
-                                .with_row_last_updated_at_version(
-                                    config.with_row_last_updated_at_version,
-                                )
-                                .with_row_created_at_version(config.with_row_created_at_version),
+                            frag_config,
                             config.with_make_deletions_null,
                             Some((scan_scheduler, priority as u32)),
                         )
@@ -499,6 +506,7 @@ pub struct LanceScanConfig {
     pub with_row_created_at_version: bool,
     pub with_make_deletions_null: bool,
     pub ordered_output: bool,
+    pub file_reader_options: Option<FileReaderOptions>,
 }
 
 // This is mostly for testing purposes, end users are unlikely to create this
@@ -516,6 +524,7 @@ impl Default for LanceScanConfig {
             with_row_created_at_version: false,
             with_make_deletions_null: false,
             ordered_output: false,
+            file_reader_options: None,
         }
     }
 }
diff --git a/rust/lance/src/io/exec/table_identifier.rs b/rust/lance/src/io/exec/table_identifier.rs
new file mode 100644
index 00000000000..9ffd1561bc9
--- /dev/null
+++ b/rust/lance/src/io/exec/table_identifier.rs
@@ -0,0 +1,186 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Helpers for converting between [`Dataset`] and [`TableIdentifier`](pb::TableIdentifier) proto.
+
+use std::sync::Arc;
+
+use lance_core::Result;
+use lance_datafusion::pb;
+use lance_io::object_store::StorageOptions;
+use prost::Message;
+
+use crate::Dataset;
+use crate::dataset::builder::DatasetBuilder;
+
+/// Build a [`TableIdentifier`] from a [`Dataset`].
+///
+/// Default: lightweight mode (uri + version + etag only, no serialized manifest).
+/// Includes the dataset's latest storage options (if any) so the remote executor
+/// can open or cache the dataset with the correct storage configuration.
+pub async fn table_identifier_from_dataset(dataset: &Dataset) -> Result<pb::TableIdentifier> {
+    Ok(pb::TableIdentifier {
+        uri: dataset.uri().to_string(),
+        version: dataset.manifest.version,
+        manifest_etag: dataset.manifest_location.e_tag.clone(),
+        serialized_manifest: None,
+        storage_options: dataset
+            .latest_storage_options()
+            .await?
+            .map(|StorageOptions(m)| m)
+            .unwrap_or_default(),
+    })
+}
+
+/// Build a [`TableIdentifier`] with serialized manifest bytes included.
+///
+/// Fast path: remote executor skips manifest read from storage.
+pub async fn table_identifier_from_dataset_with_manifest(
+    dataset: &Dataset,
+) -> Result<pb::TableIdentifier> {
+    let manifest_proto = lance_table::format::pb::Manifest::from(dataset.manifest.as_ref());
+    Ok(pb::TableIdentifier {
+        uri: dataset.uri().to_string(),
+        version: dataset.manifest.version,
+        manifest_etag: dataset.manifest_location.e_tag.clone(),
+        serialized_manifest: Some(manifest_proto.encode_to_vec()),
+        storage_options: dataset
+            .latest_storage_options()
+            .await?
+            .map(|StorageOptions(m)| m)
+            .unwrap_or_default(),
+    })
+}
+
+/// Open a dataset from a table identifier proto.
+pub async fn open_dataset_from_table_identifier(
+    table_id: &pb::TableIdentifier,
+) -> Result<Arc<Dataset>> {
+    let mut builder = DatasetBuilder::from_uri(&table_id.uri).with_version(table_id.version);
+    if let Some(manifest_bytes) = &table_id.serialized_manifest {
+        builder = builder.with_serialized_manifest(manifest_bytes)?;
+    }
+    if !table_id.storage_options.is_empty() {
+        builder = builder.with_storage_options(table_id.storage_options.clone());
+    }
+    Ok(Arc::new(builder.load().await?))
+}
+
+/// Resolve a dataset from an optional pre-loaded instance or from a table identifier.
+///
+/// If `dataset` is `Some`, returns it directly. Otherwise, opens the dataset
+/// from the table identifier proto.
+pub async fn resolve_dataset(
+    dataset: Option<Arc<Dataset>>,
+    table_id: Option<&pb::TableIdentifier>,
+) -> Result<Arc<Dataset>> {
+    use lance_core::Error;
+    match dataset {
+        Some(ds) => Ok(ds),
+        None => {
+            let table_id = table_id.ok_or_else(|| {
+                Error::invalid_input_source("Missing TableIdentifier in proto".into())
+            })?;
+            open_dataset_from_table_identifier(table_id).await
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow_array::RecordBatchIterator;
+    use arrow_array::types::UInt32Type;
+    use lance_datagen::{array, gen_batch};
+    use std::collections::HashMap;
+
+    async fn make_test_dataset() -> (Arc<Dataset>, tempfile::TempDir) {
+        let dir = tempfile::tempdir().unwrap();
+        let batch = gen_batch()
+            .col("x", array::step::<UInt32Type>())
+            .col("y", array::step::<UInt32Type>())
+            .into_batch_rows(lance_datagen::RowCount::from(100))
+            .unwrap();
+        let path = dir.path().join("test.lance");
+        let ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()),
+            path.to_str().unwrap(),
+            None,
+        )
+        .await
+        .unwrap();
+        (Arc::new(ds), dir)
+    }
+
+    #[test]
+    fn test_table_identifier_proto_roundtrip() {
+        let id = pb::TableIdentifier {
+            uri: "s3://bucket/table.lance".to_string(),
+            version: 42,
+            manifest_etag: Some("etag123".to_string()),
+            serialized_manifest: None,
+            storage_options: HashMap::new(),
+        };
+        let bytes = id.encode_to_vec();
+        let back = pb::TableIdentifier::decode(bytes.as_slice()).unwrap();
+        assert_eq!(id.uri, back.uri);
+        assert_eq!(id.version, back.version);
+        assert_eq!(id.manifest_etag, back.manifest_etag);
+        assert!(back.serialized_manifest.is_none());
+    }
+
+    #[test]
+    fn test_table_identifier_proto_with_storage_options() {
+        let mut opts = HashMap::new();
+        opts.insert("region".to_string(), "us-east-1".to_string());
+        opts.insert("endpoint".to_string(), "https://s3.example.com".to_string());
+
+        let id = pb::TableIdentifier {
+            uri: "s3://bucket/table.lance".to_string(),
+            version: 7,
+            manifest_etag: None,
+            serialized_manifest: None,
+            storage_options: opts.clone(),
+        };
+        let bytes = id.encode_to_vec();
+        let back = pb::TableIdentifier::decode(bytes.as_slice()).unwrap();
+        assert_eq!(back.storage_options, opts);
+    }
+
+    #[tokio::test]
+    async fn test_table_identifier_from_dataset_roundtrip() {
+        let (dataset, _dir) = make_test_dataset().await;
+
+        let id = table_identifier_from_dataset(&dataset).await.unwrap();
+        assert_eq!(id.uri, dataset.uri());
+        assert_eq!(id.version, dataset.manifest.version);
+        assert!(id.serialized_manifest.is_none());
+
+        // Roundtrip: open the dataset back from the identifier
+        let back = open_dataset_from_table_identifier(&id).await.unwrap();
+        assert_eq!(back.uri(), dataset.uri());
+        assert_eq!(back.manifest.version, dataset.manifest.version);
+    }
+
+    #[tokio::test]
+    async fn test_table_identifier_with_manifest_roundtrip() {
+        let (dataset, _dir) = make_test_dataset().await;
+
+        let id = table_identifier_from_dataset_with_manifest(&dataset)
+            .await
+            .unwrap();
+        assert_eq!(id.uri, dataset.uri());
+        assert_eq!(id.version, dataset.manifest.version);
+        assert!(id.serialized_manifest.is_some());
+
+        // Verify the serialized manifest bytes decode
+        let manifest_bytes = id.serialized_manifest.as_ref().unwrap();
+        let _manifest_proto =
+            lance_table::format::pb::Manifest::decode(manifest_bytes.as_slice()).unwrap();
+
+        // Roundtrip: open the dataset back from the identifier (with manifest)
+        let back = open_dataset_from_table_identifier(&id).await.unwrap();
+        assert_eq!(back.uri(), dataset.uri());
+        assert_eq!(back.manifest.version, dataset.manifest.version);
+    }
+}
diff --git a/rust/lance/src/lib.rs b/rust/lance/src/lib.rs
index 934be0e519c..284e10a9b6f 100644
--- a/rust/lance/src/lib.rs
+++ b/rust/lance/src/lib.rs
@@ -72,7 +72,7 @@
 use arrow_schema::DataType;
 use dataset::builder::DatasetBuilder;
 pub use lance_core::datatypes;
-pub use lance_core::{Error, Result};
+pub use lance_core::{Error, Result, cache};
 use std::sync::LazyLock;
 
 pub mod arrow;
@@ -85,6 +85,11 @@ pub mod session;
 pub mod table;
 pub mod utils;
 
+pub mod pb {
+    #![allow(clippy::use_self)]
+    include!(concat!(env!("OUT_DIR"), "/lance.pb.rs"));
+}
+
 pub use blob::{BlobArrayBuilder, blob_field};
 pub use dataset::Dataset;
 use lance_index::vector::DIST_COL;
diff --git a/rust/lance/src/session.rs b/rust/lance/src/session.rs
index c67345fba32..b032cbaa15e 100644
--- a/rust/lance/src/session.rs
+++ b/rust/lance/src/session.rs
@@ -5,7 +5,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use deepsize::DeepSizeOf;
-use lance_core::cache::LanceCache;
+use lance_core::cache::{CacheBackend, LanceCache};
 use lance_core::{Error, Result};
 use lance_index::IndexType;
 use lance_io::object_store::ObjectStoreRegistry;
@@ -17,7 +17,7 @@ use crate::session::index_caches::GlobalIndexCache;
 use self::index_extension::IndexExtension;
 
 pub(crate) mod caches;
-pub(crate) mod index_caches;
+pub mod index_caches;
 pub(crate) mod index_extension;
 
 /// A user session holds the runtime state for a [`crate::Dataset`]
@@ -77,11 +77,7 @@ impl std::fmt::Debug for Session {
             )
             .field(
                 "file_metadata_cache",
-                &format!(
-                    "LanceCache(items={}, size_bytes={})",
-                    self.metadata_cache.0.approx_size(),
-                    self.metadata_cache.0.approx_size_bytes(),
-                ),
+                &format!("LanceCache(items={})", self.metadata_cache.0.approx_size(),),
             )
             .field(
                 "index_extensions",
@@ -114,6 +110,23 @@ impl Session {
         }
     }
 
+    /// Create a session with a custom index cache backend.
+    ///
+    /// The provided backend will be used for caching index data. The metadata
+    /// cache will use the default Moka-based backend with the given capacity.
+    pub fn with_index_cache_backend(
+        index_cache_backend: Arc<dyn CacheBackend>,
+        metadata_cache_size: usize,
+        store_registry: Arc<ObjectStoreRegistry>,
+    ) -> Self {
+        Self {
+            index_cache: GlobalIndexCache(LanceCache::with_backend(index_cache_backend)),
+            metadata_cache: GlobalMetadataCache(LanceCache::with_capacity(metadata_cache_size)),
+            index_extensions: HashMap::new(),
+            store_registry,
+        }
+    }
+
     /// Register a new index extension.
     ///
     /// A name can only be registered once per type of index extension.
@@ -182,6 +195,11 @@ impl Session {
         self.store_registry.clone()
     }
 
+    /// Get a reference to the raw metadata cache (for use in index reconstruction).
+    pub fn file_metadata_cache(&self) -> &LanceCache {
+        &self.metadata_cache.0
+    }
+
     /// Fetch statistics for the metadata cache
     pub async fn metadata_cache_stats(&self) -> lance_core::cache::CacheStats {
         self.metadata_cache.0.stats().await
@@ -206,7 +224,21 @@ impl Default for Session {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use lance_core::cache::UnsizedCacheKey;
     use lance_index::vector::VectorIndex;
+    use std::borrow::Cow;
+
+    struct TestUnsizedKey(&'static str);
+    impl UnsizedCacheKey for TestUnsizedKey {
+        type ValueType = dyn VectorIndex;
+        fn key(&self) -> Cow<'_, str> {
+            Cow::Borrowed(self.0)
+        }
+
+        fn type_name() -> &'static str {
+            "TestUnsized"
+        }
+    }
 
     #[tokio::test]
     async fn test_disable_index_cache() {
@@ -214,7 +246,7 @@ mod tests {
         assert!(
             no_cache
                 .index_cache
-                .get_unsized::<dyn VectorIndex>("abc")
+                .get_unsized_with_key(&TestUnsizedKey("abc"))
                 .await
                 .is_none()
         );
diff --git a/rust/lance/src/session/caches.rs b/rust/lance/src/session/caches.rs
index 67c684c98de..f8cdd520dad 100644
--- a/rust/lance/src/session/caches.rs
+++ b/rust/lance/src/session/caches.rs
@@ -34,12 +34,6 @@ impl GlobalMetadataCache {
         // This prevents collisions between different datasets.
         DSMetadataCache(self.0.with_key_prefix(uri))
     }
-
-    /// Create a file-specific metadata cache with the given prefix.
-    /// This is used by file readers and other components that need file-level caching.
-    pub(crate) fn file_metadata_cache(&self, path: &Path) -> LanceCache {
-        self.0.with_key_prefix(path.as_ref())
-    }
 }
 
 impl Clone for GlobalMetadataCache {
@@ -75,7 +69,6 @@ pub struct ManifestKey<'a> {
 
 impl CacheKey for ManifestKey<'_> {
     type ValueType = Manifest;
-
     fn key(&self) -> Cow<'_, str> {
         if let Some(e_tag) = self.e_tag {
             Cow::Owned(format!("manifest/{}/{}", self.version, e_tag))
@@ -83,6 +76,9 @@ impl CacheKey for ManifestKey<'_> {
             Cow::Owned(format!("manifest/{}", self.version))
         }
     }
+    fn type_name() -> &'static str {
+        "Manifest"
+    }
 }
 
 #[derive(Debug)]
@@ -92,10 +88,12 @@ pub struct TransactionKey {
 
 impl CacheKey for TransactionKey {
     type ValueType = Transaction;
-
     fn key(&self) -> Cow<'_, str> {
         Cow::Owned(format!("txn/{}", self.version))
     }
+    fn type_name() -> &'static str {
+        "Transaction"
+    }
 }
 
 #[derive(Debug)]
@@ -106,7 +104,6 @@ pub struct DeletionFileKey<'a> {
 
 impl CacheKey for DeletionFileKey<'_> {
     type ValueType = DeletionVector;
-
     fn key(&self) -> Cow<'_, str> {
         Cow::Owned(format!(
             "deletion/{}/{}/{}/{}",
@@ -116,6 +113,9 @@ impl CacheKey for DeletionFileKey<'_> {
             self.deletion_file.file_type.suffix()
         ))
     }
+    fn type_name() -> &'static str {
+        "DeletionVector"
+    }
 }
 
 #[derive(Debug)]
@@ -125,10 +125,12 @@ pub struct RowAddrMaskKey {
 
 impl CacheKey for RowAddrMaskKey {
     type ValueType = RowAddrMask;
-
     fn key(&self) -> Cow<'_, str> {
         Cow::Owned(format!("row_addr_mask/{}", self.version))
     }
+    fn type_name() -> &'static str {
+        "RowAddrMask"
+    }
 }
 
 #[derive(Debug)]
@@ -138,10 +140,12 @@ pub struct RowIdIndexKey {
 
 impl CacheKey for RowIdIndexKey {
     type ValueType = RowIdIndex;
-
     fn key(&self) -> Cow<'_, str> {
         Cow::Owned(format!("row_id_index/{}", self.version))
     }
+    fn type_name() -> &'static str {
+        "RowIdIndex"
+    }
 }
 
 #[derive(Debug)]
@@ -151,10 +155,12 @@ pub struct RowIdSequenceKey {
 
 impl CacheKey for RowIdSequenceKey {
     type ValueType = RowIdSequence;
-
     fn key(&self) -> Cow<'_, str> {
         Cow::Owned(format!("row_id_sequence/{}", self.fragment_id))
     }
+    fn type_name() -> &'static str {
+        "RowIdSequence"
+    }
 }
 
 impl DSMetadataCache {
diff --git a/rust/lance/src/session/index_caches.rs b/rust/lance/src/session/index_caches.rs
index d9578d43112..3ae777880aa 100644
--- a/rust/lance/src/session/index_caches.rs
+++ b/rust/lance/src/session/index_caches.rs
@@ -88,6 +88,10 @@ impl CacheKey for FragReuseIndexKey<'_> {
     fn key(&self) -> Cow<'_, str> {
         Cow::Owned(format!("frag_reuse/{}", self.uuid))
     }
+
+    fn type_name() -> &'static str {
+        "FragReuseIndex"
+    }
 }
 
 #[derive(Debug)]
@@ -101,6 +105,14 @@ impl CacheKey for IndexMetadataKey {
     fn key(&self) -> Cow<'_, str> {
         Cow::Owned(self.version.to_string())
     }
+
+    fn type_name() -> &'static str {
+        "Vec<IndexMetadata>"
+    }
+
+    fn codec() -> Option<lance_core::cache::CacheCodec> {
+        Some(lance_table::format::index_metadata_codec())
+    }
 }
 
 pub struct ProstAny(pub Arc<prost_types::Any>);
@@ -128,4 +140,8 @@ impl CacheKey for ScalarIndexDetailsKey<'_> {
     fn key(&self) -> Cow<'_, str> {
         Cow::Owned(format!("type/{}", self.uuid))
     }
+
+    fn type_name() -> &'static str {
+        "ScalarIndexDetails"
+    }
 }
diff --git a/rust/lance/src/session/index_extension.rs b/rust/lance/src/session/index_extension.rs
index b500d52086c..2055f64e340 100644
--- a/rust/lance/src/session/index_extension.rs
+++ b/rust/lance/src/session/index_extension.rs
@@ -65,6 +65,7 @@ mod test {
         sync::{Arc, atomic::AtomicBool},
     };
 
+    use crate::index::DatasetIndexExt;
     use arrow_array::{Float32Array, RecordBatch, UInt32Array};
     use arrow_schema::Schema;
     use datafusion::execution::SendableRecordBatchStream;
@@ -75,8 +76,7 @@ mod test {
     use lance_file::version::LanceFileVersion;
     use lance_index::vector::v3::subindex::SubIndexType;
     use lance_index::{
-        DatasetIndexExt, INDEX_FILE_NAME, INDEX_METADATA_SCHEMA_KEY, Index, IndexMetadata,
-        IndexType,
+        INDEX_FILE_NAME, INDEX_METADATA_SCHEMA_KEY, Index, IndexMetadata, IndexType,
         vector::{Query, hnsw::VECTOR_ID_FIELD},
     };
     use lance_index::{
diff --git a/rust/lance/src/utils/future.rs b/rust/lance/src/utils/future.rs
index e43f2055111..595d5c2a297 100644
--- a/rust/lance/src/utils/future.rs
+++ b/rust/lance/src/utils/future.rs
@@ -16,17 +16,6 @@ use tracing::Instrument;
 pub struct SharedPrerequisite<T: Clone>(Arc<AsyncCell<std::result::Result<T, String>>>);
 
 impl<T: Clone> SharedPrerequisite<T> {
-    /// Asynchronously get a cloned copy of the output
-    ///
-    /// If the child task failed then a PrerequisiteFailed error is raised.
-    #[allow(dead_code)]
-    pub async fn get_fut(&self) -> crate::Result<T> {
-        self.0
-            .get()
-            .await
-            .map_err(|err| crate::Error::prerequisite_failed(err))
-    }
-
     /// Synchronously get a cloned copy of the cached output
     ///
     /// Must be called after a call to `wait_ready`
diff --git a/rust/lance/src/utils/test.rs b/rust/lance/src/utils/test.rs
index f3b037aa02e..ae6d2a67d4b 100644
--- a/rust/lance/src/utils/test.rs
+++ b/rust/lance/src/utils/test.rs
@@ -52,7 +52,6 @@ impl TestDatasetGenerator {
     /// Set the seed for the random number generator.
     ///
     /// If not set, a random seed will be generated on each call to [`Self::make_hostile`].
-    #[allow(dead_code)]
     pub fn seed(mut self, seed: u64) -> Self {
         self.seed = Some(seed);
         self
@@ -263,7 +262,7 @@ fn field_structure(fragment: &Fragment) -> Vec<Vec<i32>> {
     fragment
         .files
         .iter()
-        .map(|file| file.fields.clone())
+        .map(|file| file.fields.to_vec())
         .collect::<Vec<_>>()
 }
 
diff --git a/rust/lance/tests/query/inverted.rs b/rust/lance/tests/query/inverted.rs
index 63b36b6a823..4215292ca7a 100644
--- a/rust/lance/tests/query/inverted.rs
+++ b/rust/lance/tests/query/inverted.rs
@@ -7,10 +7,11 @@ use arrow_array::{ArrayRef, Int32Array, RecordBatch, StringArray, UInt32Array};
 use lance::Dataset;
 use lance::dataset::scanner::ColumnOrdering;
 use lance::dataset::{InsertBuilder, WriteParams};
+use lance::index::DatasetIndexExt;
+use lance_index::IndexType;
+use lance_index::scalar::inverted::Language;
 use lance_index::scalar::inverted::query::{FtsQuery, PhraseQuery};
 use lance_index::scalar::{FullTextSearchQuery, InvertedIndexParams};
-use lance_index::{DatasetIndexExt, IndexType};
-use tantivy::tokenizer::Language;
 
 use super::{strip_score_column, test_fts, test_scan, test_take};
 use crate::utils::DatasetTestCases;
diff --git a/rust/lance/tests/query/primitives.rs b/rust/lance/tests/query/primitives.rs
index 6c72f66bd23..65fa6f4e4d3 100644
--- a/rust/lance/tests/query/primitives.rs
+++ b/rust/lance/tests/query/primitives.rs
@@ -13,8 +13,9 @@ use lance::Dataset;
 use lance::dataset::WriteParams;
 use lance::dataset::optimize::{CompactionOptions, compact_files};
 
+use lance::index::DatasetIndexExt;
 use lance_datagen::{ArrayGeneratorExt, RowCount, array, gen_batch};
-use lance_index::{DatasetIndexExt, IndexType};
+use lance_index::IndexType;
 
 use super::{test_filter, test_scan, test_take};
 use crate::utils::DatasetTestCases;
diff --git a/rust/lance/tests/resource_test/mod.rs b/rust/lance/tests/resource_test/mod.rs
index 80ec1ab9d20..15951e92fbd 100644
--- a/rust/lance/tests/resource_test/mod.rs
+++ b/rust/lance/tests/resource_test/mod.rs
@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 mod utils;
+mod vector;
 mod write;
diff --git a/rust/lance/tests/resource_test/vector.rs b/rust/lance/tests/resource_test/vector.rs
new file mode 100644
index 00000000000..dcd299284ae
--- /dev/null
+++ b/rust/lance/tests/resource_test/vector.rs
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::sync::Arc;
+
+use super::utils::AllocTracker;
+use all_asserts::assert_le;
+use arrow_array::{Array, ArrayRef, RecordBatch, RecordBatchIterator, types::Float32Type};
+use arrow_schema::{DataType, Field, Schema};
+use lance::Dataset;
+use lance::dataset::WriteParams;
+use lance::index::vector::utils::maybe_sample_training_data;
+use lance_arrow::FixedSizeListArrayExt;
+
+#[tokio::test]
+async fn test_nullable_fragment_sampling_memory_stays_bounded() {
+    let dim = 1024;
+    let num_fragments = 4;
+    let rows_per_fragment = 8_192;
+    let sample_size = 32;
+    let schema = Arc::new(Schema::new(vec![Field::new(
+        "vec",
+        DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), dim),
+        true,
+    )]));
+
+    let batches = (0..num_fragments)
+        .map(|seed| {
+            let values = lance_testing::datagen::generate_random_array_with_seed::<Float32Type>(
+                rows_per_fragment * dim as usize,
+                [seed as u8; 32],
+            );
+            let vectors = Arc::new(
+                arrow_array::FixedSizeListArray::try_new_from_values(values, dim).unwrap(),
+            ) as ArrayRef;
+            RecordBatch::try_new(schema.clone(), vec![vectors]).unwrap()
+        })
+        .collect::<Vec<_>>();
+
+    let tmp_dir = tempfile::tempdir().unwrap();
+    let uri = tmp_dir.path().to_str().unwrap();
+    let dataset = Dataset::write(
+        RecordBatchIterator::new(batches.into_iter().map(Ok), schema),
+        uri,
+        Some(WriteParams {
+            max_rows_per_file: rows_per_fragment,
+            max_rows_per_group: rows_per_fragment,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+    let fragment_ids = dataset
+        .get_fragments()
+        .into_iter()
+        .take(2)
+        .map(|fragment| fragment.id() as u32)
+        .collect::<Vec<_>>();
+
+    let alloc_tracker = AllocTracker::new();
+    let training_data = {
+        let _guard = alloc_tracker.enter();
+        maybe_sample_training_data(&dataset, "vec", sample_size, Some(&fragment_ids))
+            .await
+            .unwrap()
+    };
+    let stats = alloc_tracker.stats();
+
+    assert_eq!(training_data.len(), sample_size);
+
+    // A full scan of the selected fragments would need at least:
+    // 2 fragments * 8192 rows * 1024 dims * 4 bytes = 64 MiB
+    // Keep a generous ceiling well below that lower bound so the test remains
+    // stable while still catching regressions back to eager materialization.
+    assert_le!(
+        stats.max_bytes_allocated,
+        24 * 1024 * 1024,
+        "nullable fragment sampling allocated too much memory: {:?}",
+        stats
+    );
+}
diff --git a/rust/lance/tests/utils/mod.rs b/rust/lance/tests/utils/mod.rs
index c407c62c03b..b8a034a50b0 100644
--- a/rust/lance/tests/utils/mod.rs
+++ b/rust/lance/tests/utils/mod.rs
@@ -7,6 +7,7 @@ use std::sync::Arc;
 
 use arrow_array::{ArrayRef, Int32Array, RecordBatch};
 use futures::FutureExt;
+use lance::index::DatasetIndexExt;
 use lance::index::vector::VectorIndexParams;
 use lance::{
     Dataset,
@@ -17,7 +18,7 @@ use lance_index::vector::hnsw::builder::HnswBuildParams;
 use lance_index::vector::ivf::IvfBuildParams;
 use lance_index::vector::pq::PQBuildParams;
 use lance_index::vector::sq::builder::SQBuildParams;
-use lance_index::{DatasetIndexExt, IndexParams, IndexType};
+use lance_index::{IndexParams, IndexType};
 use lance_linalg::distance::{DistanceType, MetricType};
 
 #[derive(Clone, Copy, Debug)]
diff --git a/test_data/pre_file_sizes/datagen.py b/test_data/pre_file_sizes/datagen.py
new file mode 100644
index 00000000000..0967118de02
--- /dev/null
+++ b/test_data/pre_file_sizes/datagen.py
@@ -0,0 +1,47 @@
+"""
+Generate test data to simulate a dataset created before the `files` field was added
+to IndexMetadata. This tests backward compatibility for the files field.
+
+To generate this test data:
+1. Check out Lance before the file sizes feature was added (or use a released version)
+2. Run: pip install -e python/
+3. Run: python test_data/pre_file_sizes/datagen.py
+
+The dataset will be created without the `files` field in IndexMetadata,
+which simulates what an older dataset would look like.
+"""
+
+import lance
+import pyarrow as pa
+import shutil
+import os
+
+assert lance.__version__ == "2.0.0-beta.1"
+
+# Create output directory
+OUTPUT_DIR = os.path.join(os.path.dirname(__file__), "index_without_file_sizes")
+if os.path.exists(OUTPUT_DIR):
+    shutil.rmtree(OUTPUT_DIR)
+
+# Create a simple dataset with a scalar index
+data = pa.table(
+    {
+        "id": pa.array(range(100)),
+        "values": pa.array([f"value_{i}" for i in range(100)]),
+    }
+)
+
+# Write dataset
+dataset = lance.write_dataset(data, OUTPUT_DIR)
+
+# Create a scalar index (BTree) using the correct method
+dataset.create_scalar_index("values", index_type="BTREE")
+
+print(f"Created dataset at {OUTPUT_DIR}")
+print(f"Dataset version: {dataset.version}")
+
+# Verify
+dataset = lance.dataset(OUTPUT_DIR)
+indices = dataset.list_indices()
+print(f"Indices: {indices}")
+print("\nTest data created successfully!")
diff --git a/test_data/pre_file_sizes/index_without_file_sizes/_indices/be61629d-904e-433f-836c-1e586be2d5c6/page_data.lance b/test_data/pre_file_sizes/index_without_file_sizes/_indices/be61629d-904e-433f-836c-1e586be2d5c6/page_data.lance
new file mode 100644
index 00000000000..6cc33bab62f
Binary files /dev/null and b/test_data/pre_file_sizes/index_without_file_sizes/_indices/be61629d-904e-433f-836c-1e586be2d5c6/page_data.lance differ
diff --git a/test_data/pre_file_sizes/index_without_file_sizes/_indices/be61629d-904e-433f-836c-1e586be2d5c6/page_lookup.lance b/test_data/pre_file_sizes/index_without_file_sizes/_indices/be61629d-904e-433f-836c-1e586be2d5c6/page_lookup.lance
new file mode 100644
index 00000000000..7753f7a2ca5
Binary files /dev/null and b/test_data/pre_file_sizes/index_without_file_sizes/_indices/be61629d-904e-433f-836c-1e586be2d5c6/page_lookup.lance differ
diff --git a/test_data/pre_file_sizes/index_without_file_sizes/_transactions/0-20f83903-0474-45d5-a4c8-58f793a0f577.txn b/test_data/pre_file_sizes/index_without_file_sizes/_transactions/0-20f83903-0474-45d5-a4c8-58f793a0f577.txn
new file mode 100644
index 00000000000..24fa590ee26
Binary files /dev/null and b/test_data/pre_file_sizes/index_without_file_sizes/_transactions/0-20f83903-0474-45d5-a4c8-58f793a0f577.txn differ
diff --git a/test_data/pre_file_sizes/index_without_file_sizes/_transactions/1-ed397f26-dd73-49f3-8781-3828da94fcea.txn b/test_data/pre_file_sizes/index_without_file_sizes/_transactions/1-ed397f26-dd73-49f3-8781-3828da94fcea.txn
new file mode 100644
index 00000000000..d6c0d3f8d75
Binary files /dev/null and b/test_data/pre_file_sizes/index_without_file_sizes/_transactions/1-ed397f26-dd73-49f3-8781-3828da94fcea.txn differ
diff --git a/test_data/pre_file_sizes/index_without_file_sizes/_versions/1.manifest b/test_data/pre_file_sizes/index_without_file_sizes/_versions/1.manifest
new file mode 100644
index 00000000000..1a4263db926
Binary files /dev/null and b/test_data/pre_file_sizes/index_without_file_sizes/_versions/1.manifest differ
diff --git a/test_data/pre_file_sizes/index_without_file_sizes/_versions/2.manifest b/test_data/pre_file_sizes/index_without_file_sizes/_versions/2.manifest
new file mode 100644
index 00000000000..b5ccc5d7ed7
Binary files /dev/null and b/test_data/pre_file_sizes/index_without_file_sizes/_versions/2.manifest differ
diff --git a/test_data/pre_file_sizes/index_without_file_sizes/data/1111100111100111100110009777b54c5f953114705fd03f9b.lance b/test_data/pre_file_sizes/index_without_file_sizes/data/1111100111100111100110009777b54c5f953114705fd03f9b.lance
new file mode 100644
index 00000000000..e729bb5641f
Binary files /dev/null and b/test_data/pre_file_sizes/index_without_file_sizes/data/1111100111100111100110009777b54c5f953114705fd03f9b.lance differ
diff --git a/test_debug.py b/test_debug.py
new file mode 100644
index 00000000000..e42798d5ff3
--- /dev/null
+++ b/test_debug.py
@@ -0,0 +1,50 @@
+import copy
+import uuid
+import lance
+import pyarrow as pa
+from lance.namespace import DirectoryNamespace
+
+CONFIG = {
+    'allow_http': 'true',
+    'aws_access_key_id': 'ACCESS_KEY',
+    'aws_secret_access_key': 'SECRET_KEY',
+    'aws_endpoint': 'http://localhost:4566',
+    'aws_region': 'us-east-1',
+}
+
+storage_options = copy.deepcopy(CONFIG)
+storage_options_with_refresh = dict(storage_options)
+storage_options_with_refresh['refresh_offset_millis'] = '1000'
+
+dir_props = {f'storage.{k}': v for k, v in storage_options_with_refresh.items()}
+dir_props['root'] = 's3://lance-namespace-integtest/namespace_root'
+dir_props['ops_metrics_enabled'] = 'true'
+dir_props['vend_input_storage_options'] = 'true'
+dir_props['vend_input_storage_options_refresh_interval_millis'] = '3600000'
+
+namespace = DirectoryNamespace(**dir_props)
+
+table1 = pa.Table.from_pylist([{'a': 1, 'b': 2}])
+table_name = 'debug_print3_' + uuid.uuid4().hex
+table_id = ['test_ns', table_name]
+
+print('=== Creating table ===')
+ds = lance.write_dataset(
+    table1,
+    namespace_client=namespace,
+    table_id=table_id,
+    mode='create',
+    storage_options=storage_options,
+)
+print('=== Table created ===')
+print('Describe count after write:', namespace.retrieve_ops_metrics().get('describe_table', 0))
+
+print('')
+print('=== Opening dataset via lance.dataset() ===')
+ds_from_namespace = lance.dataset(
+    namespace_client=namespace,
+    table_id=table_id,
+    storage_options=storage_options,
+)
+print('=== Dataset opened ===')
+print('Describe count after lance.dataset():', namespace.retrieve_ops_metrics().get('describe_table', 0))